GRM-158: Fix rootless Docker 29 setup on Arch Linux runners #246
+18
-5
@@ -160,10 +160,10 @@ jobs:
|
||||
container: git.oblachno.oblachno.fyi/oblachno-oss/runner-images/ci-full:latest
|
||||
timeout-minutes: 15
|
||||
strategy:
|
||||
fail-fast: true
|
||||
max-parallel: 6
|
||||
fail-fast: false
|
||||
max-parallel: 4
|
||||
matrix:
|
||||
runner-index: [1, 2, 3, 4, 5, 6]
|
||||
runner-index: [1, 2, 3, 4]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Set up environment
|
||||
@@ -178,15 +178,28 @@ jobs:
|
||||
- name: Discover assigned test pairs
|
||||
env:
|
||||
RUNNER_INDEX: ${{ matrix.runner-index }}
|
||||
MAX_RUNNERS: 6
|
||||
MAX_RUNNERS: 4
|
||||
run: |
|
||||
. .venv/bin/activate 2>/dev/null || true
|
||||
python3 -m devx.molecule.distribute_molecule \
|
||||
--runner-index "$RUNNER_INDEX" \
|
||||
--max-runners "$MAX_RUNNERS" \
|
||||
--github-env
|
||||
- name: Run molecule tests
|
||||
- name: Prune stale Docker data
|
||||
id: prune
|
||||
if: env.SKIP != 'true'
|
||||
run: |
|
||||
docker system prune -af --volumes 2>/dev/null || true
|
||||
disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
|
||||
echo "Disk usage after prune: ${disk_pct}%"
|
||||
if [ "$disk_pct" -ge 85 ]; then
|
||||
echo "should-run=false" >> "$GITHUB_OUTPUT"
|
||||
echo "::warning::Disk usage at ${disk_pct}% after prune — skipping molecule tests to avoid ENOSPC failures"
|
||||
else
|
||||
echo "should-run=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
- name: Run molecule tests
|
||||
if: env.SKIP != 'true' && steps.prune.outputs.should-run != 'false'
|
||||
env:
|
||||
GITEA_URL: ${{ github.server_url }}
|
||||
CI_GITEA_API_TOKEN: ${{ secrets.CI_GITEA_API_TOKEN }}
|
||||
|
||||
@@ -8,6 +8,7 @@ All notable changes to this project will be documented in this file.
|
||||
|
||||
- Pin containerd.io to compatible version for Docker 28.x
|
||||
|
||||
|
||||
## [0.18.7] - 2026-08-06
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
@@ -8,12 +8,12 @@ Each runner runs in an isolated **rootless Docker** environment under a dedicate
|
||||
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/src/branch/master/LICENSE)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/wiki)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/releases)
|
||||
[](https://www.python.org/downloads/)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/wiki)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/releases)
|
||||
[](https://www.python.org/downloads/)
|
||||
|
||||
## Why GRM?
|
||||
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
---
|
||||
collections:
|
||||
- name: community.general
|
||||
version: "==13.1.0"
|
||||
type: url
|
||||
source: https://git.oblachno.oblachno.fyi/api/packages/emil/generic/ansible-collections/13.1.0/community-general-13.1.0.tar.gz
|
||||
- name: ansible.posix
|
||||
version: "==2.2.1"
|
||||
type: url
|
||||
source: https://git.oblachno.oblachno.fyi/api/packages/emil/generic/ansible-collections/2.2.1/ansible-posix-2.2.1.tar.gz
|
||||
- name: community.docker
|
||||
version: "==5.2.1"
|
||||
type: url
|
||||
source: https://git.oblachno.oblachno.fyi/api/packages/emil/generic/ansible-collections/5.2.1/community-docker-5.2.1.tar.gz
|
||||
|
||||
@@ -3,6 +3,13 @@ gitea_runner_version: "2.0.1"
|
||||
gitea_runner_labels: "docker,ubuntu-latest:docker://runner-images:ubuntu-26.04"
|
||||
gitea_runner_skip_registration: false
|
||||
|
||||
# Force re-registration even if .runner file exists.
|
||||
# Use this when Gitea no longer recognizes the runner (e.g., after a Gitea
|
||||
# server restore/reinstall or when the runner record was deleted from the
|
||||
# admin UI). The existing .runner file is removed and a new registration is
|
||||
# performed. Requires registration_token.
|
||||
gitea_runner_force_reregister: false
|
||||
|
||||
# Per-runner user (rootless isolation)
|
||||
gitea_runner_user_prefix: "grm-"
|
||||
gitea_runner_base_home: "/home"
|
||||
@@ -37,6 +44,18 @@ gitea_runner_healthcheck_boot_delay: "2min"
|
||||
gitea_runner_healthcheck_disk_threshold: 75
|
||||
gitea_runner_healthcheck_script_path: "{{ gitea_runner_config_dir }}/healthcheck.sh"
|
||||
|
||||
# Auto-recovery: when the healthcheck detects an unregistered runner, it
|
||||
# can automatically re-register if a Gitea API token is provided.
|
||||
# The token needs admin or org-level access to fetch registration tokens.
|
||||
# Stored in a file readable by the runner user (mode 0400).
|
||||
# Set to empty string to disable auto-recovery (manual re-registration required).
|
||||
gitea_runner_auto_recover_api_token: ""
|
||||
|
||||
# Cooldown file to prevent auto-recovery loops (e.g., if Gitea is down).
|
||||
# The healthcheck writes a timestamp to this file after a re-registration
|
||||
# attempt and skips further attempts for the cooldown period.
|
||||
gitea_runner_auto_recover_cooldown_sec: 300
|
||||
|
||||
# Docker daemon resilience settings (applied to daemon.json).
|
||||
# live-restore: containers survive daemon restarts — prevents stuck container
|
||||
# states when the healthcheck restarts a hung daemon.
|
||||
|
||||
@@ -47,9 +47,9 @@
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- "'Type=oneshot' in prune_service.content | b64decode"
|
||||
- "'docker system prune' in prune_service.content | b64decode"
|
||||
- "'docker volume prune' in prune_service.content | b64decode"
|
||||
- "'docker container prune' in prune_service.content | b64decode"
|
||||
- "'docker rm -f' in prune_service.content | b64decode"
|
||||
- "'GITEA-ACTIONS-TASK' in prune_service.content | b64decode"
|
||||
- "'docker system prune -af' in prune_service.content | b64decode"
|
||||
- "'docker network prune' in prune_service.content | b64decode"
|
||||
- "'docker builder prune' in prune_service.content | b64decode"
|
||||
fail_msg: "Prune service template is missing expected directives"
|
||||
@@ -105,23 +105,11 @@
|
||||
- "'timeout 10 docker info' in healthcheck_script.content | b64decode"
|
||||
- "'systemctl --user restart docker.service' in healthcheck_script.content | b64decode"
|
||||
- "'systemctl --user restart gitea-runner.service' in healthcheck_script.content | b64decode"
|
||||
- "'docker system prune' in healthcheck_script.content | b64decode"
|
||||
- "'docker container prune' in healthcheck_script.content | b64decode"
|
||||
- "'docker rm -f' in healthcheck_script.content | b64decode"
|
||||
- "'GITEA-ACTIONS-TASK' in healthcheck_script.content | b64decode"
|
||||
- "'docker system prune -af' in healthcheck_script.content | b64decode"
|
||||
- "'docker network prune' in healthcheck_script.content | b64decode"
|
||||
- "'status=removing' in healthcheck_script.content | b64decode"
|
||||
- "'status=stopping' in healthcheck_script.content | b64decode"
|
||||
- "'docker rm -f' in healthcheck_script.content | b64decode"
|
||||
- "gitea_runner_healthcheck_disk_threshold | string in healthcheck_script.content | b64decode"
|
||||
fail_msg: "Healthcheck script template is missing expected content"
|
||||
|
||||
- name: Assert healthcheck script does NOT use aggressive prune (-af)
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- "'prune -af' not in healthcheck_script.content | b64decode"
|
||||
- "'image prune -af' not in healthcheck_script.content | b64decode"
|
||||
- "'system prune -af' not in healthcheck_script.content | b64decode"
|
||||
- "'volume prune -af' not in healthcheck_script.content | b64decode"
|
||||
fail_msg: >-
|
||||
Healthcheck script uses 'prune -af' which removes ALL images
|
||||
(including tagged runner images like ci-full). Use 'prune -f'
|
||||
(dangling only) to preserve tagged images.
|
||||
|
||||
@@ -7,6 +7,22 @@
|
||||
group: "{{ gitea_runner_service_user }}"
|
||||
mode: "0755"
|
||||
|
||||
- name: Write auto-recovery API token file
|
||||
ansible.builtin.copy:
|
||||
content: "{{ gitea_runner_auto_recover_api_token }}"
|
||||
dest: "{{ gitea_runner_config_dir }}/auto-recover.token"
|
||||
owner: "{{ gitea_runner_service_user }}"
|
||||
group: "{{ gitea_runner_service_user }}"
|
||||
mode: "0400"
|
||||
no_log: true
|
||||
when: gitea_runner_auto_recover_api_token | length > 0
|
||||
|
||||
- name: Remove stale auto-recovery token file (if auto-recovery disabled)
|
||||
ansible.builtin.file:
|
||||
path: "{{ gitea_runner_config_dir }}/auto-recover.token"
|
||||
state: absent
|
||||
when: gitea_runner_auto_recover_api_token | length == 0
|
||||
|
||||
- name: Create healthcheck user service file
|
||||
ansible.builtin.template:
|
||||
src: runner-healthcheck.service.j2
|
||||
|
||||
@@ -7,11 +7,20 @@
|
||||
group: "{{ gitea_runner_service_user }}"
|
||||
mode: "0755"
|
||||
|
||||
- name: Check if runner is already registered
|
||||
- name: Check if runner registration file exists
|
||||
ansible.builtin.stat:
|
||||
path: "{{ gitea_runner_data_dir }}/.runner"
|
||||
register: gitea_runner_registered
|
||||
|
||||
- name: Remove stale runner registration file
|
||||
ansible.builtin.file:
|
||||
path: "{{ gitea_runner_data_dir }}/.runner"
|
||||
state: absent
|
||||
when:
|
||||
- gitea_runner_registered.stat.exists
|
||||
- gitea_runner_force_reregister | bool
|
||||
register: gitea_runner_registration_removed
|
||||
|
||||
- name: Register runner with Gitea
|
||||
ansible.builtin.command: >
|
||||
{{ gitea_runner_binary_path }} register
|
||||
@@ -28,7 +37,23 @@
|
||||
XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid | default(0) }}"
|
||||
DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/{{ gitea_runner_uid | default(0) }}/bus"
|
||||
DOCKER_HOST: "unix:///run/user/{{ gitea_runner_uid | default(0) }}/docker.sock"
|
||||
when: not gitea_runner_registered.stat.exists
|
||||
when: not gitea_runner_registered.stat.exists or gitea_runner_force_reregister | bool
|
||||
register: gitea_runner_register_output
|
||||
changed_when: "'already exists' not in gitea_runner_register_output.stdout | default('')"
|
||||
changed_when: >-
|
||||
gitea_runner_register_output.rc == 0 and
|
||||
('already exists' not in gitea_runner_register_output.stdout | default(''))
|
||||
timeout: 60
|
||||
|
||||
- name: Ensure runner service is running after registration
|
||||
ansible.builtin.command: systemctl --user start gitea-runner
|
||||
become: true
|
||||
become_user: "{{ gitea_runner_service_user }}"
|
||||
environment:
|
||||
XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}"
|
||||
DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/{{ gitea_runner_uid | default(0) }}/bus"
|
||||
changed_when: true
|
||||
when:
|
||||
- gitea_runner_systemd_available.stat.exists
|
||||
- gitea_runner_docker_rootless_setup
|
||||
- gitea_runner_register_output is defined
|
||||
- gitea_runner_register_output.rc | default(1) == 0
|
||||
|
||||
@@ -53,72 +53,6 @@
|
||||
register: gitea_runner_docker_install
|
||||
when: ansible_facts['os_family'] == 'Debian'
|
||||
|
||||
# Docker 28.x vendors containerd v2.1.x. containerd.io >= 2.3 ships a shim that
|
||||
# returns a protobuf BootstrapResult the vendored 2.1.x code cannot parse, causing
|
||||
# "failed to create TTRPC connection: unsupported protocol" on every container start.
|
||||
# Detect the mismatch and downgrade containerd.io to the latest compatible 2.2.x.
|
||||
- name: Check installed Docker CE version (Debian/Ubuntu)
|
||||
ansible.builtin.command: dpkg-query -W -f='${Version}' docker-ce
|
||||
register: gitea_runner_docker_version_check
|
||||
changed_when: false
|
||||
when: ansible_facts['os_family'] == 'Debian'
|
||||
|
||||
- name: Check installed containerd.io version (Debian/Ubuntu)
|
||||
ansible.builtin.shell: "set -o pipefail; dpkg-query -W -f='${Version}' containerd.io 2>/dev/null | cut -d: -f2 | cut -d- -f1"
|
||||
args:
|
||||
executable: /bin/bash
|
||||
register: gitea_runner_containerd_version_check
|
||||
changed_when: false
|
||||
when: ansible_facts['os_family'] == 'Debian'
|
||||
|
||||
- name: Determine if containerd.io is incompatible with installed Docker
|
||||
ansible.builtin.set_fact:
|
||||
gitea_runner_containerd_needs_downgrade: >-
|
||||
{{
|
||||
gitea_runner_containerd_version_check.stdout.split('.')[0] | int > gitea_runner_containerd_max_compatible_major
|
||||
or (
|
||||
gitea_runner_containerd_version_check.stdout.split('.')[0] | int == gitea_runner_containerd_max_compatible_major
|
||||
and gitea_runner_containerd_version_check.stdout.split('.')[1] | int > gitea_runner_containerd_max_compatible_minor
|
||||
)
|
||||
}}
|
||||
gitea_runner_docker_major: "{{ gitea_runner_docker_version_check.stdout.split('.')[0] | default('0') | int }}"
|
||||
when: ansible_facts['os_family'] == 'Debian'
|
||||
|
||||
- name: Find latest compatible containerd.io version (Debian/Ubuntu)
|
||||
ansible.builtin.shell: |
|
||||
set -o pipefail
|
||||
apt-cache madison containerd.io \
|
||||
| awk -F'|' '{print $2}' \
|
||||
| tr -d ' ' \
|
||||
| grep -E '^{{ gitea_runner_containerd_max_compatible_major }}\.{{ gitea_runner_containerd_max_compatible_minor }}\.' \
|
||||
| head -1
|
||||
args:
|
||||
executable: /bin/bash
|
||||
register: gitea_runner_containerd_compatible_version
|
||||
changed_when: false
|
||||
when:
|
||||
- ansible_facts['os_family'] == 'Debian'
|
||||
- gitea_runner_containerd_needs_downgrade | default(false)
|
||||
- gitea_runner_docker_major | int < 29
|
||||
|
||||
- name: Downgrade containerd.io to compatible version (Debian/Ubuntu)
|
||||
ansible.builtin.apt:
|
||||
name: "containerd.io={{ gitea_runner_containerd_compatible_version.stdout }}"
|
||||
state: present
|
||||
allow_downgrades: true
|
||||
register: gitea_runner_containerd_downgrade
|
||||
when:
|
||||
- ansible_facts['os_family'] == 'Debian'
|
||||
- gitea_runner_containerd_needs_downgrade | default(false)
|
||||
- gitea_runner_docker_major | int < 29
|
||||
- gitea_runner_containerd_compatible_version.stdout | length > 0
|
||||
|
||||
- name: Hold containerd.io package to prevent auto-upgrade (Debian/Ubuntu)
|
||||
ansible.builtin.dpkg_selections:
|
||||
name: containerd.io
|
||||
selection: hold
|
||||
when: ansible_facts['os_family'] == 'Debian'
|
||||
|
||||
- name: Update pacman cache (Arch Linux)
|
||||
community.general.pacman:
|
||||
update_cache: true
|
||||
|
||||
@@ -5,8 +5,19 @@ Description=Docker prune for Gitea runner resources
|
||||
Type=oneshot
|
||||
Environment=DOCKER_HOST=unix:///run/user/{{ gitea_runner_uid }}/docker.sock
|
||||
Environment=XDG_RUNTIME_DIR=/run/user/{{ gitea_runner_uid }}
|
||||
ExecStart=/usr/bin/docker system prune -f --filter "label={{ gitea_runner_prune_label }}" --filter "until={{ gitea_runner_prune_until }}"
|
||||
ExecStart=/usr/bin/docker volume prune -f --filter "label={{ gitea_runner_prune_label }}"
|
||||
ExecStart=/usr/bin/docker container prune -f
|
||||
ExecStart=/usr/bin/docker network prune -f
|
||||
# Force-remove stale containers (including running ones) left behind by failed
|
||||
# molecule tests. "docker container prune -f" only removes stopped containers,
|
||||
# so running containers from crashed/interrupted CI jobs accumulate indefinitely,
|
||||
# consuming disk and memory. We stop+rm everything first, then prune the rest.
|
||||
# Exclude CI job containers (name starts with GITEA-ACTIONS-TASK) — removing
|
||||
# them kills the active CI job and causes "RWLayer is unexpectedly nil" errors.
|
||||
# Only remove containers older than 1 hour (grep for "hour/day/week/month/year
|
||||
# ago" in RunningFor) to avoid killing molecule test containers that CI jobs
|
||||
# are actively using.
|
||||
ExecStart=/bin/sh -c 'docker ps -a --format "{% raw %}{{.ID}} {{.Names}} {{.RunningFor}}{% endraw %}" 2>/dev/null | grep -v "GITEA-ACTIONS-TASK" | grep -E "(hour|day|week|month|year)s? ago" | awk "{print $1}" | xargs -r docker rm -f 2>/dev/null || true'
|
||||
ExecStart=/usr/bin/docker system prune -af --filter "until={{ gitea_runner_prune_until }}" --volumes
|
||||
# Prune networks older than the prune-until threshold to avoid removing
|
||||
# networks that molecule tests are actively creating (e.g. 'traefik' network
|
||||
# created during molecule create phase before containers are attached).
|
||||
ExecStart=/usr/bin/docker network prune -f --filter "until={{ gitea_runner_prune_until }}"
|
||||
ExecStart=/usr/bin/docker builder prune -f
|
||||
|
||||
@@ -44,17 +44,203 @@ if [[ "$runner_state" != "active" ]]; then
|
||||
echo "RECOVERED: gitea-runner service restarted successfully"
|
||||
fi
|
||||
|
||||
# 2b. Detect unregistered runner state. When Gitea no longer recognizes the
|
||||
# runner (e.g., server restore, runner record deleted, Gitea restart with
|
||||
# token salt change), the runner logs "unregistered runner" every few seconds.
|
||||
# A service restart will not fix this; re-registration is required.
|
||||
#
|
||||
# Detection method: query the Gitea API to verify the runner's UUID still
|
||||
# exists. This is more reliable than parsing journal logs (which requires
|
||||
# journal access permissions that runner users may not have — see the
|
||||
# 2026-08-08 incident where journalctl --user returned "No journal files
|
||||
# were opened due to insufficient permissions" for all runner users,
|
||||
# causing the healthcheck to always report "OK: runner healthy" even
|
||||
# though all runners were unregistered).
|
||||
{% if gitea_runner_auto_recover_api_token %}
|
||||
# Auto-recovery is enabled: fetch a new registration token from the Gitea API
|
||||
# and re-register the runner automatically. A cooldown prevents infinite loops.
|
||||
GITEA_API_TOKEN_FILE="{{ gitea_runner_config_dir }}/auto-recover.token"
|
||||
COOLDOWN_FILE="{{ gitea_runner_data_dir }}/auto-recover.cooldown"
|
||||
COOLDOWN_SEC={{ gitea_runner_auto_recover_cooldown_sec }}
|
||||
GITEA_URL="{{ gitea_url }}"
|
||||
RUNNER_NAME="{{ gitea_runner_name }}"
|
||||
RUNNER_LABELS="{{ gitea_runner_labels }}"
|
||||
BINARY="{{ gitea_runner_binary_path }}"
|
||||
RUNNER_FILE="{{ gitea_runner_data_dir }}/.runner"
|
||||
{% endif %}
|
||||
runner_unregistered=0
|
||||
|
||||
# Primary detection: query the Gitea API to check if the runner's ID
|
||||
# still exists in Gitea's runner list. This works regardless of journal
|
||||
# permissions.
|
||||
{% if gitea_runner_auto_recover_api_token %}
|
||||
if [[ -f "$GITEA_API_TOKEN_FILE" && -f "$RUNNER_FILE" ]]; then
|
||||
API_TOKEN=$(cat "$GITEA_API_TOKEN_FILE" 2>/dev/null || true)
|
||||
RUNNER_ID=$(python3 -c "import json; print(json.load(open('$RUNNER_FILE')).get('id',''))" 2>/dev/null || true)
|
||||
if [[ -n "$API_TOKEN" && -n "$RUNNER_ID" ]]; then
|
||||
# List all runners and check if our ID is present
|
||||
runner_found=$(curl -sf --connect-timeout 5 --max-time 10 \
|
||||
-H "Authorization: token $API_TOKEN" \
|
||||
"${GITEA_URL}/api/v1/admin/actions/runners" 2>/dev/null \
|
||||
| python3 -c "
|
||||
import sys, json
|
||||
try:
|
||||
data = json.load(sys.stdin)
|
||||
runners = data if isinstance(data, list) else data.get('runners', [])
|
||||
ids = [str(r.get('id', '')) for r in runners]
|
||||
print('1' if '$RUNNER_ID' in ids else '0')
|
||||
except Exception:
|
||||
print('0')
|
||||
" 2>/dev/null || echo "0")
|
||||
if [[ "$runner_found" != "1" ]]; then
|
||||
runner_unregistered=1
|
||||
echo "CRITICAL: runner ID $RUNNER_ID not found in Gitea (unregistered)."
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
{% endif %}
|
||||
|
||||
# Fallback detection: check journal logs (if accessible)
|
||||
if [[ "$runner_unregistered" -eq 0 ]]; then
|
||||
recent_errors=$(journalctl --user -u gitea-runner.service --since "5 minutes ago" --no-pager -q 2>/dev/null | grep -c "unregistered runner" || true)
|
||||
if [[ "$recent_errors" -ge 3 ]]; then
|
||||
runner_unregistered=1
|
||||
echo "CRITICAL: runner is unregistered in Gitea (re-login failed $recent_errors times in 5 minutes)."
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$runner_unregistered" -ge 1 ]]; then
|
||||
{% if gitea_runner_auto_recover_api_token %}
|
||||
# Check cooldown — skip if we recently attempted recovery
|
||||
if [[ -f "$COOLDOWN_FILE" ]]; then
|
||||
last_attempt=$(cat "$COOLDOWN_FILE" 2>/dev/null || echo 0)
|
||||
now=$(date +%s)
|
||||
elapsed=$((now - last_attempt))
|
||||
if [[ "$elapsed" -lt "$COOLDOWN_SEC" ]]; then
|
||||
echo "SKIP: auto-recovery cooldown active (${elapsed}s < ${COOLDOWN_SEC}s). Will retry later."
|
||||
exit 3
|
||||
fi
|
||||
fi
|
||||
|
||||
# Mark attempt time BEFORE trying (so failures also get cooldown)
|
||||
date +%s > "$COOLDOWN_FILE" 2>/dev/null || true
|
||||
|
||||
# Read the API token
|
||||
if [[ ! -f "$GITEA_API_TOKEN_FILE" ]]; then
|
||||
echo "ERROR: auto-recover token file not found at $GITEA_API_TOKEN_FILE. Manual re-registration required."
|
||||
exit 3
|
||||
fi
|
||||
API_TOKEN=$(cat "$GITEA_API_TOKEN_FILE" 2>/dev/null || true)
|
||||
if [[ -z "$API_TOKEN" ]]; then
|
||||
echo "ERROR: auto-recover token file is empty. Manual re-registration required."
|
||||
exit 3
|
||||
fi
|
||||
|
||||
echo "ATTEMPT: auto-recovering by fetching new registration token and re-registering..."
|
||||
|
||||
# Fetch a new registration token from the Gitea API
|
||||
# Try org-level first (for org-scoped runners), then instance-level
|
||||
REG_TOKEN=""
|
||||
for endpoint in \
|
||||
"api/v1/orgs/{{ gitea_runner_org | default('oblachno') }}/actions/runners/registration-token" \
|
||||
"api/v1/admin/actions/runners/registration-token"; do
|
||||
REG_TOKEN=$(curl -sf --connect-timeout 5 --max-time 10 -X POST \
|
||||
-H "Authorization: token $API_TOKEN" \
|
||||
"${GITEA_URL}/${endpoint}" 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin).get('token',''))" 2>/dev/null || true)
|
||||
if [[ -n "$REG_TOKEN" ]]; then
|
||||
echo "INFO: fetched registration token from ${endpoint}"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ -z "$REG_TOKEN" ]]; then
|
||||
echo "ERROR: failed to fetch registration token from Gitea API. Is Gitea reachable?"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
# Stop the runner service
|
||||
systemctl --user stop gitea-runner.service 2>/dev/null || true
|
||||
sleep 1
|
||||
|
||||
# Remove the stale .runner file
|
||||
rm -f "{{ gitea_runner_data_dir }}/.runner" 2>/dev/null || true
|
||||
|
||||
# Re-register
|
||||
cd "{{ gitea_runner_data_dir }}"
|
||||
if "$BINARY" register \
|
||||
--token "$REG_TOKEN" \
|
||||
--name "$RUNNER_NAME" \
|
||||
--instance "$GITEA_URL" \
|
||||
--labels "$RUNNER_LABELS" \
|
||||
--no-interactive 2>&1; then
|
||||
echo "RECOVERED: runner re-registered successfully"
|
||||
else
|
||||
echo "ERROR: re-registration failed. Manual intervention required."
|
||||
exit 3
|
||||
fi
|
||||
|
||||
# Start the runner service
|
||||
systemctl --user start gitea-runner.service
|
||||
sleep 3
|
||||
|
||||
# Verify recovery — query the Gitea API to confirm the new ID is registered
|
||||
NEW_ID=$(python3 -c "import json; print(json.load(open('$RUNNER_FILE')).get('id',''))" 2>/dev/null || true)
|
||||
if [[ -n "$NEW_ID" ]]; then
|
||||
new_found=$(curl -sf --connect-timeout 5 --max-time 10 \
|
||||
-H "Authorization: token $API_TOKEN" \
|
||||
"${GITEA_URL}/api/v1/admin/actions/runners" 2>/dev/null \
|
||||
| python3 -c "
|
||||
import sys, json
|
||||
try:
|
||||
data = json.load(sys.stdin)
|
||||
runners = data if isinstance(data, list) else data.get('runners', [])
|
||||
ids = [str(r.get('id', '')) for r in runners]
|
||||
print('1' if '$NEW_ID' in ids else '0')
|
||||
except Exception:
|
||||
print('0')
|
||||
" 2>/dev/null || echo "0")
|
||||
if [[ "$new_found" == "1" ]]; then
|
||||
echo "OK: runner recovered and registered with new ID $NEW_ID"
|
||||
# Clear cooldown on success
|
||||
rm -f "$COOLDOWN_FILE" 2>/dev/null || true
|
||||
else
|
||||
echo "WARN: runner re-registered but ID not found in Gitea API. Will retry after cooldown."
|
||||
exit 3
|
||||
fi
|
||||
else
|
||||
echo "WARN: could not read new .runner file after re-registration. Will retry after cooldown."
|
||||
exit 3
|
||||
fi
|
||||
{% else %}
|
||||
echo "Manual re-registration required: re-run gitea_runner role with gitea_runner_force_reregister=true."
|
||||
# Restart the service once in case it is a transient token refresh issue,
|
||||
# but this cannot recover an unregistered runner without re-registration.
|
||||
systemctl --user restart gitea-runner.service
|
||||
sleep 2
|
||||
exit 3
|
||||
{% endif %}
|
||||
fi
|
||||
|
||||
# 3. Check disk space — prune aggressively if below threshold
|
||||
disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
|
||||
if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_threshold }} ]]; then
|
||||
echo "WARN: Disk usage at ${disk_pct}%, pruning all runner resources"
|
||||
docker system prune -f --filter "label={{ gitea_runner_prune_label }}" --filter "until=1h" || true
|
||||
docker volume prune -f --filter "label={{ gitea_runner_prune_label }}" || true
|
||||
# Only prune dangling (untagged) images — keep tagged runner images (ci-full, ci-quality)
|
||||
docker image prune -f || true
|
||||
# Clean up stopped containers and dangling networks that accumulate from failed jobs
|
||||
docker container prune -f || true
|
||||
docker network prune -f || true
|
||||
# Force-remove stale containers (including running ones from failed molecule tests)
|
||||
# that are older than 1 hour. "docker container prune -f" only removes stopped
|
||||
# containers, so running containers from crashed CI jobs accumulate and consume
|
||||
# disk/memory. Exclude CI job containers (name starts with GITEA-ACTIONS-TASK).
|
||||
# Only remove containers older than 1 hour to avoid killing molecule test
|
||||
# containers that CI jobs are actively using.
|
||||
docker ps -a --format '{% raw %}{{.ID}} {{.Names}} {{.RunningFor}}{% endraw %}' 2>/dev/null \
|
||||
| grep -v 'GITEA-ACTIONS-TASK' \
|
||||
| grep -E '(hour|day|week|month|year)s? ago' \
|
||||
| awk '{print $1}' \
|
||||
| xargs -r docker rm -f 2>/dev/null || true
|
||||
docker system prune -af --filter "until=1h" --volumes || true
|
||||
# Prune networks older than 1 hour to avoid removing networks that
|
||||
# molecule tests are actively creating (e.g. 'traefik' network created
|
||||
# during molecule create phase before containers are attached).
|
||||
docker network prune -f --filter "until=1h" || true
|
||||
disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
|
||||
echo "INFO: Disk usage after prune: ${disk_pct}%"
|
||||
fi
|
||||
|
||||
@@ -33,5 +33,5 @@
|
||||
become_user: "{{ gitea_runner_service_user }}"
|
||||
environment:
|
||||
XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}"
|
||||
when: systemd_available.stat.exists
|
||||
when: gitea_runner_systemd_available.stat.exists
|
||||
changed_when: true
|
||||
|
||||
+6
-6
@@ -8,12 +8,12 @@ Each runner runs in an isolated **rootless Docker** environment under a dedicate
|
||||
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/src/branch/master/LICENSE)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/wiki)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/releases)
|
||||
[](https://www.python.org/downloads/)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/wiki)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/releases)
|
||||
[](https://www.python.org/downloads/)
|
||||
|
||||
## Overview
|
||||
|
||||
|
||||
+2
-2
@@ -36,7 +36,7 @@ ci = [
|
||||
"build==1.5.1",
|
||||
"twine==6.2.0",
|
||||
# Reusable CI/CD and dev tools (auto-merge, pr-review, pre-push checks, etc.)
|
||||
"devx @ git+https://git.oblachno.oblachno.fyi/oblachno-oss/devx.git@v0.47.8",
|
||||
"devx @ git+https://git.oblachno.oblachno.fyi/oblachno-oss/devx.git@v0.48.1",
|
||||
]
|
||||
# Lint and type-checking tools (validate job)
|
||||
lint = [
|
||||
@@ -56,7 +56,7 @@ molecule = [
|
||||
dev = [
|
||||
"grm[ci,lint,molecule]",
|
||||
# Reusable CI/CD and dev tools (pre-push hooks, create-task, create-pr)
|
||||
"devx @ git+https://git.oblachno.oblachno.fyi/oblachno-oss/devx.git@v0.47.8",
|
||||
"devx @ git+https://git.oblachno.oblachno.fyi/oblachno-oss/devx.git@v0.48.1",
|
||||
# Non-Python dev dependency: checkmake (Makefile linter)
|
||||
# Install via: go install github.com/checkmake/checkmake/cmd/checkmake@latest
|
||||
]
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Clean up stale runner registrations from Gitea.
|
||||
|
||||
A runner is considered stale if it hasn't been online for more than a
|
||||
configurable threshold (default: 1 hour). Stale runners accumulate when:
|
||||
- A runner host is rebuilt or re-provisioned (old registration remains)
|
||||
- A runner is re-registered (old entry remains alongside the new one)
|
||||
- A runner process dies and the healthcheck can't auto-recover
|
||||
|
||||
This script queries the Gitea API for all runners, identifies stale ones,
|
||||
and deletes them via ``DELETE /api/v1/admin/actions/runners/{id}``.
|
||||
|
||||
Usage::
|
||||
|
||||
python3 scripts/cleanup_stale_runners.py --gitea-url https://git.example.com --token <admin-token>
|
||||
python3 scripts/cleanup_stale_runners.py --gitea-url https://git.example.com --token <admin-token> --dry-run
|
||||
python3 scripts/cleanup_stale_runners.py --gitea-url https://git.example.com --token <admin-token> \\
|
||||
--stale-threshold 3600
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request # noqa: PTH123 # nosec B404
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _api_request(base_url: str, token: str, method: str, path: str) -> Any:
|
||||
url = f"{base_url.rstrip('/')}/api/v1{path}"
|
||||
req = urllib.request.Request(url, method=method) # nosec B310
|
||||
req.add_header("Authorization", f"token {token}")
|
||||
req.add_header("Accept", "application/json")
|
||||
try:
|
||||
with urllib.request.urlopen(req) as resp: # noqa: PTH123 # nosec B310
|
||||
if resp.status == 204:
|
||||
return None
|
||||
raw = resp.read()
|
||||
return json.loads(raw) if raw else None
|
||||
except urllib.error.HTTPError as e:
|
||||
detail = e.read().decode("utf-8", errors="replace")
|
||||
raise RuntimeError(f"Gitea API error {e.code}: {detail}") from e
|
||||
|
||||
|
||||
def list_runners(base_url: str, token: str) -> list[dict[str, Any]]:
|
||||
data = _api_request(base_url, token, "GET", "/admin/actions/runners")
|
||||
if data is None:
|
||||
return []
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
if isinstance(data, dict):
|
||||
return data.get("runners", [])
|
||||
return []
|
||||
|
||||
|
||||
def delete_runner(base_url: str, token: str, runner_id: int) -> bool:
|
||||
try:
|
||||
_api_request(base_url, token, "DELETE", f"/admin/actions/runners/{runner_id}")
|
||||
return True
|
||||
except RuntimeError as e:
|
||||
print(f" ERROR deleting runner {runner_id}: {e}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="Clean up stale Gitea runner registrations")
|
||||
parser.add_argument("--gitea-url", required=True, help="Gitea base URL")
|
||||
parser.add_argument("--token", required=True, help="Gitea admin API token")
|
||||
parser.add_argument(
|
||||
"--stale-threshold",
|
||||
type=int,
|
||||
default=3600,
|
||||
help="Seconds since last_online before a runner is considered stale (default: 3600 = 1h)",
|
||||
)
|
||||
parser.add_argument("--dry-run", action="store_true", help="List stale runners without deleting")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
runners = list_runners(args.gitea_url, args.token)
|
||||
if not runners:
|
||||
print("No runners found.")
|
||||
return 0
|
||||
|
||||
now = int(time.time())
|
||||
stale: list[dict[str, Any]] = []
|
||||
online: list[dict[str, Any]] = []
|
||||
|
||||
for runner in runners:
|
||||
last_online = runner.get("last_online", 0) or 0
|
||||
seconds_since = now - last_online
|
||||
runner["seconds_since_online"] = seconds_since
|
||||
if seconds_since > args.stale_threshold:
|
||||
stale.append(runner)
|
||||
else:
|
||||
online.append(runner)
|
||||
|
||||
print(f"Total runners: {len(runners)}")
|
||||
print(f"Online (within {args.stale_threshold}s): {len(online)}")
|
||||
print(f"Stale (>{args.stale_threshold}s): {len(stale)}")
|
||||
print()
|
||||
|
||||
if not stale:
|
||||
print("No stale runners to clean up.")
|
||||
return 0
|
||||
|
||||
print("Stale runners:")
|
||||
for r in stale:
|
||||
rid = r.get("id", "?")
|
||||
name = r.get("name", "?")
|
||||
uuid = r.get("uuid", "?")[:8]
|
||||
secs = r.get("seconds_since_online", 0)
|
||||
hours = secs / 3600
|
||||
print(f" id={rid} name={name} uuid={uuid}... offline={hours:.1f}h ago")
|
||||
|
||||
if args.dry_run:
|
||||
print("\n--dry-run: not deleting. Remove --dry-run to clean up.")
|
||||
return 0
|
||||
|
||||
print(f"\nDeleting {len(stale)} stale runners...")
|
||||
deleted = 0
|
||||
for r in stale:
|
||||
rid = r.get("id")
|
||||
if rid is None:
|
||||
continue
|
||||
if delete_runner(args.gitea_url, args.token, rid):
|
||||
deleted += 1
|
||||
print(f" Deleted runner id={rid} ({r.get('name', '?')})")
|
||||
|
||||
print(f"\nDone: {deleted}/{len(stale)} stale runners deleted.")
|
||||
return 0 if deleted == len(stale) else 1
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,217 @@
|
||||
"""Tests for cleanup_stale_runners.py."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from scripts.cleanup_stale_runners import (
|
||||
_api_request,
|
||||
delete_runner,
|
||||
list_runners,
|
||||
main,
|
||||
)
|
||||
|
||||
|
||||
class TestListRunners:
|
||||
"""Tests for list_runners()."""
|
||||
|
||||
@patch("scripts.cleanup_stale_runners._api_request")
|
||||
def test_returns_list_of_runners(self, mock_req: MagicMock) -> None:
|
||||
mock_req.return_value = [{"id": 1, "name": "runner-1"}, {"id": 2, "name": "runner-2"}]
|
||||
result = list_runners("https://git.example.com", "token")
|
||||
assert len(result) == 2
|
||||
assert result[0]["id"] == 1
|
||||
|
||||
@patch("scripts.cleanup_stale_runners._api_request")
|
||||
def test_returns_empty_on_none(self, mock_req: MagicMock) -> None:
|
||||
mock_req.return_value = None
|
||||
result = list_runners("https://git.example.com", "token")
|
||||
assert result == []
|
||||
|
||||
@patch("scripts.cleanup_stale_runners._api_request")
|
||||
def test_extracts_runners_from_dict(self, mock_req: MagicMock) -> None:
|
||||
mock_req.return_value = {"runners": [{"id": 1}]}
|
||||
result = list_runners("https://git.example.com", "token")
|
||||
assert len(result) == 1
|
||||
assert result[0]["id"] == 1
|
||||
|
||||
@patch("scripts.cleanup_stale_runners._api_request")
|
||||
def test_returns_empty_on_non_list_non_dict(self, mock_req: MagicMock) -> None:
|
||||
mock_req.return_value = "not a list"
|
||||
result = list_runners("https://git.example.com", "token")
|
||||
assert result == []
|
||||
|
||||
|
||||
class TestDeleteRunner:
|
||||
"""Tests for delete_runner()."""
|
||||
|
||||
@patch("scripts.cleanup_stale_runners._api_request")
|
||||
def test_returns_true_on_success(self, mock_req: MagicMock) -> None:
|
||||
mock_req.return_value = None
|
||||
assert delete_runner("https://git.example.com", "token", 42) is True
|
||||
|
||||
@patch("scripts.cleanup_stale_runners._api_request")
|
||||
def test_returns_false_on_error(self, mock_req: MagicMock) -> None:
|
||||
mock_req.side_effect = RuntimeError("API error 404: not found")
|
||||
assert delete_runner("https://git.example.com", "token", 42) is False
|
||||
|
||||
|
||||
class TestApiRequest:
|
||||
"""Tests for _api_request()."""
|
||||
|
||||
@patch("scripts.cleanup_stale_runners.urllib.request.urlopen")
|
||||
def test_returns_json_on_success(self, mock_urlopen: MagicMock) -> None:
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status = 200
|
||||
mock_resp.read.return_value = b'{"key": "value"}'
|
||||
mock_urlopen.return_value.__enter__.return_value = mock_resp
|
||||
result = _api_request("https://git.example.com", "token", "GET", "/test")
|
||||
assert result == {"key": "value"}
|
||||
|
||||
@patch("scripts.cleanup_stale_runners.urllib.request.urlopen")
|
||||
def test_returns_none_on_204(self, mock_urlopen: MagicMock) -> None:
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status = 204
|
||||
mock_urlopen.return_value.__enter__.return_value = mock_resp
|
||||
result = _api_request("https://git.example.com", "token", "DELETE", "/test/1")
|
||||
assert result is None
|
||||
|
||||
@patch("scripts.cleanup_stale_runners.urllib.request.urlopen")
|
||||
def test_returns_none_on_empty_body(self, mock_urlopen: MagicMock) -> None:
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status = 200
|
||||
mock_resp.read.return_value = b""
|
||||
mock_urlopen.return_value.__enter__.return_value = mock_resp
|
||||
result = _api_request("https://git.example.com", "token", "GET", "/test")
|
||||
assert result is None
|
||||
|
||||
@patch("scripts.cleanup_stale_runners.urllib.request.urlopen")
|
||||
def test_raises_on_http_error(self, mock_urlopen: MagicMock) -> None:
|
||||
import urllib.error
|
||||
|
||||
mock_error = urllib.error.HTTPError(
|
||||
"url",
|
||||
404,
|
||||
"Not Found",
|
||||
{},
|
||||
None,
|
||||
)
|
||||
mock_error.read = MagicMock(return_value=b'{"message": "not found"}')
|
||||
mock_urlopen.side_effect = mock_error
|
||||
import pytest
|
||||
|
||||
with pytest.raises(RuntimeError, match="404"):
|
||||
_api_request("https://git.example.com", "token", "GET", "/test")
|
||||
|
||||
|
||||
class TestMain:
|
||||
"""Tests for main()."""
|
||||
|
||||
@patch("scripts.cleanup_stale_runners.list_runners")
|
||||
def test_no_runners(self, mock_list: MagicMock) -> None:
|
||||
mock_list.return_value = []
|
||||
rc = main(["--gitea-url", "https://git.example.com", "--token", "t"])
|
||||
assert rc == 0
|
||||
|
||||
@patch("scripts.cleanup_stale_runners.list_runners")
|
||||
def test_no_stale_runners(self, mock_list: MagicMock) -> None:
|
||||
now = int(time.time())
|
||||
mock_list.return_value = [
|
||||
{"id": 1, "name": "runner-1", "last_online": now - 60},
|
||||
]
|
||||
rc = main(["--gitea-url", "https://git.example.com", "--token", "t"])
|
||||
assert rc == 0
|
||||
|
||||
@patch("scripts.cleanup_stale_runners.list_runners")
|
||||
def test_dry_run_does_not_delete(self, mock_list: MagicMock) -> None:
|
||||
now = int(time.time())
|
||||
mock_list.return_value = [
|
||||
{"id": 1, "name": "runner-1", "last_online": now - 7200},
|
||||
]
|
||||
with patch("scripts.cleanup_stale_runners.delete_runner") as mock_del:
|
||||
rc = main(
|
||||
[
|
||||
"--gitea-url",
|
||||
"https://git.example.com",
|
||||
"--token",
|
||||
"t",
|
||||
"--dry-run",
|
||||
]
|
||||
)
|
||||
assert rc == 0
|
||||
mock_del.assert_not_called()
|
||||
|
||||
@patch("scripts.cleanup_stale_runners.list_runners")
|
||||
@patch("scripts.cleanup_stale_runners.delete_runner")
|
||||
def test_deletes_stale_runners(self, mock_del: MagicMock, mock_list: MagicMock) -> None:
|
||||
now = int(time.time())
|
||||
mock_list.return_value = [
|
||||
{"id": 1, "name": "runner-1", "last_online": now - 60},
|
||||
{"id": 2, "name": "runner-2", "last_online": now - 7200},
|
||||
{"id": 3, "name": "runner-3", "last_online": now - 9999},
|
||||
]
|
||||
mock_del.return_value = True
|
||||
rc = main(
|
||||
[
|
||||
"--gitea-url",
|
||||
"https://git.example.com",
|
||||
"--token",
|
||||
"t",
|
||||
"--stale-threshold",
|
||||
"3600",
|
||||
]
|
||||
)
|
||||
assert rc == 0
|
||||
assert mock_del.call_count == 2
|
||||
|
||||
@patch("scripts.cleanup_stale_runners.list_runners")
|
||||
@patch("scripts.cleanup_stale_runners.delete_runner")
|
||||
def test_returns_1_on_partial_failure(self, mock_del: MagicMock, mock_list: MagicMock) -> None:
|
||||
now = int(time.time())
|
||||
mock_list.return_value = [
|
||||
{"id": 1, "name": "runner-1", "last_online": now - 7200},
|
||||
{"id": 2, "name": "runner-2", "last_online": now - 7200},
|
||||
]
|
||||
mock_del.side_effect = [True, False]
|
||||
rc = main(["--gitea-url", "https://git.example.com", "--token", "t"])
|
||||
assert rc == 1
|
||||
|
||||
@patch("scripts.cleanup_stale_runners.list_runners")
|
||||
def test_runner_with_zero_last_online(self, mock_list: MagicMock) -> None:
|
||||
"""Runners with last_online=0 should be considered stale."""
|
||||
mock_list.return_value = [
|
||||
{"id": 1, "name": "runner-1", "last_online": 0},
|
||||
]
|
||||
with patch("scripts.cleanup_stale_runners.delete_runner") as mock_del:
|
||||
mock_del.return_value = True
|
||||
rc = main(["--gitea-url", "https://git.example.com", "--token", "t"])
|
||||
assert rc == 0
|
||||
mock_del.assert_called_once()
|
||||
|
||||
@patch("scripts.cleanup_stale_runners.list_runners")
|
||||
def test_runner_with_missing_last_online(self, mock_list: MagicMock) -> None:
|
||||
"""Runners with missing last_online should be considered stale."""
|
||||
mock_list.return_value = [
|
||||
{"id": 1, "name": "runner-1"},
|
||||
]
|
||||
with patch("scripts.cleanup_stale_runners.delete_runner") as mock_del:
|
||||
mock_del.return_value = True
|
||||
rc = main(["--gitea-url", "https://git.example.com", "--token", "t"])
|
||||
assert rc == 0
|
||||
mock_del.assert_called_once()
|
||||
|
||||
@patch("scripts.cleanup_stale_runners.list_runners")
|
||||
@patch("scripts.cleanup_stale_runners.delete_runner")
|
||||
def test_skips_runner_with_none_id(self, mock_del: MagicMock, mock_list: MagicMock) -> None:
|
||||
"""Runners with id=None should be skipped during deletion."""
|
||||
now = int(time.time())
|
||||
mock_list.return_value = [
|
||||
{"id": None, "name": "bad-runner", "last_online": now - 7200},
|
||||
{"id": 2, "name": "runner-2", "last_online": now - 7200},
|
||||
]
|
||||
mock_del.return_value = True
|
||||
rc = main(["--gitea-url", "https://git.example.com", "--token", "t"])
|
||||
# 1/2 deleted (None id skipped), so rc=1 (partial)
|
||||
assert rc == 1
|
||||
mock_del.assert_called_once_with("https://git.example.com", "t", 2)
|
||||
+21
-1
@@ -145,11 +145,25 @@ def cli(ctx: click.Context, become_password_file: str | None, verbose: bool) ->
|
||||
"Example: docker:docker://alpine:latest"
|
||||
),
|
||||
)
|
||||
@click.option(
|
||||
"--force-reregister/--no-force-reregister",
|
||||
default=False,
|
||||
help=_("Force re-registration even if .runner file exists (env: GITEA_FORCE_REREGISTER)"),
|
||||
)
|
||||
@click.option(
|
||||
"--ask-become-pass/--no-ask-become-pass",
|
||||
default=True,
|
||||
help=_("Prompt for sudo password (default)"),
|
||||
)
|
||||
@click.option(
|
||||
"--auto-recover-token",
|
||||
default=None,
|
||||
help=_(
|
||||
"Gitea API token for healthcheck auto-recovery (env: GITEA_AUTO_RECOVER_TOKEN). "
|
||||
"When set, the healthcheck can automatically re-register the runner "
|
||||
"if it becomes unregistered. Requires admin or org-level access."
|
||||
),
|
||||
)
|
||||
@_handle_errors("Installation failed: {error}")
|
||||
def install(
|
||||
host: str,
|
||||
@@ -161,10 +175,14 @@ def install(
|
||||
admin_token: str | None,
|
||||
integration_retries: int,
|
||||
labels: str | None,
|
||||
force_reregister: bool,
|
||||
ask_become_pass: bool,
|
||||
auto_recover_token: str | None,
|
||||
) -> None:
|
||||
if labels is None:
|
||||
labels = os.getenv("GITEA_RUNNER_LABELS")
|
||||
if auto_recover_token is None:
|
||||
auto_recover_token = os.getenv("GITEA_AUTO_RECOVER_TOKEN")
|
||||
manager = RunnerManager()
|
||||
manager.install(
|
||||
host=host,
|
||||
@@ -176,9 +194,11 @@ def install(
|
||||
admin_token=admin_token,
|
||||
integration_retries=integration_retries,
|
||||
labels=labels,
|
||||
force_reregister=force_reregister,
|
||||
ask_become_pass=ask_become_pass,
|
||||
become_password_file=_get_become_password_file(),
|
||||
verbose=_get_verbose(),
|
||||
auto_recover_token=auto_recover_token,
|
||||
)
|
||||
|
||||
|
||||
@@ -506,7 +526,7 @@ def list_runners(ask_become_pass: bool, no_status: bool) -> None:
|
||||
click.echo(f"{_('NAME'):<18} {_('HOST'):<16} {_('USER'):<10} {_('LABELS'):<30} {_('STATUS')}")
|
||||
click.echo("-" * 90)
|
||||
for r in runners:
|
||||
click.echo(f"{r['name']:<18} {r['host']:<16} {r['user']:<10} {r['labels']:<30} {r['status']}")
|
||||
click.echo(f"{r['name']:<18} {r['host']:<16} {r['user']:<10} {(r['labels'] or ''):<30} {r['status']}")
|
||||
|
||||
|
||||
@cli.command(name="trigger-workflow", help=_("Trigger a Gitea Actions workflow via the API."))
|
||||
|
||||
@@ -87,8 +87,10 @@ class RunnerManager:
|
||||
integration_retries: int = 3,
|
||||
ask_become_pass: bool = False,
|
||||
labels: str | None = None,
|
||||
force_reregister: bool = False,
|
||||
become_password_file: str | None = None,
|
||||
verbose: bool = False,
|
||||
auto_recover_token: str | None = None,
|
||||
) -> None:
|
||||
"""Install a runner on a remote host using Ansible."""
|
||||
if not name:
|
||||
@@ -98,16 +100,19 @@ class RunnerManager:
|
||||
if not token:
|
||||
raise AnsibleError(_("GITEA_REGISTRATION_TOKEN must be set (or pass --token)"))
|
||||
|
||||
extra_vars: dict[str, str | int] = {
|
||||
extra_vars: dict[str, str | int | bool] = {
|
||||
"registration_token": token,
|
||||
"gitea_runner_name": name,
|
||||
"gitea_url": gitea_url,
|
||||
"gitea_runner_integration_retries": integration_retries,
|
||||
"gitea_runner_force_reregister": force_reregister,
|
||||
}
|
||||
if admin_token:
|
||||
extra_vars["gitea_admin_token"] = admin_token
|
||||
if labels is not None:
|
||||
extra_vars["gitea_runner_labels"] = labels
|
||||
if auto_recover_token:
|
||||
extra_vars["gitea_runner_auto_recover_api_token"] = auto_recover_token
|
||||
|
||||
with track_steps() as tracker:
|
||||
tracker.begin(_("Installing Gitea Runner on {host}", host=host))
|
||||
|
||||
@@ -367,6 +367,14 @@
|
||||
"ru": "Токен регистрации (env: GITEA_REGISTRATION_TOKEN)",
|
||||
"zh": "注册令牌（环境变量: GITEA_REGISTRATION_TOKEN）"
|
||||
},
|
||||
"Force re-registration even if .runner file exists (env: GITEA_FORCE_REREGISTER)": {
|
||||
"bg": "Принудителна повторна регистрация, дори ако .runner файлът съществува (env: GITEA_FORCE_REREGISTER)",
|
||||
"de": "Erneute Registrierung erzwingen, auch wenn .runner-Datei existiert (env: GITEA_FORCE_REREGISTER)",
|
||||
"en": "Force re-registration even if .runner file exists (env: GITEA_FORCE_REREGISTER)",
|
||||
"pl": "Wymuś ponowną rejestrację, nawet jeśli plik .runner istnieje (env: GITEA_FORCE_REREGISTER)",
|
||||
"ru": "Принудительно повторно зарегистрировать, даже если файл .runner существует (env: GITEA_FORCE_REREGISTER)",
|
||||
"zh": "即使存在 .runner 文件也强制重新注册（环境变量: GITEA_FORCE_REREGISTER）"
|
||||
},
|
||||
"Remove a registered Gitea Runner completely.": {
|
||||
"bg": "Пълно премахване на регистриран Gitea Runner.",
|
||||
"de": "Einen registrierten Gitea Runner vollständig entfernen.",
|
||||
@@ -774,5 +782,13 @@
|
||||
"pl": "Workflow uruchomiony pomyślnie. ID uruchomienia: {run_id}",
|
||||
"ru": "Workflow успешно запущен. ID запуска: {run_id}",
|
||||
"zh": "工作流触发成功。运行 ID：{run_id}"
|
||||
},
|
||||
"Gitea API token for healthcheck auto-recovery (env: GITEA_AUTO_RECOVER_TOKEN). When set, the healthcheck can automatically re-register the runner if it becomes unregistered. Requires admin or org-level access.": {
|
||||
"bg": "Gitea API token for healthcheck auto-recovery (env: GITEA_AUTO_RECOVER_TOKEN). When set, the healthcheck can automatically re-register the runner if it becomes unregistered. Requires admin or org-level access.",
|
||||
"de": "Gitea API token for healthcheck auto-recovery (env: GITEA_AUTO_RECOVER_TOKEN). When set, the healthcheck can automatically re-register the runner if it becomes unregistered. Requires admin or org-level access.",
|
||||
"en": "Gitea API token for healthcheck auto-recovery (env: GITEA_AUTO_RECOVER_TOKEN). When set, the healthcheck can automatically re-register the runner if it becomes unregistered. Requires admin or org-level access.",
|
||||
"pl": "Gitea API token for healthcheck auto-recovery (env: GITEA_AUTO_RECOVER_TOKEN). When set, the healthcheck can automatically re-register the runner if it becomes unregistered. Requires admin or org-level access.",
|
||||
"ru": "Gitea API token for healthcheck auto-recovery (env: GITEA_AUTO_RECOVER_TOKEN). When set, the healthcheck can automatically re-register the runner if it becomes unregistered. Requires admin or org-level access.",
|
||||
"zh": "Gitea API token for healthcheck auto-recovery (env: GITEA_AUTO_RECOVER_TOKEN). When set, the healthcheck can automatically re-register the runner if it becomes unregistered. Requires admin or org-level access."
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,9 +50,11 @@ class TestCLI:
|
||||
admin_token="",
|
||||
integration_retries=3,
|
||||
labels=None,
|
||||
force_reregister=False,
|
||||
ask_become_pass=True,
|
||||
become_password_file=None,
|
||||
verbose=False,
|
||||
auto_recover_token=None,
|
||||
)
|
||||
|
||||
@patch("grm.cli.RunnerManager")
|
||||
@@ -73,9 +75,11 @@ class TestCLI:
|
||||
admin_token="",
|
||||
integration_retries=3,
|
||||
labels=None,
|
||||
force_reregister=False,
|
||||
ask_become_pass=False,
|
||||
become_password_file=None,
|
||||
verbose=False,
|
||||
auto_recover_token=None,
|
||||
)
|
||||
|
||||
@patch("grm.cli.RunnerManager")
|
||||
@@ -114,9 +118,11 @@ class TestCLI:
|
||||
admin_token=None,
|
||||
integration_retries=3,
|
||||
labels=None,
|
||||
force_reregister=False,
|
||||
ask_become_pass=True,
|
||||
become_password_file=None,
|
||||
verbose=False,
|
||||
auto_recover_token=None,
|
||||
)
|
||||
|
||||
@patch("grm.cli.RunnerManager")
|
||||
@@ -165,9 +171,11 @@ class TestCLI:
|
||||
admin_token="",
|
||||
integration_retries=3,
|
||||
labels=None,
|
||||
force_reregister=False,
|
||||
ask_become_pass=True,
|
||||
become_password_file=None,
|
||||
verbose=False,
|
||||
auto_recover_token=None,
|
||||
)
|
||||
|
||||
@patch("grm.cli.RunnerManager")
|
||||
@@ -188,9 +196,11 @@ class TestCLI:
|
||||
admin_token="",
|
||||
integration_retries=3,
|
||||
labels=None,
|
||||
force_reregister=False,
|
||||
ask_become_pass=True,
|
||||
become_password_file=None,
|
||||
verbose=False,
|
||||
auto_recover_token=None,
|
||||
)
|
||||
|
||||
@patch("grm.cli.RunnerManager")
|
||||
@@ -226,9 +236,92 @@ class TestCLI:
|
||||
admin_token="",
|
||||
integration_retries=3,
|
||||
labels="docker:docker://alpine:latest",
|
||||
force_reregister=False,
|
||||
ask_become_pass=True,
|
||||
become_password_file=None,
|
||||
verbose=False,
|
||||
auto_recover_token=None,
|
||||
)
|
||||
|
||||
@patch("grm.cli.RunnerManager")
|
||||
def test_install_force_reregister(self, mock_manager_class: MagicMock) -> None:
|
||||
mock_manager = MagicMock()
|
||||
mock_manager_class.return_value = mock_manager
|
||||
|
||||
runner = CliRunner(env=_TEST_ENV)
|
||||
result = runner.invoke(cli, ["install", "host1", "--user", "ubuntu", "--token", "tok", "--force-reregister"])
|
||||
assert result.exit_code == 0
|
||||
mock_manager.install.assert_called_once_with(
|
||||
host="host1",
|
||||
user="ubuntu",
|
||||
key=None,
|
||||
name=None,
|
||||
token="tok",
|
||||
gitea_url="https://git.example.com",
|
||||
admin_token="",
|
||||
integration_retries=3,
|
||||
labels=None,
|
||||
force_reregister=True,
|
||||
ask_become_pass=True,
|
||||
become_password_file=None,
|
||||
verbose=False,
|
||||
auto_recover_token=None,
|
||||
)
|
||||
|
||||
@patch("grm.cli.RunnerManager")
|
||||
def test_install_with_auto_recover_token(self, mock_manager_class: MagicMock) -> None:
|
||||
"""--auto-recover-token passes the token to the manager for healthcheck auto-recovery."""
|
||||
mock_manager = MagicMock()
|
||||
mock_manager_class.return_value = mock_manager
|
||||
|
||||
runner = CliRunner(env=_TEST_ENV)
|
||||
result = runner.invoke(
|
||||
cli,
|
||||
["install", "host1", "--user", "ubuntu", "--token", "tok", "--auto-recover-token", "api-tok"],
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
mock_manager.install.assert_called_once_with(
|
||||
host="host1",
|
||||
user="ubuntu",
|
||||
key=None,
|
||||
name=None,
|
||||
token="tok",
|
||||
gitea_url="https://git.example.com",
|
||||
admin_token="",
|
||||
integration_retries=3,
|
||||
labels=None,
|
||||
force_reregister=False,
|
||||
ask_become_pass=True,
|
||||
become_password_file=None,
|
||||
verbose=False,
|
||||
auto_recover_token="api-tok",
|
||||
)
|
||||
|
||||
@patch("grm.cli.RunnerManager")
|
||||
def test_install_auto_recover_token_from_env(self, mock_manager_class: MagicMock) -> None:
|
||||
"""GITEA_AUTO_RECOVER_TOKEN env var is used when --auto-recover-token is not passed."""
|
||||
mock_manager = MagicMock()
|
||||
mock_manager_class.return_value = mock_manager
|
||||
|
||||
env = {**_TEST_ENV, "GITEA_AUTO_RECOVER_TOKEN": "env-tok"}
|
||||
runner = CliRunner(env=env)
|
||||
result = runner.invoke(cli, ["install", "host1", "--user", "ubuntu", "--token", "tok"])
|
||||
assert result.exit_code == 0
|
||||
mock_manager.install.assert_called_once_with(
|
||||
host="host1",
|
||||
user="ubuntu",
|
||||
key=None,
|
||||
name=None,
|
||||
token="tok",
|
||||
gitea_url="https://git.example.com",
|
||||
admin_token="",
|
||||
integration_retries=3,
|
||||
labels=None,
|
||||
force_reregister=False,
|
||||
ask_become_pass=True,
|
||||
become_password_file=None,
|
||||
verbose=False,
|
||||
auto_recover_token="env-tok",
|
||||
)
|
||||
|
||||
@patch("grm.cli.RunnerManager")
|
||||
@@ -250,9 +343,11 @@ class TestCLI:
|
||||
admin_token="",
|
||||
integration_retries=3,
|
||||
labels="",
|
||||
force_reregister=False,
|
||||
ask_become_pass=True,
|
||||
become_password_file=None,
|
||||
verbose=False,
|
||||
auto_recover_token=None,
|
||||
)
|
||||
|
||||
@patch("grm.cli.RunnerManager")
|
||||
@@ -274,9 +369,11 @@ class TestCLI:
|
||||
admin_token="",
|
||||
integration_retries=3,
|
||||
labels="docker:docker://alpine:latest",
|
||||
force_reregister=False,
|
||||
ask_become_pass=True,
|
||||
become_password_file=None,
|
||||
verbose=False,
|
||||
auto_recover_token=None,
|
||||
)
|
||||
|
||||
@patch("grm.cli.RunnerManager")
|
||||
@@ -307,9 +404,11 @@ class TestCLI:
|
||||
admin_token="",
|
||||
integration_retries=3,
|
||||
labels=None,
|
||||
force_reregister=False,
|
||||
ask_become_pass=True,
|
||||
become_password_file=pw_file,
|
||||
verbose=False,
|
||||
auto_recover_token=None,
|
||||
)
|
||||
finally:
|
||||
import os
|
||||
@@ -334,9 +433,11 @@ class TestCLI:
|
||||
admin_token="",
|
||||
integration_retries=3,
|
||||
labels=None,
|
||||
force_reregister=False,
|
||||
ask_become_pass=True,
|
||||
become_password_file=None,
|
||||
verbose=True,
|
||||
auto_recover_token=None,
|
||||
)
|
||||
|
||||
@patch("grm.cli.RunnerManager")
|
||||
@@ -741,6 +842,27 @@ class TestCLI:
|
||||
assert "active" in result.output
|
||||
mock_manager.list_runners.assert_called_once_with(become_pass=None, no_status=False)
|
||||
|
||||
@patch("grm.cli.RunnerManager")
|
||||
def test_list_with_none_labels(self, mock_manager_class: MagicMock) -> None:
|
||||
"""Runners with labels=None should not crash the list command."""
|
||||
mock_manager = MagicMock()
|
||||
mock_manager.list_runners.return_value = [
|
||||
{
|
||||
"name": "r1",
|
||||
"host": "10.0.0.1",
|
||||
"user": "ubuntu",
|
||||
"labels": None,
|
||||
"status": "active",
|
||||
},
|
||||
]
|
||||
mock_manager_class.return_value = mock_manager
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["list"])
|
||||
assert result.exit_code == 0
|
||||
assert "r1" in result.output
|
||||
assert "active" in result.output
|
||||
|
||||
@patch("grm.cli.RunnerManager")
|
||||
def test_list_no_status(self, mock_manager_class: MagicMock) -> None:
|
||||
"""--no-status skips SSH checks and shows registry only."""
|
||||
|
||||
@@ -51,6 +51,7 @@ class TestRunnerManager:
|
||||
assert manager._captured_extra_vars["registration_token"] == "tok"
|
||||
assert manager._captured_extra_vars["gitea_runner_name"] == "192.168.1.10"
|
||||
assert manager._captured_extra_vars["gitea_url"] == "https://git.example.com"
|
||||
assert manager._captured_extra_vars["gitea_runner_force_reregister"] is False
|
||||
assert "Installing Gitea Runner on 192.168.1.10" in mock_executor.run.call_args.kwargs["description"]
|
||||
mock_registry.add.assert_called_once_with(
|
||||
name="192.168.1.10",
|
||||
@@ -76,6 +77,7 @@ class TestRunnerManager:
|
||||
assert "/key" in cmd_str
|
||||
assert manager._captured_extra_vars["registration_token"] == "preset"
|
||||
assert manager._captured_extra_vars["gitea_runner_name"] == "my-runner"
|
||||
assert manager._captured_extra_vars["gitea_runner_force_reregister"] is False
|
||||
assert "--ask-become-pass" not in cmd_str
|
||||
mock_registry.add.assert_called_once_with(
|
||||
name="my-runner",
|
||||
@@ -97,6 +99,52 @@ class TestRunnerManager:
|
||||
cmd_str = " ".join(cmd)
|
||||
assert "--ask-become-pass" in cmd_str
|
||||
|
||||
def test_install_force_reregister(self) -> None:
|
||||
mock_registry = MagicMock()
|
||||
manager = RunnerManager(registry=mock_registry)
|
||||
mock_executor = MagicMock()
|
||||
manager._executor = mock_executor
|
||||
|
||||
manager.install(
|
||||
"host1",
|
||||
"root",
|
||||
token="tok",
|
||||
gitea_url="https://git.example.com",
|
||||
force_reregister=True,
|
||||
)
|
||||
assert manager._captured_extra_vars["gitea_runner_force_reregister"] is True
|
||||
|
||||
def test_install_with_auto_recover_token(self) -> None:
|
||||
"""auto_recover_token is passed as extra_var to Ansible."""
|
||||
mock_registry = MagicMock()
|
||||
manager = RunnerManager(registry=mock_registry)
|
||||
mock_executor = MagicMock()
|
||||
manager._executor = mock_executor
|
||||
|
||||
manager.install(
|
||||
"host1",
|
||||
"root",
|
||||
token="tok",
|
||||
gitea_url="https://git.example.com",
|
||||
auto_recover_token="api-tok",
|
||||
)
|
||||
assert manager._captured_extra_vars["gitea_runner_auto_recover_api_token"] == "api-tok"
|
||||
|
||||
def test_install_without_auto_recover_token(self) -> None:
|
||||
"""When auto_recover_token is None, the extra_var is not set."""
|
||||
mock_registry = MagicMock()
|
||||
manager = RunnerManager(registry=mock_registry)
|
||||
mock_executor = MagicMock()
|
||||
manager._executor = mock_executor
|
||||
|
||||
manager.install(
|
||||
"host1",
|
||||
"root",
|
||||
token="tok",
|
||||
gitea_url="https://git.example.com",
|
||||
)
|
||||
assert "gitea_runner_auto_recover_api_token" not in manager._captured_extra_vars
|
||||
|
||||
def test_install_missing_gitea_url(self) -> None:
|
||||
manager = RunnerManager()
|
||||
with pytest.raises(AnsibleError, match="GITEA_URL must be set"):
|
||||
|
||||
Reference in New Issue
Block a user