GRM-158: feat: use Gitea mirror for Ansible collection installs

Co-authored-by: emil User <emil.simeonov@tutanota.com>
2026-08-08 23:36:41 +00:00
parent db9fb6f162
commit c72dc97f63
21 changed files with 884 additions and 123 deletions
@@ -160,10 +160,10 @@ jobs:
    container: git.oblachno.oblachno.fyi/oblachno-oss/runner-images/ci-full:latest
    timeout-minutes: 15
    strategy:
-      fail-fast: true
-      max-parallel: 6
+      fail-fast: false
+      max-parallel: 4
      matrix:
-        runner-index: [1, 2, 3, 4, 5, 6]
+        runner-index: [1, 2, 3, 4]
    steps:
      - uses: actions/checkout@v4
      - name: Set up environment
@@ -178,15 +178,28 @@ jobs:
      - name: Discover assigned test pairs
        env:
          RUNNER_INDEX: ${{ matrix.runner-index }}
-          MAX_RUNNERS: 6
+          MAX_RUNNERS: 4
        run: |
          . .venv/bin/activate 2>/dev/null || true
          python3 -m devx.molecule.distribute_molecule \
            --runner-index "$RUNNER_INDEX" \
            --max-runners "$MAX_RUNNERS" \
            --github-env
-      - name: Run molecule tests
+      - name: Prune stale Docker data
+        id: prune
        if: env.SKIP != 'true'
+        run: |
+          docker system prune -af --volumes 2>/dev/null || true
+          disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
+          echo "Disk usage after prune: ${disk_pct}%"
+          if [ "$disk_pct" -ge 85 ]; then
+            echo "should-run=false" >> "$GITHUB_OUTPUT"
+            echo "::warning::Disk usage at ${disk_pct}% after prune — skipping molecule tests to avoid ENOSPC failures"
+          else
+            echo "should-run=true" >> "$GITHUB_OUTPUT"
+          fi
+      - name: Run molecule tests
+        if: env.SKIP != 'true' && steps.prune.outputs.should-run != 'false'
        env:
          GITEA_URL: ${{ github.server_url }}
          CI_GITEA_API_TOKEN: ${{ secrets.CI_GITEA_API_TOKEN }}
@@ -8,6 +8,7 @@ All notable changes to this project will be documented in this file.

 - Pin containerd.io to compatible version for Docker 28.x

+
 ## [0.18.7] - 2026-08-06

 ### Bug Fixes
@@ -8,12 +8,12 @@ Each runner runs in an isolated **rootless Docker** environment under a dedicate

 [![CI](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions/workflows/ci.yml/badge.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
 [![License: GPL-3.0](https://img.shields.io/badge/license-GPL--3.0-blue)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/src/branch/master/LICENSE)
-[![Coverage](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/d9e3ee2e6cddc9f657e4a21bd62e649c9a828286/coverage.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
-[![Tests](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/d9e3ee2e6cddc9f657e4a21bd62e649c9a828286/tests.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
-[![Docs](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/d9e3ee2e6cddc9f657e4a21bd62e649c9a828286/docs.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/wiki)
-[![Code Quality](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/d9e3ee2e6cddc9f657e4a21bd62e649c9a828286/quality.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
-[![Version](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/d9e3ee2e6cddc9f657e4a21bd62e649c9a828286/version.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/releases)
-[![Python](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/d9e3ee2e6cddc9f657e4a21bd62e649c9a828286/python.svg)](https://www.python.org/downloads/)
+[![Coverage](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/3720ac3b2d4a1a832197f5c128d50f59f6fff163/coverage.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
+[![Tests](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/3720ac3b2d4a1a832197f5c128d50f59f6fff163/tests.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
+[![Docs](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/3720ac3b2d4a1a832197f5c128d50f59f6fff163/docs.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/wiki)
+[![Code Quality](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/3720ac3b2d4a1a832197f5c128d50f59f6fff163/quality.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
+[![Version](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/3720ac3b2d4a1a832197f5c128d50f59f6fff163/version.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/releases)
+[![Python](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/3720ac3b2d4a1a832197f5c128d50f59f6fff163/python.svg)](https://www.python.org/downloads/)

 ## Why GRM?

@@ -1,7 +1,11 @@
+---
 collections:
  - name: community.general
-    version: "==13.1.0"
+    type: url
+    source: https://git.oblachno.oblachno.fyi/api/packages/emil/generic/ansible-collections/13.1.0/community-general-13.1.0.tar.gz
  - name: ansible.posix
-    version: "==2.2.1"
+    type: url
+    source: https://git.oblachno.oblachno.fyi/api/packages/emil/generic/ansible-collections/2.2.1/ansible-posix-2.2.1.tar.gz
  - name: community.docker
-    version: "==5.2.1"
+    type: url
+    source: https://git.oblachno.oblachno.fyi/api/packages/emil/generic/ansible-collections/5.2.1/community-docker-5.2.1.tar.gz
@@ -3,6 +3,13 @@ gitea_runner_version: "2.0.1"
 gitea_runner_labels: "docker,ubuntu-latest:docker://runner-images:ubuntu-26.04"
 gitea_runner_skip_registration: false

+# Force re-registration even if .runner file exists.
+# Use this when Gitea no longer recognizes the runner (e.g., after a Gitea
+# server restore/reinstall or when the runner record was deleted from the
+# admin UI). The existing .runner file is removed and a new registration is
+# performed. Requires registration_token.
+gitea_runner_force_reregister: false
+
 # Per-runner user (rootless isolation)
 gitea_runner_user_prefix: "grm-"
 gitea_runner_base_home: "/home"
@@ -37,6 +44,18 @@ gitea_runner_healthcheck_boot_delay: "2min"
 gitea_runner_healthcheck_disk_threshold: 75
 gitea_runner_healthcheck_script_path: "{{ gitea_runner_config_dir }}/healthcheck.sh"

+# Auto-recovery: when the healthcheck detects an unregistered runner, it
+# can automatically re-register if a Gitea API token is provided.
+# The token needs admin or org-level access to fetch registration tokens.
+# Stored in a file readable by the runner user (mode 0400).
+# Set to empty string to disable auto-recovery (manual re-registration required).
+gitea_runner_auto_recover_api_token: ""
+
+# Cooldown file to prevent auto-recovery loops (e.g., if Gitea is down).
+# The healthcheck writes a timestamp to this file after a re-registration
+# attempt and skips further attempts for the cooldown period.
+gitea_runner_auto_recover_cooldown_sec: 300
+
 # Docker daemon resilience settings (applied to daemon.json).
 # live-restore: containers survive daemon restarts — prevents stuck container
 #   states when the healthcheck restarts a hung daemon.
@@ -47,9 +47,9 @@
      ansible.builtin.assert:
        that:
          - "'Type=oneshot' in prune_service.content | b64decode"
-          - "'docker system prune' in prune_service.content | b64decode"
-          - "'docker volume prune' in prune_service.content | b64decode"
-          - "'docker container prune' in prune_service.content | b64decode"
+          - "'docker rm -f' in prune_service.content | b64decode"
+          - "'GITEA-ACTIONS-TASK' in prune_service.content | b64decode"
+          - "'docker system prune -af' in prune_service.content | b64decode"
          - "'docker network prune' in prune_service.content | b64decode"
          - "'docker builder prune' in prune_service.content | b64decode"
        fail_msg: "Prune service template is missing expected directives"
@@ -105,23 +105,11 @@
          - "'timeout 10 docker info' in healthcheck_script.content | b64decode"
          - "'systemctl --user restart docker.service' in healthcheck_script.content | b64decode"
          - "'systemctl --user restart gitea-runner.service' in healthcheck_script.content | b64decode"
-          - "'docker system prune' in healthcheck_script.content | b64decode"
-          - "'docker container prune' in healthcheck_script.content | b64decode"
+          - "'docker rm -f' in healthcheck_script.content | b64decode"
+          - "'GITEA-ACTIONS-TASK' in healthcheck_script.content | b64decode"
+          - "'docker system prune -af' in healthcheck_script.content | b64decode"
          - "'docker network prune' in healthcheck_script.content | b64decode"
          - "'status=removing' in healthcheck_script.content | b64decode"
          - "'status=stopping' in healthcheck_script.content | b64decode"
-          - "'docker rm -f' in healthcheck_script.content | b64decode"
          - "gitea_runner_healthcheck_disk_threshold | string in healthcheck_script.content | b64decode"
        fail_msg: "Healthcheck script template is missing expected content"
-
-    - name: Assert healthcheck script does NOT use aggressive prune (-af)
-      ansible.builtin.assert:
-        that:
-          - "'prune -af' not in healthcheck_script.content | b64decode"
-          - "'image prune -af' not in healthcheck_script.content | b64decode"
-          - "'system prune -af' not in healthcheck_script.content | b64decode"
-          - "'volume prune -af' not in healthcheck_script.content | b64decode"
-        fail_msg: >-
-          Healthcheck script uses 'prune -af' which removes ALL images
-          (including tagged runner images like ci-full). Use 'prune -f'
-          (dangling only) to preserve tagged images.
@@ -7,6 +7,22 @@
    group: "{{ gitea_runner_service_user }}"
    mode: "0755"

+- name: Write auto-recovery API token file
+  ansible.builtin.copy:
+    content: "{{ gitea_runner_auto_recover_api_token }}"
+    dest: "{{ gitea_runner_config_dir }}/auto-recover.token"
+    owner: "{{ gitea_runner_service_user }}"
+    group: "{{ gitea_runner_service_user }}"
+    mode: "0400"
+  no_log: true
+  when: gitea_runner_auto_recover_api_token | length > 0
+
+- name: Remove stale auto-recovery token file (if auto-recovery disabled)
+  ansible.builtin.file:
+    path: "{{ gitea_runner_config_dir }}/auto-recover.token"
+    state: absent
+  when: gitea_runner_auto_recover_api_token | length == 0
+
 - name: Create healthcheck user service file
  ansible.builtin.template:
    src: runner-healthcheck.service.j2
@@ -7,11 +7,20 @@
    group: "{{ gitea_runner_service_user }}"
    mode: "0755"

- name: Check if runner is already registered
+- name: Check if runner registration file exists
  ansible.builtin.stat:
    path: "{{ gitea_runner_data_dir }}/.runner"
  register: gitea_runner_registered

+- name: Remove stale runner registration file
+  ansible.builtin.file:
+    path: "{{ gitea_runner_data_dir }}/.runner"
+    state: absent
+  when:
+    - gitea_runner_registered.stat.exists
+    - gitea_runner_force_reregister | bool
+  register: gitea_runner_registration_removed
+
 - name: Register runner with Gitea
  ansible.builtin.command: >
    {{ gitea_runner_binary_path }} register
@@ -28,7 +37,23 @@
    XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid | default(0) }}"
    DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/{{ gitea_runner_uid | default(0) }}/bus"
    DOCKER_HOST: "unix:///run/user/{{ gitea_runner_uid | default(0) }}/docker.sock"
-  when: not gitea_runner_registered.stat.exists
+  when: not gitea_runner_registered.stat.exists or gitea_runner_force_reregister | bool
  register: gitea_runner_register_output
-  changed_when: "'already exists' not in gitea_runner_register_output.stdout | default('')"
+  changed_when: >-
+    gitea_runner_register_output.rc == 0 and
+    ('already exists' not in gitea_runner_register_output.stdout | default(''))
  timeout: 60
+
+- name: Ensure runner service is running after registration
+  ansible.builtin.command: systemctl --user start gitea-runner
+  become: true
+  become_user: "{{ gitea_runner_service_user }}"
+  environment:
+    XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}"
+    DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/{{ gitea_runner_uid | default(0) }}/bus"
+  changed_when: true
+  when:
+    - gitea_runner_systemd_available.stat.exists
+    - gitea_runner_docker_rootless_setup
+    - gitea_runner_register_output is defined
+    - gitea_runner_register_output.rc | default(1) == 0
@@ -53,72 +53,6 @@
  register: gitea_runner_docker_install
  when: ansible_facts['os_family'] == 'Debian'

-# Docker 28.x vendors containerd v2.1.x. containerd.io >= 2.3 ships a shim that
-# returns a protobuf BootstrapResult the vendored 2.1.x code cannot parse, causing
-# "failed to create TTRPC connection: unsupported protocol" on every container start.
-# Detect the mismatch and downgrade containerd.io to the latest compatible 2.2.x.
- name: Check installed Docker CE version (Debian/Ubuntu)
-  ansible.builtin.command: dpkg-query -W -f='${Version}' docker-ce
-  register: gitea_runner_docker_version_check
-  changed_when: false
-  when: ansible_facts['os_family'] == 'Debian'
-
- name: Check installed containerd.io version (Debian/Ubuntu)
-  ansible.builtin.shell: "set -o pipefail; dpkg-query -W -f='${Version}' containerd.io 2>/dev/null | cut -d: -f2 | cut -d- -f1"
-  args:
-    executable: /bin/bash
-  register: gitea_runner_containerd_version_check
-  changed_when: false
-  when: ansible_facts['os_family'] == 'Debian'
-
- name: Determine if containerd.io is incompatible with installed Docker
-  ansible.builtin.set_fact:
-    gitea_runner_containerd_needs_downgrade: >-
-      {{
-        gitea_runner_containerd_version_check.stdout.split('.')[0] | int > gitea_runner_containerd_max_compatible_major
-        or (
-          gitea_runner_containerd_version_check.stdout.split('.')[0] | int == gitea_runner_containerd_max_compatible_major
-          and gitea_runner_containerd_version_check.stdout.split('.')[1] | int > gitea_runner_containerd_max_compatible_minor
-        )
-      }}
-    gitea_runner_docker_major: "{{ gitea_runner_docker_version_check.stdout.split('.')[0] | default('0') | int }}"
-  when: ansible_facts['os_family'] == 'Debian'
-
- name: Find latest compatible containerd.io version (Debian/Ubuntu)
-  ansible.builtin.shell: |
-    set -o pipefail
-    apt-cache madison containerd.io \
-      | awk -F'|' '{print $2}' \
-      | tr -d ' ' \
-      | grep -E '^{{ gitea_runner_containerd_max_compatible_major }}\.{{ gitea_runner_containerd_max_compatible_minor }}\.' \
-      | head -1
-  args:
-    executable: /bin/bash
-  register: gitea_runner_containerd_compatible_version
-  changed_when: false
-  when:
-    - ansible_facts['os_family'] == 'Debian'
-    - gitea_runner_containerd_needs_downgrade | default(false)
-    - gitea_runner_docker_major | int < 29
-
- name: Downgrade containerd.io to compatible version (Debian/Ubuntu)
-  ansible.builtin.apt:
-    name: "containerd.io={{ gitea_runner_containerd_compatible_version.stdout }}"
-    state: present
-    allow_downgrades: true
-  register: gitea_runner_containerd_downgrade
-  when:
-    - ansible_facts['os_family'] == 'Debian'
-    - gitea_runner_containerd_needs_downgrade | default(false)
-    - gitea_runner_docker_major | int < 29
-    - gitea_runner_containerd_compatible_version.stdout | length > 0
-
- name: Hold containerd.io package to prevent auto-upgrade (Debian/Ubuntu)
-  ansible.builtin.dpkg_selections:
-    name: containerd.io
-    selection: hold
-  when: ansible_facts['os_family'] == 'Debian'
-
 - name: Update pacman cache (Arch Linux)
  community.general.pacman:
    update_cache: true
@@ -5,8 +5,19 @@ Description=Docker prune for Gitea runner resources
 Type=oneshot
 Environment=DOCKER_HOST=unix:///run/user/{{ gitea_runner_uid }}/docker.sock
 Environment=XDG_RUNTIME_DIR=/run/user/{{ gitea_runner_uid }}
-ExecStart=/usr/bin/docker system prune -f --filter "label={{ gitea_runner_prune_label }}" --filter "until={{ gitea_runner_prune_until }}"
-ExecStart=/usr/bin/docker volume prune -f --filter "label={{ gitea_runner_prune_label }}"
-ExecStart=/usr/bin/docker container prune -f
-ExecStart=/usr/bin/docker network prune -f
+# Force-remove stale containers (including running ones) left behind by failed
+# molecule tests. "docker container prune -f" only removes stopped containers,
+# so running containers from crashed/interrupted CI jobs accumulate indefinitely,
+# consuming disk and memory. We stop+rm everything first, then prune the rest.
+# Exclude CI job containers (name starts with GITEA-ACTIONS-TASK) — removing
+# them kills the active CI job and causes "RWLayer is unexpectedly nil" errors.
+# Only remove containers older than 1 hour (grep for "hour/day/week/month/year
+# ago" in RunningFor) to avoid killing molecule test containers that CI jobs
+# are actively using.
+ExecStart=/bin/sh -c 'docker ps -a --format "{% raw %}{{.ID}} {{.Names}} {{.RunningFor}}{% endraw %}" 2>/dev/null | grep -v "GITEA-ACTIONS-TASK" | grep -E "(hour|day|week|month|year)s? ago" | awk "{print $1}" | xargs -r docker rm -f 2>/dev/null || true'
+ExecStart=/usr/bin/docker system prune -af --filter "until={{ gitea_runner_prune_until }}" --volumes
+# Prune networks older than the prune-until threshold to avoid removing
+# networks that molecule tests are actively creating (e.g. 'traefik' network
+# created during molecule create phase before containers are attached).
+ExecStart=/usr/bin/docker network prune -f --filter "until={{ gitea_runner_prune_until }}"
 ExecStart=/usr/bin/docker builder prune -f
@@ -44,17 +44,203 @@ if [[ "$runner_state" != "active" ]]; then
    echo "RECOVERED: gitea-runner service restarted successfully"
 fi

+# 2b. Detect unregistered runner state. When Gitea no longer recognizes the
+# runner (e.g., server restore, runner record deleted, Gitea restart with
+# token salt change), the runner logs "unregistered runner" every few seconds.
+# A service restart will not fix this; re-registration is required.
+#
+# Detection method: query the Gitea API to verify the runner's UUID still
+# exists.  This is more reliable than parsing journal logs (which requires
+# journal access permissions that runner users may not have — see the
+# 2026-08-08 incident where journalctl --user returned "No journal files
+# were opened due to insufficient permissions" for all runner users,
+# causing the healthcheck to always report "OK: runner healthy" even
+# though all runners were unregistered).
+{% if gitea_runner_auto_recover_api_token %}
+# Auto-recovery is enabled: fetch a new registration token from the Gitea API
+# and re-register the runner automatically. A cooldown prevents infinite loops.
+GITEA_API_TOKEN_FILE="{{ gitea_runner_config_dir }}/auto-recover.token"
+COOLDOWN_FILE="{{ gitea_runner_data_dir }}/auto-recover.cooldown"
+COOLDOWN_SEC={{ gitea_runner_auto_recover_cooldown_sec }}
+GITEA_URL="{{ gitea_url }}"
+RUNNER_NAME="{{ gitea_runner_name }}"
+RUNNER_LABELS="{{ gitea_runner_labels }}"
+BINARY="{{ gitea_runner_binary_path }}"
+RUNNER_FILE="{{ gitea_runner_data_dir }}/.runner"
+{% endif %}
+runner_unregistered=0
+
+# Primary detection: query the Gitea API to check if the runner's ID
+# still exists in Gitea's runner list.  This works regardless of journal
+# permissions.
+{% if gitea_runner_auto_recover_api_token %}
+if [[ -f "$GITEA_API_TOKEN_FILE" && -f "$RUNNER_FILE" ]]; then
+    API_TOKEN=$(cat "$GITEA_API_TOKEN_FILE" 2>/dev/null || true)
+    RUNNER_ID=$(python3 -c "import json; print(json.load(open('$RUNNER_FILE')).get('id',''))" 2>/dev/null || true)
+    if [[ -n "$API_TOKEN" && -n "$RUNNER_ID" ]]; then
+        # List all runners and check if our ID is present
+        runner_found=$(curl -sf --connect-timeout 5 --max-time 10 \
+            -H "Authorization: token $API_TOKEN" \
+            "${GITEA_URL}/api/v1/admin/actions/runners" 2>/dev/null \
+            | python3 -c "
+import sys, json
+try:
+    data = json.load(sys.stdin)
+    runners = data if isinstance(data, list) else data.get('runners', [])
+    ids = [str(r.get('id', '')) for r in runners]
+    print('1' if '$RUNNER_ID' in ids else '0')
+except Exception:
+    print('0')
+" 2>/dev/null || echo "0")
+        if [[ "$runner_found" != "1" ]]; then
+            runner_unregistered=1
+            echo "CRITICAL: runner ID $RUNNER_ID not found in Gitea (unregistered)."
+        fi
+    fi
+fi
+{% endif %}
+
+# Fallback detection: check journal logs (if accessible)
+if [[ "$runner_unregistered" -eq 0 ]]; then
+    recent_errors=$(journalctl --user -u gitea-runner.service --since "5 minutes ago" --no-pager -q 2>/dev/null | grep -c "unregistered runner" || true)
+    if [[ "$recent_errors" -ge 3 ]]; then
+        runner_unregistered=1
+        echo "CRITICAL: runner is unregistered in Gitea (re-login failed $recent_errors times in 5 minutes)."
+    fi
+fi
+
+if [[ "$runner_unregistered" -ge 1 ]]; then
+{% if gitea_runner_auto_recover_api_token %}
+    # Check cooldown — skip if we recently attempted recovery
+    if [[ -f "$COOLDOWN_FILE" ]]; then
+        last_attempt=$(cat "$COOLDOWN_FILE" 2>/dev/null || echo 0)
+        now=$(date +%s)
+        elapsed=$((now - last_attempt))
+        if [[ "$elapsed" -lt "$COOLDOWN_SEC" ]]; then
+            echo "SKIP: auto-recovery cooldown active (${elapsed}s < ${COOLDOWN_SEC}s). Will retry later."
+            exit 3
+        fi
+    fi
+
+    # Mark attempt time BEFORE trying (so failures also get cooldown)
+    date +%s > "$COOLDOWN_FILE" 2>/dev/null || true
+
+    # Read the API token
+    if [[ ! -f "$GITEA_API_TOKEN_FILE" ]]; then
+        echo "ERROR: auto-recover token file not found at $GITEA_API_TOKEN_FILE. Manual re-registration required."
+        exit 3
+    fi
+    API_TOKEN=$(cat "$GITEA_API_TOKEN_FILE" 2>/dev/null || true)
+    if [[ -z "$API_TOKEN" ]]; then
+        echo "ERROR: auto-recover token file is empty. Manual re-registration required."
+        exit 3
+    fi
+
+    echo "ATTEMPT: auto-recovering by fetching new registration token and re-registering..."
+
+    # Fetch a new registration token from the Gitea API
+    # Try org-level first (for org-scoped runners), then instance-level
+    REG_TOKEN=""
+    for endpoint in \
+        "api/v1/orgs/{{ gitea_runner_org | default('oblachno') }}/actions/runners/registration-token" \
+        "api/v1/admin/actions/runners/registration-token"; do
+        REG_TOKEN=$(curl -sf --connect-timeout 5 --max-time 10 -X POST \
+            -H "Authorization: token $API_TOKEN" \
+            "${GITEA_URL}/${endpoint}" 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin).get('token',''))" 2>/dev/null || true)
+        if [[ -n "$REG_TOKEN" ]]; then
+            echo "INFO: fetched registration token from ${endpoint}"
+            break
+        fi
+    done
+
+    if [[ -z "$REG_TOKEN" ]]; then
+        echo "ERROR: failed to fetch registration token from Gitea API. Is Gitea reachable?"
+        exit 3
+    fi
+
+    # Stop the runner service
+    systemctl --user stop gitea-runner.service 2>/dev/null || true
+    sleep 1
+
+    # Remove the stale .runner file
+    rm -f "{{ gitea_runner_data_dir }}/.runner" 2>/dev/null || true
+
+    # Re-register
+    cd "{{ gitea_runner_data_dir }}"
+    if "$BINARY" register \
+        --token "$REG_TOKEN" \
+        --name "$RUNNER_NAME" \
+        --instance "$GITEA_URL" \
+        --labels "$RUNNER_LABELS" \
+        --no-interactive 2>&1; then
+        echo "RECOVERED: runner re-registered successfully"
+    else
+        echo "ERROR: re-registration failed. Manual intervention required."
+        exit 3
+    fi
+
+    # Start the runner service
+    systemctl --user start gitea-runner.service
+    sleep 3
+
+    # Verify recovery — query the Gitea API to confirm the new ID is registered
+    NEW_ID=$(python3 -c "import json; print(json.load(open('$RUNNER_FILE')).get('id',''))" 2>/dev/null || true)
+    if [[ -n "$NEW_ID" ]]; then
+        new_found=$(curl -sf --connect-timeout 5 --max-time 10 \
+            -H "Authorization: token $API_TOKEN" \
+            "${GITEA_URL}/api/v1/admin/actions/runners" 2>/dev/null \
+            | python3 -c "
+import sys, json
+try:
+    data = json.load(sys.stdin)
+    runners = data if isinstance(data, list) else data.get('runners', [])
+    ids = [str(r.get('id', '')) for r in runners]
+    print('1' if '$NEW_ID' in ids else '0')
+except Exception:
+    print('0')
+" 2>/dev/null || echo "0")
+        if [[ "$new_found" == "1" ]]; then
+            echo "OK: runner recovered and registered with new ID $NEW_ID"
+            # Clear cooldown on success
+            rm -f "$COOLDOWN_FILE" 2>/dev/null || true
+        else
+            echo "WARN: runner re-registered but ID not found in Gitea API. Will retry after cooldown."
+            exit 3
+        fi
+    else
+        echo "WARN: could not read new .runner file after re-registration. Will retry after cooldown."
+        exit 3
+    fi
+{% else %}
+    echo "Manual re-registration required: re-run gitea_runner role with gitea_runner_force_reregister=true."
+    # Restart the service once in case it is a transient token refresh issue,
+    # but this cannot recover an unregistered runner without re-registration.
+    systemctl --user restart gitea-runner.service
+    sleep 2
+    exit 3
+{% endif %}
+fi
+
 # 3. Check disk space — prune aggressively if below threshold
 disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
 if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_threshold }} ]]; then
    echo "WARN: Disk usage at ${disk_pct}%, pruning all runner resources"
-    docker system prune -f --filter "label={{ gitea_runner_prune_label }}" --filter "until=1h" || true
-    docker volume prune -f --filter "label={{ gitea_runner_prune_label }}" || true
-    # Only prune dangling (untagged) images — keep tagged runner images (ci-full, ci-quality)
-    docker image prune -f || true
-    # Clean up stopped containers and dangling networks that accumulate from failed jobs
-    docker container prune -f || true
-    docker network prune -f || true
+    # Force-remove stale containers (including running ones from failed molecule tests)
+    # that are older than 1 hour. "docker container prune -f" only removes stopped
+    # containers, so running containers from crashed CI jobs accumulate and consume
+    # disk/memory. Exclude CI job containers (name starts with GITEA-ACTIONS-TASK).
+    # Only remove containers older than 1 hour to avoid killing molecule test
+    # containers that CI jobs are actively using.
+    docker ps -a --format '{% raw %}{{.ID}} {{.Names}} {{.RunningFor}}{% endraw %}' 2>/dev/null \
+        | grep -v 'GITEA-ACTIONS-TASK' \
+        | grep -E '(hour|day|week|month|year)s? ago' \
+        | awk '{print $1}' \
+        | xargs -r docker rm -f 2>/dev/null || true
+    docker system prune -af --filter "until=1h" --volumes || true
+    # Prune networks older than 1 hour to avoid removing networks that
+    # molecule tests are actively creating (e.g. 'traefik' network created
+    # during molecule create phase before containers are attached).
+    docker network prune -f --filter "until=1h" || true
    disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
    echo "INFO: Disk usage after prune: ${disk_pct}%"
 fi
@@ -33,5 +33,5 @@
      become_user: "{{ gitea_runner_service_user }}"
      environment:
        XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}"
-      when: systemd_available.stat.exists
+      when: gitea_runner_systemd_available.stat.exists
      changed_when: true
@@ -8,12 +8,12 @@ Each runner runs in an isolated **rootless Docker** environment under a dedicate

 [![CI](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions/workflows/ci.yml/badge.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
 [![License: GPL-3.0](https://img.shields.io/badge/license-GPL--3.0-blue)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/src/branch/master/LICENSE)
-[![Coverage](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/d9e3ee2e6cddc9f657e4a21bd62e649c9a828286/coverage.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
-[![Tests](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/d9e3ee2e6cddc9f657e4a21bd62e649c9a828286/tests.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
-[![Docs](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/d9e3ee2e6cddc9f657e4a21bd62e649c9a828286/docs.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/wiki)
-[![Code Quality](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/d9e3ee2e6cddc9f657e4a21bd62e649c9a828286/quality.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
-[![Version](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/d9e3ee2e6cddc9f657e4a21bd62e649c9a828286/version.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/releases)
-[![Python](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/d9e3ee2e6cddc9f657e4a21bd62e649c9a828286/python.svg)](https://www.python.org/downloads/)
+[![Coverage](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/3720ac3b2d4a1a832197f5c128d50f59f6fff163/coverage.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
+[![Tests](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/3720ac3b2d4a1a832197f5c128d50f59f6fff163/tests.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
+[![Docs](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/3720ac3b2d4a1a832197f5c128d50f59f6fff163/docs.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/wiki)
+[![Code Quality](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/3720ac3b2d4a1a832197f5c128d50f59f6fff163/quality.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
+[![Version](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/3720ac3b2d4a1a832197f5c128d50f59f6fff163/version.svg)](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/releases)
+[![Python](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/raw/commit/3720ac3b2d4a1a832197f5c128d50f59f6fff163/python.svg)](https://www.python.org/downloads/)

 ## Overview

@@ -36,7 +36,7 @@ ci = [
    "build==1.5.1",
    "twine==6.2.0",
    # Reusable CI/CD and dev tools (auto-merge, pr-review, pre-push checks, etc.)
-    "devx @ git+https://git.oblachno.oblachno.fyi/oblachno-oss/devx.git@v0.47.8",
+    "devx @ git+https://git.oblachno.oblachno.fyi/oblachno-oss/devx.git@v0.48.1",
 ]
 # Lint and type-checking tools (validate job)
 lint = [
@@ -56,7 +56,7 @@ molecule = [
 dev = [
    "grm[ci,lint,molecule]",
    # Reusable CI/CD and dev tools (pre-push hooks, create-task, create-pr)
-    "devx @ git+https://git.oblachno.oblachno.fyi/oblachno-oss/devx.git@v0.47.8",
+    "devx @ git+https://git.oblachno.oblachno.fyi/oblachno-oss/devx.git@v0.48.1",
    # Non-Python dev dependency: checkmake (Makefile linter)
    # Install via: go install github.com/checkmake/checkmake/cmd/checkmake@latest
 ]
@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+"""Clean up stale runner registrations from Gitea.
+
+A runner is considered stale if it hasn't been online for more than a
+configurable threshold (default: 1 hour).  Stale runners accumulate when:
+- A runner host is rebuilt or re-provisioned (old registration remains)
+- A runner is re-registered (old entry remains alongside the new one)
+- A runner process dies and the healthcheck can't auto-recover
+
+This script queries the Gitea API for all runners, identifies stale ones,
+and deletes them via ``DELETE /api/v1/admin/actions/runners/{id}``.
+
+Usage::
+
+    python3 scripts/cleanup_stale_runners.py --gitea-url https://git.example.com --token <admin-token>
+    python3 scripts/cleanup_stale_runners.py --gitea-url https://git.example.com --token <admin-token> --dry-run
+    python3 scripts/cleanup_stale_runners.py --gitea-url https://git.example.com --token <admin-token> \\
+        --stale-threshold 3600
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import time
+import urllib.error
+import urllib.request  # noqa: PTH123  # nosec B404
+from typing import Any
+
+
+def _api_request(base_url: str, token: str, method: str, path: str) -> Any:
+    url = f"{base_url.rstrip('/')}/api/v1{path}"
+    req = urllib.request.Request(url, method=method)  # nosec B310
+    req.add_header("Authorization", f"token {token}")
+    req.add_header("Accept", "application/json")
+    try:
+        with urllib.request.urlopen(req) as resp:  # noqa: PTH123  # nosec B310
+            if resp.status == 204:
+                return None
+            raw = resp.read()
+            return json.loads(raw) if raw else None
+    except urllib.error.HTTPError as e:
+        detail = e.read().decode("utf-8", errors="replace")
+        raise RuntimeError(f"Gitea API error {e.code}: {detail}") from e
+
+
+def list_runners(base_url: str, token: str) -> list[dict[str, Any]]:
+    data = _api_request(base_url, token, "GET", "/admin/actions/runners")
+    if data is None:
+        return []
+    if isinstance(data, list):
+        return data
+    if isinstance(data, dict):
+        return data.get("runners", [])
+    return []
+
+
+def delete_runner(base_url: str, token: str, runner_id: int) -> bool:
+    try:
+        _api_request(base_url, token, "DELETE", f"/admin/actions/runners/{runner_id}")
+        return True
+    except RuntimeError as e:
+        print(f"  ERROR deleting runner {runner_id}: {e}", file=sys.stderr)
+        return False
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description="Clean up stale Gitea runner registrations")
+    parser.add_argument("--gitea-url", required=True, help="Gitea base URL")
+    parser.add_argument("--token", required=True, help="Gitea admin API token")
+    parser.add_argument(
+        "--stale-threshold",
+        type=int,
+        default=3600,
+        help="Seconds since last_online before a runner is considered stale (default: 3600 = 1h)",
+    )
+    parser.add_argument("--dry-run", action="store_true", help="List stale runners without deleting")
+    args = parser.parse_args(argv)
+
+    runners = list_runners(args.gitea_url, args.token)
+    if not runners:
+        print("No runners found.")
+        return 0
+
+    now = int(time.time())
+    stale: list[dict[str, Any]] = []
+    online: list[dict[str, Any]] = []
+
+    for runner in runners:
+        last_online = runner.get("last_online", 0) or 0
+        seconds_since = now - last_online
+        runner["seconds_since_online"] = seconds_since
+        if seconds_since > args.stale_threshold:
+            stale.append(runner)
+        else:
+            online.append(runner)
+
+    print(f"Total runners: {len(runners)}")
+    print(f"Online (within {args.stale_threshold}s): {len(online)}")
+    print(f"Stale (>{args.stale_threshold}s): {len(stale)}")
+    print()
+
+    if not stale:
+        print("No stale runners to clean up.")
+        return 0
+
+    print("Stale runners:")
+    for r in stale:
+        rid = r.get("id", "?")
+        name = r.get("name", "?")
+        uuid = r.get("uuid", "?")[:8]
+        secs = r.get("seconds_since_online", 0)
+        hours = secs / 3600
+        print(f"  id={rid} name={name} uuid={uuid}... offline={hours:.1f}h ago")
+
+    if args.dry_run:
+        print("\n--dry-run: not deleting. Remove --dry-run to clean up.")
+        return 0
+
+    print(f"\nDeleting {len(stale)} stale runners...")
+    deleted = 0
+    for r in stale:
+        rid = r.get("id")
+        if rid is None:
+            continue
+        if delete_runner(args.gitea_url, args.token, rid):
+            deleted += 1
+            print(f"  Deleted runner id={rid} ({r.get('name', '?')})")
+
+    print(f"\nDone: {deleted}/{len(stale)} stale runners deleted.")
+    return 0 if deleted == len(stale) else 1
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())
@@ -0,0 +1,217 @@
+"""Tests for cleanup_stale_runners.py."""
+
+from __future__ import annotations
+
+import time
+from unittest.mock import MagicMock, patch
+
+from scripts.cleanup_stale_runners import (
+    _api_request,
+    delete_runner,
+    list_runners,
+    main,
+)
+
+
+class TestListRunners:
+    """Tests for list_runners()."""
+
+    @patch("scripts.cleanup_stale_runners._api_request")
+    def test_returns_list_of_runners(self, mock_req: MagicMock) -> None:
+        mock_req.return_value = [{"id": 1, "name": "runner-1"}, {"id": 2, "name": "runner-2"}]
+        result = list_runners("https://git.example.com", "token")
+        assert len(result) == 2
+        assert result[0]["id"] == 1
+
+    @patch("scripts.cleanup_stale_runners._api_request")
+    def test_returns_empty_on_none(self, mock_req: MagicMock) -> None:
+        mock_req.return_value = None
+        result = list_runners("https://git.example.com", "token")
+        assert result == []
+
+    @patch("scripts.cleanup_stale_runners._api_request")
+    def test_extracts_runners_from_dict(self, mock_req: MagicMock) -> None:
+        mock_req.return_value = {"runners": [{"id": 1}]}
+        result = list_runners("https://git.example.com", "token")
+        assert len(result) == 1
+        assert result[0]["id"] == 1
+
+    @patch("scripts.cleanup_stale_runners._api_request")
+    def test_returns_empty_on_non_list_non_dict(self, mock_req: MagicMock) -> None:
+        mock_req.return_value = "not a list"
+        result = list_runners("https://git.example.com", "token")
+        assert result == []
+
+
+class TestDeleteRunner:
+    """Tests for delete_runner()."""
+
+    @patch("scripts.cleanup_stale_runners._api_request")
+    def test_returns_true_on_success(self, mock_req: MagicMock) -> None:
+        mock_req.return_value = None
+        assert delete_runner("https://git.example.com", "token", 42) is True
+
+    @patch("scripts.cleanup_stale_runners._api_request")
+    def test_returns_false_on_error(self, mock_req: MagicMock) -> None:
+        mock_req.side_effect = RuntimeError("API error 404: not found")
+        assert delete_runner("https://git.example.com", "token", 42) is False
+
+
+class TestApiRequest:
+    """Tests for _api_request()."""
+
+    @patch("scripts.cleanup_stale_runners.urllib.request.urlopen")
+    def test_returns_json_on_success(self, mock_urlopen: MagicMock) -> None:
+        mock_resp = MagicMock()
+        mock_resp.status = 200
+        mock_resp.read.return_value = b'{"key": "value"}'
+        mock_urlopen.return_value.__enter__.return_value = mock_resp
+        result = _api_request("https://git.example.com", "token", "GET", "/test")
+        assert result == {"key": "value"}
+
+    @patch("scripts.cleanup_stale_runners.urllib.request.urlopen")
+    def test_returns_none_on_204(self, mock_urlopen: MagicMock) -> None:
+        mock_resp = MagicMock()
+        mock_resp.status = 204
+        mock_urlopen.return_value.__enter__.return_value = mock_resp
+        result = _api_request("https://git.example.com", "token", "DELETE", "/test/1")
+        assert result is None
+
+    @patch("scripts.cleanup_stale_runners.urllib.request.urlopen")
+    def test_returns_none_on_empty_body(self, mock_urlopen: MagicMock) -> None:
+        mock_resp = MagicMock()
+        mock_resp.status = 200
+        mock_resp.read.return_value = b""
+        mock_urlopen.return_value.__enter__.return_value = mock_resp
+        result = _api_request("https://git.example.com", "token", "GET", "/test")
+        assert result is None
+
+    @patch("scripts.cleanup_stale_runners.urllib.request.urlopen")
+    def test_raises_on_http_error(self, mock_urlopen: MagicMock) -> None:
+        import urllib.error
+
+        mock_error = urllib.error.HTTPError(
+            "url",
+            404,
+            "Not Found",
+            {},
+            None,
+        )
+        mock_error.read = MagicMock(return_value=b'{"message": "not found"}')
+        mock_urlopen.side_effect = mock_error
+        import pytest
+
+        with pytest.raises(RuntimeError, match="404"):
+            _api_request("https://git.example.com", "token", "GET", "/test")
+
+
+class TestMain:
+    """Tests for main()."""
+
+    @patch("scripts.cleanup_stale_runners.list_runners")
+    def test_no_runners(self, mock_list: MagicMock) -> None:
+        mock_list.return_value = []
+        rc = main(["--gitea-url", "https://git.example.com", "--token", "t"])
+        assert rc == 0
+
+    @patch("scripts.cleanup_stale_runners.list_runners")
+    def test_no_stale_runners(self, mock_list: MagicMock) -> None:
+        now = int(time.time())
+        mock_list.return_value = [
+            {"id": 1, "name": "runner-1", "last_online": now - 60},
+        ]
+        rc = main(["--gitea-url", "https://git.example.com", "--token", "t"])
+        assert rc == 0
+
+    @patch("scripts.cleanup_stale_runners.list_runners")
+    def test_dry_run_does_not_delete(self, mock_list: MagicMock) -> None:
+        now = int(time.time())
+        mock_list.return_value = [
+            {"id": 1, "name": "runner-1", "last_online": now - 7200},
+        ]
+        with patch("scripts.cleanup_stale_runners.delete_runner") as mock_del:
+            rc = main(
+                [
+                    "--gitea-url",
+                    "https://git.example.com",
+                    "--token",
+                    "t",
+                    "--dry-run",
+                ]
+            )
+            assert rc == 0
+            mock_del.assert_not_called()
+
+    @patch("scripts.cleanup_stale_runners.list_runners")
+    @patch("scripts.cleanup_stale_runners.delete_runner")
+    def test_deletes_stale_runners(self, mock_del: MagicMock, mock_list: MagicMock) -> None:
+        now = int(time.time())
+        mock_list.return_value = [
+            {"id": 1, "name": "runner-1", "last_online": now - 60},
+            {"id": 2, "name": "runner-2", "last_online": now - 7200},
+            {"id": 3, "name": "runner-3", "last_online": now - 9999},
+        ]
+        mock_del.return_value = True
+        rc = main(
+            [
+                "--gitea-url",
+                "https://git.example.com",
+                "--token",
+                "t",
+                "--stale-threshold",
+                "3600",
+            ]
+        )
+        assert rc == 0
+        assert mock_del.call_count == 2
+
+    @patch("scripts.cleanup_stale_runners.list_runners")
+    @patch("scripts.cleanup_stale_runners.delete_runner")
+    def test_returns_1_on_partial_failure(self, mock_del: MagicMock, mock_list: MagicMock) -> None:
+        now = int(time.time())
+        mock_list.return_value = [
+            {"id": 1, "name": "runner-1", "last_online": now - 7200},
+            {"id": 2, "name": "runner-2", "last_online": now - 7200},
+        ]
+        mock_del.side_effect = [True, False]
+        rc = main(["--gitea-url", "https://git.example.com", "--token", "t"])
+        assert rc == 1
+
+    @patch("scripts.cleanup_stale_runners.list_runners")
+    def test_runner_with_zero_last_online(self, mock_list: MagicMock) -> None:
+        """Runners with last_online=0 should be considered stale."""
+        mock_list.return_value = [
+            {"id": 1, "name": "runner-1", "last_online": 0},
+        ]
+        with patch("scripts.cleanup_stale_runners.delete_runner") as mock_del:
+            mock_del.return_value = True
+            rc = main(["--gitea-url", "https://git.example.com", "--token", "t"])
+            assert rc == 0
+            mock_del.assert_called_once()
+
+    @patch("scripts.cleanup_stale_runners.list_runners")
+    def test_runner_with_missing_last_online(self, mock_list: MagicMock) -> None:
+        """Runners with missing last_online should be considered stale."""
+        mock_list.return_value = [
+            {"id": 1, "name": "runner-1"},
+        ]
+        with patch("scripts.cleanup_stale_runners.delete_runner") as mock_del:
+            mock_del.return_value = True
+            rc = main(["--gitea-url", "https://git.example.com", "--token", "t"])
+            assert rc == 0
+            mock_del.assert_called_once()
+
+    @patch("scripts.cleanup_stale_runners.list_runners")
+    @patch("scripts.cleanup_stale_runners.delete_runner")
+    def test_skips_runner_with_none_id(self, mock_del: MagicMock, mock_list: MagicMock) -> None:
+        """Runners with id=None should be skipped during deletion."""
+        now = int(time.time())
+        mock_list.return_value = [
+            {"id": None, "name": "bad-runner", "last_online": now - 7200},
+            {"id": 2, "name": "runner-2", "last_online": now - 7200},
+        ]
+        mock_del.return_value = True
+        rc = main(["--gitea-url", "https://git.example.com", "--token", "t"])
+        # 1/2 deleted (None id skipped), so rc=1 (partial)
+        assert rc == 1
+        mock_del.assert_called_once_with("https://git.example.com", "t", 2)
@@ -145,11 +145,25 @@ def cli(ctx: click.Context, become_password_file: str | None, verbose: bool) ->
        "Example: docker:docker://alpine:latest"
    ),
 )
+@click.option(
+    "--force-reregister/--no-force-reregister",
+    default=False,
+    help=_("Force re-registration even if .runner file exists (env: GITEA_FORCE_REREGISTER)"),
+)
@click.option(
    "--ask-become-pass/--no-ask-become-pass",
    default=True,
    help=_("Prompt for sudo password (default)"),
 )
+@click.option(
+    "--auto-recover-token",
+    default=None,
+    help=_(
+        "Gitea API token for healthcheck auto-recovery (env: GITEA_AUTO_RECOVER_TOKEN). "
+        "When set, the healthcheck can automatically re-register the runner "
+        "if it becomes unregistered. Requires admin or org-level access."
+    ),
+)
@_handle_errors("Installation failed: {error}")
 def install(
    host: str,
@@ -161,10 +175,14 @@ def install(
    admin_token: str | None,
    integration_retries: int,
    labels: str | None,
+    force_reregister: bool,
    ask_become_pass: bool,
+    auto_recover_token: str | None,
 ) -> None:
    if labels is None:
        labels = os.getenv("GITEA_RUNNER_LABELS")
+    if auto_recover_token is None:
+        auto_recover_token = os.getenv("GITEA_AUTO_RECOVER_TOKEN")
    manager = RunnerManager()
    manager.install(
        host=host,
@@ -176,9 +194,11 @@ def install(
        admin_token=admin_token,
        integration_retries=integration_retries,
        labels=labels,
+        force_reregister=force_reregister,
        ask_become_pass=ask_become_pass,
        become_password_file=_get_become_password_file(),
        verbose=_get_verbose(),
+        auto_recover_token=auto_recover_token,
    )


@@ -506,7 +526,7 @@ def list_runners(ask_become_pass: bool, no_status: bool) -> None:
    click.echo(f"{_('NAME'):<18} {_('HOST'):<16} {_('USER'):<10} {_('LABELS'):<30} {_('STATUS')}")
    click.echo("-" * 90)
    for r in runners:
-        click.echo(f"{r['name']:<18} {r['host']:<16} {r['user']:<10} {r['labels']:<30} {r['status']}")
+        click.echo(f"{r['name']:<18} {r['host']:<16} {r['user']:<10} {(r['labels'] or ''):<30} {r['status']}")


@cli.command(name="trigger-workflow", help=_("Trigger a Gitea Actions workflow via the API."))
@@ -87,8 +87,10 @@ class RunnerManager:
        integration_retries: int = 3,
        ask_become_pass: bool = False,
        labels: str | None = None,
+        force_reregister: bool = False,
        become_password_file: str | None = None,
        verbose: bool = False,
+        auto_recover_token: str | None = None,
    ) -> None:
        """Install a runner on a remote host using Ansible."""
        if not name:
@@ -98,16 +100,19 @@ class RunnerManager:
        if not token:
            raise AnsibleError(_("GITEA_REGISTRATION_TOKEN must be set (or pass --token)"))

-        extra_vars: dict[str, str | int] = {
+        extra_vars: dict[str, str | int | bool] = {
            "registration_token": token,
            "gitea_runner_name": name,
            "gitea_url": gitea_url,
            "gitea_runner_integration_retries": integration_retries,
+            "gitea_runner_force_reregister": force_reregister,
        }
        if admin_token:
            extra_vars["gitea_admin_token"] = admin_token
        if labels is not None:
            extra_vars["gitea_runner_labels"] = labels
+        if auto_recover_token:
+            extra_vars["gitea_runner_auto_recover_api_token"] = auto_recover_token

        with track_steps() as tracker:
            tracker.begin(_("Installing Gitea Runner on {host}", host=host))
@@ -367,6 +367,14 @@
    "ru": "Токен регистрации (env: GITEA_REGISTRATION_TOKEN)",
    "zh": "注册令牌（环境变量: GITEA_REGISTRATION_TOKEN）"
  },
+  "Force re-registration even if .runner file exists (env: GITEA_FORCE_REREGISTER)": {
+    "bg": "Принудителна повторна регистрация, дори ако .runner файлът съществува (env: GITEA_FORCE_REREGISTER)",
+    "de": "Erneute Registrierung erzwingen, auch wenn .runner-Datei existiert (env: GITEA_FORCE_REREGISTER)",
+    "en": "Force re-registration even if .runner file exists (env: GITEA_FORCE_REREGISTER)",
+    "pl": "Wymuś ponowną rejestrację, nawet jeśli plik .runner istnieje (env: GITEA_FORCE_REREGISTER)",
+    "ru": "Принудительно повторно зарегистрировать, даже если файл .runner существует (env: GITEA_FORCE_REREGISTER)",
+    "zh": "即使存在 .runner 文件也强制重新注册（环境变量: GITEA_FORCE_REREGISTER）"
+  },
  "Remove a registered Gitea Runner completely.": {
    "bg": "Пълно премахване на регистриран Gitea Runner.",
    "de": "Einen registrierten Gitea Runner vollständig entfernen.",
@@ -774,5 +782,13 @@
    "pl": "Workflow uruchomiony pomyślnie. ID uruchomienia: {run_id}",
    "ru": "Workflow успешно запущен. ID запуска: {run_id}",
    "zh": "工作流触发成功。运行 ID：{run_id}"
+  },
+  "Gitea API token for healthcheck auto-recovery (env: GITEA_AUTO_RECOVER_TOKEN). When set, the healthcheck can automatically re-register the runner if it becomes unregistered. Requires admin or org-level access.": {
+    "bg": "Gitea API token for healthcheck auto-recovery (env: GITEA_AUTO_RECOVER_TOKEN). When set, the healthcheck can automatically re-register the runner if it becomes unregistered. Requires admin or org-level access.",
+    "de": "Gitea API token for healthcheck auto-recovery (env: GITEA_AUTO_RECOVER_TOKEN). When set, the healthcheck can automatically re-register the runner if it becomes unregistered. Requires admin or org-level access.",
+    "en": "Gitea API token for healthcheck auto-recovery (env: GITEA_AUTO_RECOVER_TOKEN). When set, the healthcheck can automatically re-register the runner if it becomes unregistered. Requires admin or org-level access.",
+    "pl": "Gitea API token for healthcheck auto-recovery (env: GITEA_AUTO_RECOVER_TOKEN). When set, the healthcheck can automatically re-register the runner if it becomes unregistered. Requires admin or org-level access.",
+    "ru": "Gitea API token for healthcheck auto-recovery (env: GITEA_AUTO_RECOVER_TOKEN). When set, the healthcheck can automatically re-register the runner if it becomes unregistered. Requires admin or org-level access.",
+    "zh": "Gitea API token for healthcheck auto-recovery (env: GITEA_AUTO_RECOVER_TOKEN). When set, the healthcheck can automatically re-register the runner if it becomes unregistered. Requires admin or org-level access."
  }
 }
@@ -50,9 +50,11 @@ class TestCLI:
            admin_token="",
            integration_retries=3,
            labels=None,
+            force_reregister=False,
            ask_become_pass=True,
            become_password_file=None,
            verbose=False,
+            auto_recover_token=None,
        )

    @patch("grm.cli.RunnerManager")
@@ -73,9 +75,11 @@ class TestCLI:
            admin_token="",
            integration_retries=3,
            labels=None,
+            force_reregister=False,
            ask_become_pass=False,
            become_password_file=None,
            verbose=False,
+            auto_recover_token=None,
        )

    @patch("grm.cli.RunnerManager")
@@ -114,9 +118,11 @@ class TestCLI:
                admin_token=None,
                integration_retries=3,
                labels=None,
+                force_reregister=False,
                ask_become_pass=True,
                become_password_file=None,
                verbose=False,
+                auto_recover_token=None,
            )

    @patch("grm.cli.RunnerManager")
@@ -165,9 +171,11 @@ class TestCLI:
            admin_token="",
            integration_retries=3,
            labels=None,
+            force_reregister=False,
            ask_become_pass=True,
            become_password_file=None,
            verbose=False,
+            auto_recover_token=None,
        )

    @patch("grm.cli.RunnerManager")
@@ -188,9 +196,11 @@ class TestCLI:
            admin_token="",
            integration_retries=3,
            labels=None,
+            force_reregister=False,
            ask_become_pass=True,
            become_password_file=None,
            verbose=False,
+            auto_recover_token=None,
        )

    @patch("grm.cli.RunnerManager")
@@ -226,9 +236,92 @@ class TestCLI:
            admin_token="",
            integration_retries=3,
            labels="docker:docker://alpine:latest",
+            force_reregister=False,
            ask_become_pass=True,
            become_password_file=None,
            verbose=False,
+            auto_recover_token=None,
+        )
+
+    @patch("grm.cli.RunnerManager")
+    def test_install_force_reregister(self, mock_manager_class: MagicMock) -> None:
+        mock_manager = MagicMock()
+        mock_manager_class.return_value = mock_manager
+
+        runner = CliRunner(env=_TEST_ENV)
+        result = runner.invoke(cli, ["install", "host1", "--user", "ubuntu", "--token", "tok", "--force-reregister"])
+        assert result.exit_code == 0
+        mock_manager.install.assert_called_once_with(
+            host="host1",
+            user="ubuntu",
+            key=None,
+            name=None,
+            token="tok",
+            gitea_url="https://git.example.com",
+            admin_token="",
+            integration_retries=3,
+            labels=None,
+            force_reregister=True,
+            ask_become_pass=True,
+            become_password_file=None,
+            verbose=False,
+            auto_recover_token=None,
+        )
+
+    @patch("grm.cli.RunnerManager")
+    def test_install_with_auto_recover_token(self, mock_manager_class: MagicMock) -> None:
+        """--auto-recover-token passes the token to the manager for healthcheck auto-recovery."""
+        mock_manager = MagicMock()
+        mock_manager_class.return_value = mock_manager
+
+        runner = CliRunner(env=_TEST_ENV)
+        result = runner.invoke(
+            cli,
+            ["install", "host1", "--user", "ubuntu", "--token", "tok", "--auto-recover-token", "api-tok"],
+        )
+        assert result.exit_code == 0
+        mock_manager.install.assert_called_once_with(
+            host="host1",
+            user="ubuntu",
+            key=None,
+            name=None,
+            token="tok",
+            gitea_url="https://git.example.com",
+            admin_token="",
+            integration_retries=3,
+            labels=None,
+            force_reregister=False,
+            ask_become_pass=True,
+            become_password_file=None,
+            verbose=False,
+            auto_recover_token="api-tok",
+        )
+
+    @patch("grm.cli.RunnerManager")
+    def test_install_auto_recover_token_from_env(self, mock_manager_class: MagicMock) -> None:
+        """GITEA_AUTO_RECOVER_TOKEN env var is used when --auto-recover-token is not passed."""
+        mock_manager = MagicMock()
+        mock_manager_class.return_value = mock_manager
+
+        env = {**_TEST_ENV, "GITEA_AUTO_RECOVER_TOKEN": "env-tok"}
+        runner = CliRunner(env=env)
+        result = runner.invoke(cli, ["install", "host1", "--user", "ubuntu", "--token", "tok"])
+        assert result.exit_code == 0
+        mock_manager.install.assert_called_once_with(
+            host="host1",
+            user="ubuntu",
+            key=None,
+            name=None,
+            token="tok",
+            gitea_url="https://git.example.com",
+            admin_token="",
+            integration_retries=3,
+            labels=None,
+            force_reregister=False,
+            ask_become_pass=True,
+            become_password_file=None,
+            verbose=False,
+            auto_recover_token="env-tok",
        )

    @patch("grm.cli.RunnerManager")
@@ -250,9 +343,11 @@ class TestCLI:
            admin_token="",
            integration_retries=3,
            labels="",
+            force_reregister=False,
            ask_become_pass=True,
            become_password_file=None,
            verbose=False,
+            auto_recover_token=None,
        )

    @patch("grm.cli.RunnerManager")
@@ -274,9 +369,11 @@ class TestCLI:
            admin_token="",
            integration_retries=3,
            labels="docker:docker://alpine:latest",
+            force_reregister=False,
            ask_become_pass=True,
            become_password_file=None,
            verbose=False,
+            auto_recover_token=None,
        )

    @patch("grm.cli.RunnerManager")
@@ -307,9 +404,11 @@ class TestCLI:
                admin_token="",
                integration_retries=3,
                labels=None,
+                force_reregister=False,
                ask_become_pass=True,
                become_password_file=pw_file,
                verbose=False,
+                auto_recover_token=None,
            )
        finally:
            import os
@@ -334,9 +433,11 @@ class TestCLI:
            admin_token="",
            integration_retries=3,
            labels=None,
+            force_reregister=False,
            ask_become_pass=True,
            become_password_file=None,
            verbose=True,
+            auto_recover_token=None,
        )

    @patch("grm.cli.RunnerManager")
@@ -741,6 +842,27 @@ class TestCLI:
        assert "active" in result.output
        mock_manager.list_runners.assert_called_once_with(become_pass=None, no_status=False)

+    @patch("grm.cli.RunnerManager")
+    def test_list_with_none_labels(self, mock_manager_class: MagicMock) -> None:
+        """Runners with labels=None should not crash the list command."""
+        mock_manager = MagicMock()
+        mock_manager.list_runners.return_value = [
+            {
+                "name": "r1",
+                "host": "10.0.0.1",
+                "user": "ubuntu",
+                "labels": None,
+                "status": "active",
+            },
+        ]
+        mock_manager_class.return_value = mock_manager
+
+        runner = CliRunner()
+        result = runner.invoke(cli, ["list"])
+        assert result.exit_code == 0
+        assert "r1" in result.output
+        assert "active" in result.output
+
    @patch("grm.cli.RunnerManager")
    def test_list_no_status(self, mock_manager_class: MagicMock) -> None:
        """--no-status skips SSH checks and shows registry only."""
@@ -51,6 +51,7 @@ class TestRunnerManager:
        assert manager._captured_extra_vars["registration_token"] == "tok"
        assert manager._captured_extra_vars["gitea_runner_name"] == "192.168.1.10"
        assert manager._captured_extra_vars["gitea_url"] == "https://git.example.com"
+        assert manager._captured_extra_vars["gitea_runner_force_reregister"] is False
        assert "Installing Gitea Runner on 192.168.1.10" in mock_executor.run.call_args.kwargs["description"]
        mock_registry.add.assert_called_once_with(
            name="192.168.1.10",
@@ -76,6 +77,7 @@ class TestRunnerManager:
        assert "/key" in cmd_str
        assert manager._captured_extra_vars["registration_token"] == "preset"
        assert manager._captured_extra_vars["gitea_runner_name"] == "my-runner"
+        assert manager._captured_extra_vars["gitea_runner_force_reregister"] is False
        assert "--ask-become-pass" not in cmd_str
        mock_registry.add.assert_called_once_with(
            name="my-runner",
@@ -97,6 +99,52 @@ class TestRunnerManager:
        cmd_str = " ".join(cmd)
        assert "--ask-become-pass" in cmd_str

+    def test_install_force_reregister(self) -> None:
+        mock_registry = MagicMock()
+        manager = RunnerManager(registry=mock_registry)
+        mock_executor = MagicMock()
+        manager._executor = mock_executor
+
+        manager.install(
+            "host1",
+            "root",
+            token="tok",
+            gitea_url="https://git.example.com",
+            force_reregister=True,
+        )
+        assert manager._captured_extra_vars["gitea_runner_force_reregister"] is True
+
+    def test_install_with_auto_recover_token(self) -> None:
+        """auto_recover_token is passed as extra_var to Ansible."""
+        mock_registry = MagicMock()
+        manager = RunnerManager(registry=mock_registry)
+        mock_executor = MagicMock()
+        manager._executor = mock_executor
+
+        manager.install(
+            "host1",
+            "root",
+            token="tok",
+            gitea_url="https://git.example.com",
+            auto_recover_token="api-tok",
+        )
+        assert manager._captured_extra_vars["gitea_runner_auto_recover_api_token"] == "api-tok"
+
+    def test_install_without_auto_recover_token(self) -> None:
+        """When auto_recover_token is None, the extra_var is not set."""
+        mock_registry = MagicMock()
+        manager = RunnerManager(registry=mock_registry)
+        mock_executor = MagicMock()
+        manager._executor = mock_executor
+
+        manager.install(
+            "host1",
+            "root",
+            token="tok",
+            gitea_url="https://git.example.com",
+        )
+        assert "gitea_runner_auto_recover_api_token" not in manager._captured_extra_vars
+
    def test_install_missing_gitea_url(self) -> None:
        manager = RunnerManager()
        with pytest.raises(AnsibleError, match="GITEA_URL must be set"):