Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 grm-ci-bot
|
4e9d033a40 | ||
 emil
|
63ef5cdbcf | ||
|
gitea-actions-bot
|
8fbe2d3f51 | ||
|
emil
|
a9178714af | ||
|
gitea-actions-bot
|
6ffcc38181 |
@@ -161,7 +161,7 @@ jobs:
|
||||
fail-fast: true
|
||||
max-parallel: 3
|
||||
matrix:
|
||||
runner-index: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
|
||||
runner-index: [1, 2, 3, 4, 5, 6]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Set up environment
|
||||
|
||||
@@ -2,6 +2,13 @@
|
||||
|
||||
All notable changes to this project will be documented in this file.
|
||||
|
||||
## [0.12.5] - 2026-06-30
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Right-size molecule-tests matrix to [1-6]
|
||||
- Cast disk threshold to string in template-content verify assertion
|
||||
|
||||
## [0.12.4] - 2026-06-29
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
@@ -8,12 +8,12 @@ Each runner runs in an isolated **rootless Docker** environment under a dedicate
|
||||
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/src/branch/master/LICENSE)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/wiki)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/releases)
|
||||
[](https://www.python.org/downloads/)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/wiki)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/releases)
|
||||
[](https://www.python.org/downloads/)
|
||||
|
||||
## Why GRM?
|
||||
|
||||
|
||||
@@ -18,6 +18,16 @@
|
||||
when: systemd_available.stat.exists
|
||||
changed_when: true
|
||||
|
||||
- name: Stop and disable healthcheck timer
|
||||
ansible.builtin.command: systemctl --user stop --disable runner-healthcheck.timer
|
||||
become: true
|
||||
become_user: "{{ gitea_runner_service_user | default('grm-' ~ runner_name) }}"
|
||||
environment:
|
||||
XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid | default('') }}"
|
||||
when: systemd_available.stat.exists
|
||||
changed_when: true
|
||||
failed_when: false
|
||||
|
||||
- name: Include deregistration
|
||||
ansible.builtin.include_role:
|
||||
name: gitea-runner
|
||||
|
||||
@@ -79,6 +79,16 @@
|
||||
tasks_from: deregister.yml
|
||||
when: not skip_runner_registration | default(false)
|
||||
|
||||
- name: Stop and disable healthcheck timer
|
||||
ansible.builtin.command: systemctl --user stop --disable runner-healthcheck.timer
|
||||
become: true
|
||||
become_user: "{{ gitea_runner_service_user | default('grm-' ~ runner_name) }}"
|
||||
environment:
|
||||
XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid | default('') }}"
|
||||
when: systemd_available.stat.exists
|
||||
changed_when: true
|
||||
failed_when: false
|
||||
|
||||
- name: Remove docker-prune user service file
|
||||
ansible.builtin.file:
|
||||
path: "{{ gitea_runner_home | default('/home/grm-' ~ runner_name) }}/.config/systemd/user/docker-prune.service"
|
||||
@@ -91,6 +101,24 @@
|
||||
state: absent
|
||||
failed_when: false
|
||||
|
||||
- name: Remove healthcheck user service file
|
||||
ansible.builtin.file:
|
||||
path: "{{ gitea_runner_home | default('/home/grm-' ~ runner_name) }}/.config/systemd/user/runner-healthcheck.service"
|
||||
state: absent
|
||||
failed_when: false
|
||||
|
||||
- name: Remove healthcheck user timer file
|
||||
ansible.builtin.file:
|
||||
path: "{{ gitea_runner_home | default('/home/grm-' ~ runner_name) }}/.config/systemd/user/runner-healthcheck.timer"
|
||||
state: absent
|
||||
failed_when: false
|
||||
|
||||
- name: Remove healthcheck script
|
||||
ansible.builtin.file:
|
||||
path: "{{ gitea_runner_config_dir | default('/etc/gitea-runner/' ~ runner_name) }}/healthcheck.sh"
|
||||
state: absent
|
||||
failed_when: false
|
||||
|
||||
- name: Remove systemd user unit file
|
||||
ansible.builtin.file:
|
||||
path: "{{ gitea_runner_home | default('/home/grm-' ~ runner_name) }}/.config/systemd/user/gitea-runner.service"
|
||||
|
||||
@@ -24,6 +24,12 @@ gitea_runner_prune_label: "gitea-runner=true"
|
||||
# Service configuration
|
||||
gitea_runner_service_restart_sec: "5"
|
||||
|
||||
# Health check configuration
|
||||
gitea_runner_healthcheck_interval: "5min"
|
||||
gitea_runner_healthcheck_boot_delay: "2min"
|
||||
gitea_runner_healthcheck_disk_threshold: 85
|
||||
gitea_runner_healthcheck_script_path: "{{ gitea_runner_config_dir }}/healthcheck.sh"
|
||||
|
||||
# Admin token for runner deregistration via Gitea API.
|
||||
# If not set, falls back to registration_token (which likely lacks admin scope).
|
||||
# Set this to a token with admin scope to enable automatic runner cleanup on removal.
|
||||
|
||||
@@ -75,3 +75,42 @@
|
||||
that:
|
||||
- timer_stat.stat.exists
|
||||
fail_msg: "Docker prune timer is missing"
|
||||
|
||||
- name: Check healthcheck script exists
|
||||
ansible.builtin.stat:
|
||||
path: "{{ gitea_runner_healthcheck_script_path }}"
|
||||
register: healthcheck_script_stat
|
||||
|
||||
- name: Assert healthcheck script exists
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- healthcheck_script_stat.stat.exists
|
||||
fail_msg: "Healthcheck script is missing"
|
||||
|
||||
- name: Assert healthcheck script is executable
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- healthcheck_script_stat.stat.mode == "0755"
|
||||
fail_msg: "Healthcheck script is not executable"
|
||||
|
||||
- name: Check healthcheck service exists
|
||||
ansible.builtin.stat:
|
||||
path: "{{ gitea_runner_home }}/.config/systemd/user/runner-healthcheck.service"
|
||||
register: healthcheck_service_stat
|
||||
|
||||
- name: Assert healthcheck service exists
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- healthcheck_service_stat.stat.exists
|
||||
fail_msg: "Healthcheck systemd service is missing"
|
||||
|
||||
- name: Check healthcheck timer exists
|
||||
ansible.builtin.stat:
|
||||
path: "{{ gitea_runner_home }}/.config/systemd/user/runner-healthcheck.timer"
|
||||
register: healthcheck_timer_stat
|
||||
|
||||
- name: Assert healthcheck timer exists
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- healthcheck_timer_stat.stat.exists
|
||||
fail_msg: "Healthcheck systemd timer is missing"
|
||||
|
||||
@@ -89,6 +89,39 @@
|
||||
- not prune_timer_stat.stat.exists
|
||||
fail_msg: "docker-prune timer unit still exists after removal"
|
||||
|
||||
- name: Check healthcheck service unit is absent
|
||||
ansible.builtin.stat:
|
||||
path: "{{ gitea_runner_home }}/.config/systemd/user/runner-healthcheck.service"
|
||||
register: healthcheck_service_stat
|
||||
|
||||
- name: Assert healthcheck service unit is absent
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- not healthcheck_service_stat.stat.exists
|
||||
fail_msg: "runner-healthcheck service unit still exists after removal"
|
||||
|
||||
- name: Check healthcheck timer unit is absent
|
||||
ansible.builtin.stat:
|
||||
path: "{{ gitea_runner_home }}/.config/systemd/user/runner-healthcheck.timer"
|
||||
register: healthcheck_timer_stat
|
||||
|
||||
- name: Assert healthcheck timer unit is absent
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- not healthcheck_timer_stat.stat.exists
|
||||
fail_msg: "runner-healthcheck timer unit still exists after removal"
|
||||
|
||||
- name: Check healthcheck script is absent
|
||||
ansible.builtin.stat:
|
||||
path: "{{ gitea_runner_config_dir }}/healthcheck.sh"
|
||||
register: healthcheck_script_stat
|
||||
|
||||
- name: Assert healthcheck script is absent
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- not healthcheck_script_stat.stat.exists
|
||||
fail_msg: "healthcheck script still exists after removal"
|
||||
|
||||
- name: Check subuid entry is absent
|
||||
ansible.builtin.command: "grep -c '^{{ gitea_runner_service_user }}:' /etc/subuid"
|
||||
register: subuid_check
|
||||
|
||||
@@ -62,3 +62,45 @@
|
||||
- "'OnCalendar={{ gitea_runner_prune_schedule }}' in prune_timer.content | b64decode"
|
||||
- "'Persistent=true' in prune_timer.content | b64decode"
|
||||
fail_msg: "Prune timer template is missing expected directives"
|
||||
|
||||
- name: Read rendered healthcheck service template
|
||||
ansible.builtin.slurp:
|
||||
src: "{{ gitea_runner_home }}/.config/systemd/user/runner-healthcheck.service"
|
||||
register: healthcheck_service
|
||||
|
||||
- name: Assert healthcheck service contains expected directives
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- "'Type=oneshot' in healthcheck_service.content | b64decode"
|
||||
- "'ExecStart={{ gitea_runner_healthcheck_script_path }}' in healthcheck_service.content | b64decode"
|
||||
- "'DOCKER_HOST=unix:///run/user/' in healthcheck_service.content | b64decode"
|
||||
- "'XDG_RUNTIME_DIR=/run/user/' in healthcheck_service.content | b64decode"
|
||||
fail_msg: "Healthcheck service template is missing expected directives"
|
||||
|
||||
- name: Read rendered healthcheck timer template
|
||||
ansible.builtin.slurp:
|
||||
src: "{{ gitea_runner_home }}/.config/systemd/user/runner-healthcheck.timer"
|
||||
register: healthcheck_timer
|
||||
|
||||
- name: Assert healthcheck timer contains expected directives
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- "'OnBootSec={{ gitea_runner_healthcheck_boot_delay }}' in healthcheck_timer.content | b64decode"
|
||||
- "'OnUnitActiveSec={{ gitea_runner_healthcheck_interval }}' in healthcheck_timer.content | b64decode"
|
||||
- "'Persistent=true' in healthcheck_timer.content | b64decode"
|
||||
fail_msg: "Healthcheck timer template is missing expected directives"
|
||||
|
||||
- name: Read rendered healthcheck script
|
||||
ansible.builtin.slurp:
|
||||
src: "{{ gitea_runner_healthcheck_script_path }}"
|
||||
register: healthcheck_script
|
||||
|
||||
- name: Assert healthcheck script contains expected content
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- "'docker info' in healthcheck_script.content | b64decode"
|
||||
- "'systemctl --user restart docker.service' in healthcheck_script.content | b64decode"
|
||||
- "'systemctl --user restart gitea-runner.service' in healthcheck_script.content | b64decode"
|
||||
- "'docker system prune' in healthcheck_script.content | b64decode"
|
||||
- "gitea_runner_healthcheck_disk_threshold | string in healthcheck_script.content | b64decode"
|
||||
fail_msg: "Healthcheck script template is missing expected content"
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
---
|
||||
- name: Create healthcheck script
|
||||
ansible.builtin.template:
|
||||
src: runner-healthcheck.sh.j2
|
||||
dest: "{{ gitea_runner_healthcheck_script_path }}"
|
||||
owner: "{{ gitea_runner_service_user }}"
|
||||
group: "{{ gitea_runner_service_user }}"
|
||||
mode: "0755"
|
||||
|
||||
- name: Create healthcheck user service file
|
||||
ansible.builtin.template:
|
||||
src: runner-healthcheck.service.j2
|
||||
dest: "{{ gitea_runner_home }}/.config/systemd/user/runner-healthcheck.service"
|
||||
owner: "{{ gitea_runner_service_user }}"
|
||||
group: "{{ gitea_runner_service_user }}"
|
||||
mode: "0644"
|
||||
|
||||
- name: Create healthcheck user timer file
|
||||
ansible.builtin.template:
|
||||
src: runner-healthcheck.timer.j2
|
||||
dest: "{{ gitea_runner_home }}/.config/systemd/user/runner-healthcheck.timer"
|
||||
owner: "{{ gitea_runner_service_user }}"
|
||||
group: "{{ gitea_runner_service_user }}"
|
||||
mode: "0644"
|
||||
|
||||
- name: Reload systemd user daemon for healthcheck timer
|
||||
ansible.builtin.command: systemctl --user daemon-reload
|
||||
become: true
|
||||
become_user: "{{ gitea_runner_service_user }}"
|
||||
environment:
|
||||
XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}"
|
||||
changed_when: true
|
||||
when:
|
||||
- systemd_available.stat.exists
|
||||
- docker_rootless_setup
|
||||
|
||||
- name: Enable and start healthcheck user timer
|
||||
ansible.builtin.command: systemctl --user enable --now runner-healthcheck.timer
|
||||
become: true
|
||||
become_user: "{{ gitea_runner_service_user }}"
|
||||
environment:
|
||||
XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}"
|
||||
changed_when: true
|
||||
when:
|
||||
- systemd_available.stat.exists
|
||||
- docker_rootless_setup
|
||||
@@ -14,6 +14,9 @@
|
||||
- name: Include prune setup
|
||||
ansible.builtin.include_tasks: prune.yml
|
||||
|
||||
- name: Include healthcheck setup
|
||||
ansible.builtin.include_tasks: healthcheck.yml
|
||||
|
||||
- name: Include integration test
|
||||
ansible.builtin.include_tasks: integration_test.yml
|
||||
when: not skip_runner_registration
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
[Unit]
|
||||
Description=Gitea Runner health check (Docker + service + disk)
|
||||
After=docker.service gitea-runner.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
Environment=DOCKER_HOST=unix:///run/user/{{ gitea_runner_uid }}/docker.sock
|
||||
Environment=XDG_RUNTIME_DIR=/run/user/{{ gitea_runner_uid }}
|
||||
ExecStart={{ gitea_runner_healthcheck_script_path }}
|
||||
@@ -0,0 +1,49 @@
|
||||
#!/bin/bash
|
||||
# Health check for gitea-runner: verifies Docker daemon and runner service.
|
||||
# Exits 0 if healthy, 1 if Docker is down (triggers restart), 2 if runner is down.
|
||||
set -euo pipefail
|
||||
|
||||
DOCKER_HOST="unix:///run/user/{{ gitea_runner_uid }}/docker.sock"
|
||||
XDG_RUNTIME_DIR="/run/user/{{ gitea_runner_uid }}"
|
||||
export DOCKER_HOST XDG_RUNTIME_DIR
|
||||
|
||||
# 1. Check Docker daemon responsiveness
|
||||
if ! docker info >/dev/null 2>&1; then
|
||||
echo "ERROR: Docker daemon not responding at ${DOCKER_HOST}"
|
||||
systemctl --user restart docker.service
|
||||
sleep 3
|
||||
if ! docker info >/dev/null 2>&1; then
|
||||
echo "CRITICAL: Docker daemon still down after restart"
|
||||
exit 1
|
||||
fi
|
||||
echo "RECOVERED: Docker daemon restarted successfully"
|
||||
fi
|
||||
|
||||
# 2. Check gitea-runner service is active
|
||||
runner_state=$(systemctl --user is-active gitea-runner.service 2>/dev/null || true)
|
||||
if [[ "$runner_state" != "active" ]]; then
|
||||
echo "ERROR: gitea-runner service is ${runner_state}, restarting"
|
||||
systemctl --user restart gitea-runner.service
|
||||
sleep 2
|
||||
runner_state=$(systemctl --user is-active gitea-runner.service 2>/dev/null || true)
|
||||
if [[ "$runner_state" != "active" ]]; then
|
||||
echo "CRITICAL: gitea-runner service still down after restart"
|
||||
exit 2
|
||||
fi
|
||||
echo "RECOVERED: gitea-runner service restarted successfully"
|
||||
fi
|
||||
|
||||
# 3. Check disk space — prune aggressively if below threshold
|
||||
disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
|
||||
if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_threshold }} ]]; then
|
||||
echo "WARN: Disk usage at ${disk_pct}%, pruning all runner resources"
|
||||
docker system prune -af --filter "label={{ gitea_runner_prune_label }}" --filter "until=1h" || true
|
||||
docker volume prune -af --filter "label={{ gitea_runner_prune_label }}" || true
|
||||
# Also prune dangling images (no label)
|
||||
docker image prune -af || true
|
||||
disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
|
||||
echo "INFO: Disk usage after prune: ${disk_pct}%"
|
||||
fi
|
||||
|
||||
echo "OK: runner healthy, disk at ${disk_pct}%"
|
||||
exit 0
|
||||
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Periodic Gitea Runner health check
|
||||
|
||||
[Timer]
|
||||
OnBootSec={{ gitea_runner_healthcheck_boot_delay }}
|
||||
OnUnitActiveSec={{ gitea_runner_healthcheck_interval }}
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
+6
-6
@@ -8,12 +8,12 @@ Each runner runs in an isolated **rootless Docker** environment under a dedicate
|
||||
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/src/branch/master/LICENSE)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/wiki)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/releases)
|
||||
[](https://www.python.org/downloads/)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/wiki)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/grm/releases)
|
||||
[](https://www.python.org/downloads/)
|
||||
|
||||
## Overview
|
||||
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
"""Gitea Runner Manager — lean CLI for managing Gitea Actions runners."""
|
||||
|
||||
__version__ = "0.12.4"
|
||||
__version__ = "0.12.5"
|
||||
|
||||
@@ -442,6 +442,44 @@ def _collect_become_pass(ask_become_pass: bool) -> str | None:
|
||||
return sys.stdin.readline().strip() or None
|
||||
|
||||
|
||||
@cli.command(name="health", help=_("Run health check on one or all registered runners."))
|
||||
@click.argument("runner_name", required=False)
|
||||
@_runner_options
|
||||
@_handle_errors("Health check failed: {error}")
|
||||
def health(
|
||||
runner_name: str | None,
|
||||
host: str | None,
|
||||
user: str | None,
|
||||
key: str | None,
|
||||
ask_become_pass: bool,
|
||||
) -> None:
|
||||
"""Check Docker, runner service, and disk health on remote hosts."""
|
||||
become_pass = _collect_become_pass(ask_become_pass)
|
||||
manager = RunnerManager()
|
||||
results = manager.health(
|
||||
name=runner_name,
|
||||
host=host,
|
||||
user=user,
|
||||
key=key,
|
||||
ask_become_pass=ask_become_pass,
|
||||
become_pass=become_pass,
|
||||
become_password_file=_get_become_password_file(),
|
||||
verbose=_get_verbose(),
|
||||
)
|
||||
if not results:
|
||||
click.echo(_("No runners registered. Use 'grm install' to add one."))
|
||||
return
|
||||
click.echo(f"{_('NAME'):<18} {_('HOST'):<16} {_('HEALTHY'):<10} {_('MESSAGE')}")
|
||||
click.echo("-" * 80)
|
||||
all_healthy = True
|
||||
for r in results:
|
||||
if r["healthy"] != "yes":
|
||||
all_healthy = False
|
||||
click.echo(f"{r['name']:<18} {r['host']:<16} {r['healthy']:<10} {r['message']}")
|
||||
if not all_healthy:
|
||||
raise click.ClickException(_("One or more runners are unhealthy"))
|
||||
|
||||
|
||||
@cli.command(name="list", help=_("List all registered runners with live status."))
|
||||
@click.option(
|
||||
"--ask-become-pass/--no-ask-become-pass",
|
||||
|
||||
@@ -487,6 +487,68 @@ class RunnerManager:
|
||||
)
|
||||
return result
|
||||
|
||||
def health(
|
||||
self,
|
||||
name: str | None = None,
|
||||
host: str | None = None,
|
||||
user: str | None = None,
|
||||
key: str | None = None,
|
||||
ask_become_pass: bool = False,
|
||||
become_pass: str | None = None,
|
||||
become_password_file: str | None = None,
|
||||
verbose: bool = False,
|
||||
) -> list[dict[str, str]]:
|
||||
"""Run health check on one or all registered runners.
|
||||
|
||||
When *name* is provided, checks only that runner. Otherwise,
|
||||
checks all registered runners. Returns a list of dicts with
|
||||
``name``, ``host``, ``healthy`` (``"yes"``/``"no"``), and
|
||||
``message`` keys.
|
||||
"""
|
||||
if name:
|
||||
actual_host, actual_user, actual_key, _gitea_url = self._resolve_runner(name, host, user, key)
|
||||
entries = [(name, actual_host, actual_user, actual_key)]
|
||||
else:
|
||||
entries = [(n, info["host"], info["user"], info.get("key")) for n, info in self._registry.list().items()]
|
||||
|
||||
results: list[dict[str, str]] = []
|
||||
for runner_name, r_host, r_user, r_key in entries:
|
||||
say(_("Checking health of {name} on {host}", name=runner_name, host=r_host))
|
||||
healthy = "no"
|
||||
message = "unknown"
|
||||
try:
|
||||
stdout = self._executor.run_ad_hoc(
|
||||
r_host,
|
||||
r_user,
|
||||
r_key,
|
||||
"shell",
|
||||
f"sudo -u grm-{runner_name} "
|
||||
f"XDG_RUNTIME_DIR=/run/user/$(id -u grm-{runner_name}) "
|
||||
f"systemctl --user start runner-healthcheck.service && "
|
||||
f"journalctl --user -u runner-healthcheck.service --no-pager -n 1",
|
||||
become=True,
|
||||
ask_become_pass=ask_become_pass or become_password_file is not None,
|
||||
check=False,
|
||||
become_pass=become_pass,
|
||||
)
|
||||
if "OK:" in stdout:
|
||||
healthy = "yes"
|
||||
# Extract the OK line
|
||||
for line in stdout.splitlines():
|
||||
if "OK:" in line:
|
||||
message = line.split("OK:", 1)[1].strip()
|
||||
break
|
||||
else:
|
||||
for line in stdout.splitlines():
|
||||
stripped = line.strip()
|
||||
if stripped and "CHANGED" not in stripped and "WARNING" not in stripped:
|
||||
message = stripped
|
||||
break
|
||||
except AnsibleError as e:
|
||||
message = str(e)
|
||||
results.append({"name": runner_name, "host": r_host, "healthy": healthy, "message": message})
|
||||
return results
|
||||
|
||||
@staticmethod
|
||||
def _parse_status(stdout: str) -> str:
|
||||
ansible_noise = (" | CHANGED | ", " | FAILED | ", " | UNREACHABLE | ", "[WARNING]", "ssh:", ">>")
|
||||
|
||||
@@ -31,6 +31,14 @@
|
||||
"ru": "Проверить состояние зарегистрированного Gitea Runner.",
|
||||
"zh": "检查已注册的 Gitea Runner 状态。"
|
||||
},
|
||||
"Checking health of {name} on {host}": {
|
||||
"bg": "Проверка на здравословното състояние на {name} на {host}",
|
||||
"de": "Gesundheitsprüfung von {name} auf {host}",
|
||||
"en": "Checking health of {name} on {host}",
|
||||
"pl": "Sprawdzanie zdrowia {name} na {host}",
|
||||
"ru": "Проверка здоровья {name} на {host}",
|
||||
"zh": "正在检查 {host} 上 {name} 的健康状态"
|
||||
},
|
||||
"Checking status of Gitea Runner {name} on {host}": {
|
||||
"bg": "Проверка на състоянието на Gitea Runner {name} на {host}",
|
||||
"de": "Prüfe Status von Gitea Runner {name} auf {host}",
|
||||
@@ -175,6 +183,22 @@
|
||||
"ru": "Токен админ API Gitea для интеграционного теста (env: CI_GITEA_TOKEN)",
|
||||
"zh": "Gitea 管理员 API 令牌，用于集成测试（环境变量: CI_GITEA_TOKEN）"
|
||||
},
|
||||
"HEALTHY": {
|
||||
"bg": "ЗДРАВ",
|
||||
"de": "GESUND",
|
||||
"en": "HEALTHY",
|
||||
"pl": "ZDROWY",
|
||||
"ru": "ЗДОРОВ",
|
||||
"zh": "健康"
|
||||
},
|
||||
"Health check failed: {error}": {
|
||||
"bg": "Проверката на здравословното състояние неуспешна: {error}",
|
||||
"de": "Gesundheitsprüfung fehlgeschlagen: {error}",
|
||||
"en": "Health check failed: {error}",
|
||||
"pl": "Sprawdzanie zdrowia nie powiodło się: {error}",
|
||||
"ru": "Проверка здоровья не удалась: {error}",
|
||||
"zh": "健康检查失败: {error}"
|
||||
},
|
||||
"HOST": {
|
||||
"bg": "ХОСТ",
|
||||
"de": "HOST",
|
||||
@@ -239,6 +263,14 @@
|
||||
"ru": "Ошибка списка: {error}",
|
||||
"zh": "列表失败: {error}"
|
||||
},
|
||||
"MESSAGE": {
|
||||
"bg": "СЪОБЩЕНИЕ",
|
||||
"de": "MELDUNG",
|
||||
"en": "MESSAGE",
|
||||
"pl": "WIADOMOŚĆ",
|
||||
"ru": "СООБЩЕНИЕ",
|
||||
"zh": "消息"
|
||||
},
|
||||
"NAME": {
|
||||
"bg": "ИМЕ",
|
||||
"de": "NAME",
|
||||
@@ -263,6 +295,14 @@
|
||||
"ru": "Нет зарегистрированных runners. Используйте 'grm install' чтобы добавить.",
|
||||
"zh": "没有已注册的 runners。使用 'grm install' 添加一个。"
|
||||
},
|
||||
"One or more runners are unhealthy": {
|
||||
"bg": "Един или повече runners са нездравословни",
|
||||
"de": "Ein oder mehrere Runner sind fehlerhaft",
|
||||
"en": "One or more runners are unhealthy",
|
||||
"pl": "Jeden lub więcej runnerów jest w złym stanie",
|
||||
"ru": "Один или несколько runners нездоровы",
|
||||
"zh": "一个或多个 runners 不健康"
|
||||
},
|
||||
"Override SSH key from registry": {
|
||||
"bg": "Замяна на SSH ключа от регистъра",
|
||||
"de": "SSH-Schlüssel aus Registrierung überschreiben",
|
||||
@@ -423,6 +463,14 @@
|
||||
"ru": "Выполнение Ansible playbook",
|
||||
"zh": "正在运行 Ansible playbook"
|
||||
},
|
||||
"Run health check on one or all registered runners.": {
|
||||
"bg": "Проверка на здравословното състояние на един или всички регистрирани runners.",
|
||||
"de": "Gesundheitsprüfung für einen oder alle registrierten Runner ausführen.",
|
||||
"en": "Run health check on one or all registered runners.",
|
||||
"pl": "Uruchom sprawdzanie zdrowia jednego lub wszystkich zarejestrowanych runnerów.",
|
||||
"ru": "Проверить здоровье одного или всех зарегистрированных runners.",
|
||||
"zh": "对一个或所有已注册 runners 运行健康检查。"
|
||||
},
|
||||
"SSH user (env: GITEA_RUNNER_USER)": {
|
||||
"bg": "SSH потребител (env: GITEA_RUNNER_USER)",
|
||||
"de": "SSH-Benutzer (env: GITEA_RUNNER_USER)",
|
||||
|
||||
@@ -852,6 +852,75 @@ class TestCLI:
|
||||
assert result.exit_code != 0
|
||||
assert "fail" in result.output
|
||||
|
||||
@patch("gitea_runner_manager.cli.RunnerManager")
|
||||
def test_health_all_healthy(self, mock_manager_class: MagicMock) -> None:
|
||||
mock_manager = MagicMock()
|
||||
mock_manager.health.return_value = [
|
||||
{"name": "r1", "host": "10.0.0.1", "healthy": "yes", "message": "runner healthy, disk at 42%"},
|
||||
{"name": "r2", "host": "10.0.0.2", "healthy": "yes", "message": "runner healthy, disk at 50%"},
|
||||
]
|
||||
mock_manager_class.return_value = mock_manager
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["health"], input="secret\n")
|
||||
assert result.exit_code == 0
|
||||
assert "r1" in result.output
|
||||
assert "r2" in result.output
|
||||
assert "yes" in result.output
|
||||
|
||||
@patch("gitea_runner_manager.cli.RunnerManager")
|
||||
def test_health_with_unhealthy(self, mock_manager_class: MagicMock) -> None:
|
||||
mock_manager = MagicMock()
|
||||
mock_manager.health.return_value = [
|
||||
{"name": "r1", "host": "10.0.0.1", "healthy": "yes", "message": "runner healthy, disk at 42%"},
|
||||
{"name": "r2", "host": "10.0.0.2", "healthy": "no", "message": "Docker daemon down"},
|
||||
]
|
||||
mock_manager_class.return_value = mock_manager
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["health"], input="secret\n")
|
||||
assert result.exit_code != 0
|
||||
assert "unhealthy" in result.output.lower()
|
||||
assert "r2" in result.output
|
||||
|
||||
@patch("gitea_runner_manager.cli.RunnerManager")
|
||||
def test_health_single_runner(self, mock_manager_class: MagicMock) -> None:
|
||||
mock_manager = MagicMock()
|
||||
mock_manager.health.return_value = [
|
||||
{"name": "r1", "host": "10.0.0.1", "healthy": "yes", "message": "runner healthy, disk at 42%"},
|
||||
]
|
||||
mock_manager_class.return_value = mock_manager
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["health", "r1"], input="secret\n")
|
||||
assert result.exit_code == 0
|
||||
assert "r1" in result.output
|
||||
mock_manager.health.assert_called_once()
|
||||
|
||||
@patch("gitea_runner_manager.cli.RunnerManager")
|
||||
def test_health_empty(self, mock_manager_class: MagicMock) -> None:
|
||||
mock_manager = MagicMock()
|
||||
mock_manager.health.return_value = []
|
||||
mock_manager_class.return_value = mock_manager
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["health"], input="secret\n")
|
||||
assert result.exit_code == 0
|
||||
assert "No runners registered" in result.output
|
||||
|
||||
@patch("gitea_runner_manager.cli.RunnerManager")
|
||||
def test_health_error(self, mock_manager_class: MagicMock) -> None:
|
||||
mock_manager = MagicMock()
|
||||
from gitea_runner_manager.exceptions import AnsibleError
|
||||
|
||||
mock_manager.health.side_effect = AnsibleError("fail")
|
||||
mock_manager_class.return_value = mock_manager
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["health"], input="secret\n")
|
||||
assert result.exit_code != 0
|
||||
assert "fail" in result.output
|
||||
|
||||
@patch("gitea_runner_manager.cli.os.getlogin", side_effect=OSError("no tty"))
|
||||
@patch("gitea_runner_manager.cli.RunnerManager")
|
||||
def test_default_user_fallback_on_getlogin_error(
|
||||
|
||||
@@ -625,6 +625,112 @@ class TestRunnerManager:
|
||||
assert manager.list_runners(no_status=True) == []
|
||||
|
||||
|
||||
class TestHealth:
|
||||
"""Tests for the ``health`` method."""
|
||||
|
||||
def test_health_single_runner_healthy(self) -> None:
|
||||
mock_registry = MagicMock()
|
||||
mock_registry.get.return_value = {"host": "10.0.0.1", "user": "ubuntu", "key": "/key"}
|
||||
manager = RunnerManager(registry=mock_registry)
|
||||
mock_executor = MagicMock()
|
||||
mock_executor.run_ad_hoc.return_value = "OK: runner healthy, disk at 42%"
|
||||
manager._executor = mock_executor
|
||||
|
||||
results = manager.health(name="r1")
|
||||
assert len(results) == 1
|
||||
assert results[0]["name"] == "r1"
|
||||
assert results[0]["healthy"] == "yes"
|
||||
assert "runner healthy" in results[0]["message"]
|
||||
mock_executor.run_ad_hoc.assert_called_once()
|
||||
|
||||
def test_health_single_runner_unhealthy(self) -> None:
|
||||
mock_registry = MagicMock()
|
||||
mock_registry.get.return_value = {"host": "10.0.0.1", "user": "ubuntu", "key": None}
|
||||
manager = RunnerManager(registry=mock_registry)
|
||||
mock_executor = MagicMock()
|
||||
mock_executor.run_ad_hoc.return_value = "CRITICAL: Docker daemon still down after restart"
|
||||
manager._executor = mock_executor
|
||||
|
||||
results = manager.health(name="r1")
|
||||
assert len(results) == 1
|
||||
assert results[0]["healthy"] == "no"
|
||||
assert "Docker daemon still down" in results[0]["message"]
|
||||
|
||||
def test_health_all_runners(self) -> None:
|
||||
mock_registry = MagicMock()
|
||||
mock_registry.list.return_value = {
|
||||
"r1": {"host": "10.0.0.1", "user": "ubuntu", "key": None},
|
||||
"r2": {"host": "10.0.0.2", "user": "ubuntu", "key": None},
|
||||
}
|
||||
manager = RunnerManager(registry=mock_registry)
|
||||
mock_executor = MagicMock()
|
||||
mock_executor.run_ad_hoc.side_effect = [
|
||||
"OK: runner healthy, disk at 42%",
|
||||
"ERROR: gitea-runner service is inactive, restarting",
|
||||
]
|
||||
manager._executor = mock_executor
|
||||
|
||||
results = manager.health()
|
||||
assert len(results) == 2
|
||||
assert results[0]["healthy"] == "yes"
|
||||
assert results[1]["healthy"] == "no"
|
||||
|
||||
def test_health_empty_registry(self) -> None:
|
||||
mock_registry = MagicMock()
|
||||
mock_registry.list.return_value = {}
|
||||
manager = RunnerManager(registry=mock_registry)
|
||||
results = manager.health()
|
||||
assert results == []
|
||||
|
||||
def test_health_ansible_error(self) -> None:
|
||||
mock_registry = MagicMock()
|
||||
mock_registry.get.return_value = {"host": "10.0.0.1", "user": "ubuntu", "key": None}
|
||||
manager = RunnerManager(registry=mock_registry)
|
||||
mock_executor = MagicMock()
|
||||
mock_executor.run_ad_hoc.side_effect = AnsibleError("ssh unreachable")
|
||||
manager._executor = mock_executor
|
||||
|
||||
results = manager.health(name="r1")
|
||||
assert len(results) == 1
|
||||
assert results[0]["healthy"] == "no"
|
||||
assert "ssh unreachable" in results[0]["message"]
|
||||
|
||||
def test_health_with_host_override(self) -> None:
|
||||
mock_registry = MagicMock()
|
||||
mock_registry.get.return_value = {"host": "10.0.0.1", "user": "ubuntu", "key": None}
|
||||
manager = RunnerManager(registry=mock_registry)
|
||||
mock_executor = MagicMock()
|
||||
mock_executor.run_ad_hoc.return_value = "OK: runner healthy, disk at 50%"
|
||||
manager._executor = mock_executor
|
||||
|
||||
results = manager.health(name="r1", host="10.0.0.99", user="root")
|
||||
assert len(results) == 1
|
||||
assert results[0]["host"] == "10.0.0.99"
|
||||
call_args = mock_executor.run_ad_hoc.call_args.args
|
||||
assert call_args[0] == "10.0.0.99"
|
||||
assert call_args[1] == "root"
|
||||
|
||||
def test_health_runner_not_found(self) -> None:
|
||||
mock_registry = MagicMock()
|
||||
mock_registry.get.return_value = None
|
||||
manager = RunnerManager(registry=mock_registry)
|
||||
with pytest.raises(AnsibleError, match="not found in registry"):
|
||||
manager.health(name="nonexistent")
|
||||
|
||||
def test_health_passes_become_pass(self) -> None:
|
||||
"""become_pass is forwarded to run_ad_hoc for sudo authentication."""
|
||||
mock_registry = MagicMock()
|
||||
mock_registry.get.return_value = {"host": "10.0.0.1", "user": "ubuntu", "key": None}
|
||||
manager = RunnerManager(registry=mock_registry)
|
||||
mock_executor = MagicMock()
|
||||
mock_executor.run_ad_hoc.return_value = "OK: runner healthy, disk at 42%"
|
||||
manager._executor = mock_executor
|
||||
|
||||
manager.health(name="r1", become_pass="s3cr3t")
|
||||
call_kwargs = mock_executor.run_ad_hoc.call_args.kwargs
|
||||
assert call_kwargs["become_pass"] == "s3cr3t"
|
||||
|
||||
|
||||
class TestExtraVarsFile:
|
||||
"""Tests for the ``_extra_vars_file`` context manager."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user