From 63ef5cdbcfefd9a517280890c8fbb988e90f7a15 Mon Sep 17 00:00:00 2001 From: emil Date: Tue, 30 Jun 2026 23:28:54 +0000 Subject: [PATCH] GRM-123: fix: cast disk threshold to string in template-content verify assertion --- ansible/disable-runner.yml | 10 ++ ansible/remove-runner.yml | 28 +++++ ansible/roles/gitea-runner/defaults/main.yml | 6 + .../gitea-runner/molecule/default/verify.yml | 39 +++++++ .../gitea-runner/molecule/remove/verify.yml | 33 ++++++ .../molecule/template-content/verify.yml | 42 +++++++ .../roles/gitea-runner/tasks/healthcheck.yml | 46 ++++++++ ansible/roles/gitea-runner/tasks/main.yml | 3 + .../templates/runner-healthcheck.service.j2 | 9 ++ .../templates/runner-healthcheck.sh.j2 | 49 ++++++++ .../templates/runner-healthcheck.timer.j2 | 10 ++ src/gitea_runner_manager/cli.py | 38 +++++++ src/gitea_runner_manager/runner_manager.py | 62 ++++++++++ src/gitea_runner_manager/translations.json | 48 ++++++++ tests/unit/test_cli.py | 69 ++++++++++++ tests/unit/test_runner_manager.py | 106 ++++++++++++++++++ 16 files changed, 598 insertions(+) create mode 100644 ansible/roles/gitea-runner/tasks/healthcheck.yml create mode 100644 ansible/roles/gitea-runner/templates/runner-healthcheck.service.j2 create mode 100644 ansible/roles/gitea-runner/templates/runner-healthcheck.sh.j2 create mode 100644 ansible/roles/gitea-runner/templates/runner-healthcheck.timer.j2 diff --git a/ansible/disable-runner.yml b/ansible/disable-runner.yml index 8dbda60..b89deb3 100644 --- a/ansible/disable-runner.yml +++ b/ansible/disable-runner.yml @@ -18,6 +18,16 @@ when: systemd_available.stat.exists changed_when: true + - name: Stop and disable healthcheck timer + ansible.builtin.command: systemctl --user stop --disable runner-healthcheck.timer + become: true + become_user: "{{ gitea_runner_service_user | default('grm-' ~ runner_name) }}" + environment: + XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid | default('') }}" + when: systemd_available.stat.exists + changed_when: true + failed_when: false + - name: Include deregistration ansible.builtin.include_role: name: gitea-runner diff --git a/ansible/remove-runner.yml b/ansible/remove-runner.yml index 867c45f..555dcea 100644 --- a/ansible/remove-runner.yml +++ b/ansible/remove-runner.yml @@ -79,6 +79,16 @@ tasks_from: deregister.yml when: not skip_runner_registration | default(false) + - name: Stop and disable healthcheck timer + ansible.builtin.command: systemctl --user stop --disable runner-healthcheck.timer + become: true + become_user: "{{ gitea_runner_service_user | default('grm-' ~ runner_name) }}" + environment: + XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid | default('') }}" + when: systemd_available.stat.exists + changed_when: true + failed_when: false + - name: Remove docker-prune user service file ansible.builtin.file: path: "{{ gitea_runner_home | default('/home/grm-' ~ runner_name) }}/.config/systemd/user/docker-prune.service" @@ -91,6 +101,24 @@ state: absent failed_when: false + - name: Remove healthcheck user service file + ansible.builtin.file: + path: "{{ gitea_runner_home | default('/home/grm-' ~ runner_name) }}/.config/systemd/user/runner-healthcheck.service" + state: absent + failed_when: false + + - name: Remove healthcheck user timer file + ansible.builtin.file: + path: "{{ gitea_runner_home | default('/home/grm-' ~ runner_name) }}/.config/systemd/user/runner-healthcheck.timer" + state: absent + failed_when: false + + - name: Remove healthcheck script + ansible.builtin.file: + path: "{{ gitea_runner_config_dir | default('/etc/gitea-runner/' ~ runner_name) }}/healthcheck.sh" + state: absent + failed_when: false + - name: Remove systemd user unit file ansible.builtin.file: path: "{{ gitea_runner_home | default('/home/grm-' ~ runner_name) }}/.config/systemd/user/gitea-runner.service" diff --git a/ansible/roles/gitea-runner/defaults/main.yml b/ansible/roles/gitea-runner/defaults/main.yml index 9b35ef6..367ecce 100644 --- a/ansible/roles/gitea-runner/defaults/main.yml +++ b/ansible/roles/gitea-runner/defaults/main.yml @@ -24,6 +24,12 @@ gitea_runner_prune_label: "gitea-runner=true" # Service configuration gitea_runner_service_restart_sec: "5" +# Health check configuration +gitea_runner_healthcheck_interval: "5min" +gitea_runner_healthcheck_boot_delay: "2min" +gitea_runner_healthcheck_disk_threshold: 85 +gitea_runner_healthcheck_script_path: "{{ gitea_runner_config_dir }}/healthcheck.sh" + # Admin token for runner deregistration via Gitea API. # If not set, falls back to registration_token (which likely lacks admin scope). # Set this to a token with admin scope to enable automatic runner cleanup on removal. diff --git a/ansible/roles/gitea-runner/molecule/default/verify.yml b/ansible/roles/gitea-runner/molecule/default/verify.yml index 8bae1f6..037c154 100644 --- a/ansible/roles/gitea-runner/molecule/default/verify.yml +++ b/ansible/roles/gitea-runner/molecule/default/verify.yml @@ -75,3 +75,42 @@ that: - timer_stat.stat.exists fail_msg: "Docker prune timer is missing" + + - name: Check healthcheck script exists + ansible.builtin.stat: + path: "{{ gitea_runner_healthcheck_script_path }}" + register: healthcheck_script_stat + + - name: Assert healthcheck script exists + ansible.builtin.assert: + that: + - healthcheck_script_stat.stat.exists + fail_msg: "Healthcheck script is missing" + + - name: Assert healthcheck script is executable + ansible.builtin.assert: + that: + - healthcheck_script_stat.stat.mode == "0755" + fail_msg: "Healthcheck script is not executable" + + - name: Check healthcheck service exists + ansible.builtin.stat: + path: "{{ gitea_runner_home }}/.config/systemd/user/runner-healthcheck.service" + register: healthcheck_service_stat + + - name: Assert healthcheck service exists + ansible.builtin.assert: + that: + - healthcheck_service_stat.stat.exists + fail_msg: "Healthcheck systemd service is missing" + + - name: Check healthcheck timer exists + ansible.builtin.stat: + path: "{{ gitea_runner_home }}/.config/systemd/user/runner-healthcheck.timer" + register: healthcheck_timer_stat + + - name: Assert healthcheck timer exists + ansible.builtin.assert: + that: + - healthcheck_timer_stat.stat.exists + fail_msg: "Healthcheck systemd timer is missing" diff --git a/ansible/roles/gitea-runner/molecule/remove/verify.yml b/ansible/roles/gitea-runner/molecule/remove/verify.yml index 9c52bfa..08a81cb 100644 --- a/ansible/roles/gitea-runner/molecule/remove/verify.yml +++ b/ansible/roles/gitea-runner/molecule/remove/verify.yml @@ -89,6 +89,39 @@ - not prune_timer_stat.stat.exists fail_msg: "docker-prune timer unit still exists after removal" + - name: Check healthcheck service unit is absent + ansible.builtin.stat: + path: "{{ gitea_runner_home }}/.config/systemd/user/runner-healthcheck.service" + register: healthcheck_service_stat + + - name: Assert healthcheck service unit is absent + ansible.builtin.assert: + that: + - not healthcheck_service_stat.stat.exists + fail_msg: "runner-healthcheck service unit still exists after removal" + + - name: Check healthcheck timer unit is absent + ansible.builtin.stat: + path: "{{ gitea_runner_home }}/.config/systemd/user/runner-healthcheck.timer" + register: healthcheck_timer_stat + + - name: Assert healthcheck timer unit is absent + ansible.builtin.assert: + that: + - not healthcheck_timer_stat.stat.exists + fail_msg: "runner-healthcheck timer unit still exists after removal" + + - name: Check healthcheck script is absent + ansible.builtin.stat: + path: "{{ gitea_runner_config_dir }}/healthcheck.sh" + register: healthcheck_script_stat + + - name: Assert healthcheck script is absent + ansible.builtin.assert: + that: + - not healthcheck_script_stat.stat.exists + fail_msg: "healthcheck script still exists after removal" + - name: Check subuid entry is absent ansible.builtin.command: "grep -c '^{{ gitea_runner_service_user }}:' /etc/subuid" register: subuid_check diff --git a/ansible/roles/gitea-runner/molecule/template-content/verify.yml b/ansible/roles/gitea-runner/molecule/template-content/verify.yml index 5b9f450..7018150 100644 --- a/ansible/roles/gitea-runner/molecule/template-content/verify.yml +++ b/ansible/roles/gitea-runner/molecule/template-content/verify.yml @@ -62,3 +62,45 @@ - "'OnCalendar={{ gitea_runner_prune_schedule }}' in prune_timer.content | b64decode" - "'Persistent=true' in prune_timer.content | b64decode" fail_msg: "Prune timer template is missing expected directives" + + - name: Read rendered healthcheck service template + ansible.builtin.slurp: + src: "{{ gitea_runner_home }}/.config/systemd/user/runner-healthcheck.service" + register: healthcheck_service + + - name: Assert healthcheck service contains expected directives + ansible.builtin.assert: + that: + - "'Type=oneshot' in healthcheck_service.content | b64decode" + - "'ExecStart={{ gitea_runner_healthcheck_script_path }}' in healthcheck_service.content | b64decode" + - "'DOCKER_HOST=unix:///run/user/' in healthcheck_service.content | b64decode" + - "'XDG_RUNTIME_DIR=/run/user/' in healthcheck_service.content | b64decode" + fail_msg: "Healthcheck service template is missing expected directives" + + - name: Read rendered healthcheck timer template + ansible.builtin.slurp: + src: "{{ gitea_runner_home }}/.config/systemd/user/runner-healthcheck.timer" + register: healthcheck_timer + + - name: Assert healthcheck timer contains expected directives + ansible.builtin.assert: + that: + - "'OnBootSec={{ gitea_runner_healthcheck_boot_delay }}' in healthcheck_timer.content | b64decode" + - "'OnUnitActiveSec={{ gitea_runner_healthcheck_interval }}' in healthcheck_timer.content | b64decode" + - "'Persistent=true' in healthcheck_timer.content | b64decode" + fail_msg: "Healthcheck timer template is missing expected directives" + + - name: Read rendered healthcheck script + ansible.builtin.slurp: + src: "{{ gitea_runner_healthcheck_script_path }}" + register: healthcheck_script + + - name: Assert healthcheck script contains expected content + ansible.builtin.assert: + that: + - "'docker info' in healthcheck_script.content | b64decode" + - "'systemctl --user restart docker.service' in healthcheck_script.content | b64decode" + - "'systemctl --user restart gitea-runner.service' in healthcheck_script.content | b64decode" + - "'docker system prune' in healthcheck_script.content | b64decode" + - "gitea_runner_healthcheck_disk_threshold | string in healthcheck_script.content | b64decode" + fail_msg: "Healthcheck script template is missing expected content" diff --git a/ansible/roles/gitea-runner/tasks/healthcheck.yml b/ansible/roles/gitea-runner/tasks/healthcheck.yml new file mode 100644 index 0000000..de5e2de --- /dev/null +++ b/ansible/roles/gitea-runner/tasks/healthcheck.yml @@ -0,0 +1,46 @@ +--- +- name: Create healthcheck script + ansible.builtin.template: + src: runner-healthcheck.sh.j2 + dest: "{{ gitea_runner_healthcheck_script_path }}" + owner: "{{ gitea_runner_service_user }}" + group: "{{ gitea_runner_service_user }}" + mode: "0755" + +- name: Create healthcheck user service file + ansible.builtin.template: + src: runner-healthcheck.service.j2 + dest: "{{ gitea_runner_home }}/.config/systemd/user/runner-healthcheck.service" + owner: "{{ gitea_runner_service_user }}" + group: "{{ gitea_runner_service_user }}" + mode: "0644" + +- name: Create healthcheck user timer file + ansible.builtin.template: + src: runner-healthcheck.timer.j2 + dest: "{{ gitea_runner_home }}/.config/systemd/user/runner-healthcheck.timer" + owner: "{{ gitea_runner_service_user }}" + group: "{{ gitea_runner_service_user }}" + mode: "0644" + +- name: Reload systemd user daemon for healthcheck timer + ansible.builtin.command: systemctl --user daemon-reload + become: true + become_user: "{{ gitea_runner_service_user }}" + environment: + XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}" + changed_when: true + when: + - systemd_available.stat.exists + - docker_rootless_setup + +- name: Enable and start healthcheck user timer + ansible.builtin.command: systemctl --user enable --now runner-healthcheck.timer + become: true + become_user: "{{ gitea_runner_service_user }}" + environment: + XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}" + changed_when: true + when: + - systemd_available.stat.exists + - docker_rootless_setup diff --git a/ansible/roles/gitea-runner/tasks/main.yml b/ansible/roles/gitea-runner/tasks/main.yml index 9bff087..d984314 100644 --- a/ansible/roles/gitea-runner/tasks/main.yml +++ b/ansible/roles/gitea-runner/tasks/main.yml @@ -14,6 +14,9 @@ - name: Include prune setup ansible.builtin.include_tasks: prune.yml +- name: Include healthcheck setup + ansible.builtin.include_tasks: healthcheck.yml + - name: Include integration test ansible.builtin.include_tasks: integration_test.yml when: not skip_runner_registration diff --git a/ansible/roles/gitea-runner/templates/runner-healthcheck.service.j2 b/ansible/roles/gitea-runner/templates/runner-healthcheck.service.j2 new file mode 100644 index 0000000..d8f7731 --- /dev/null +++ b/ansible/roles/gitea-runner/templates/runner-healthcheck.service.j2 @@ -0,0 +1,9 @@ +[Unit] +Description=Gitea Runner health check (Docker + service + disk) +After=docker.service gitea-runner.service + +[Service] +Type=oneshot +Environment=DOCKER_HOST=unix:///run/user/{{ gitea_runner_uid }}/docker.sock +Environment=XDG_RUNTIME_DIR=/run/user/{{ gitea_runner_uid }} +ExecStart={{ gitea_runner_healthcheck_script_path }} diff --git a/ansible/roles/gitea-runner/templates/runner-healthcheck.sh.j2 b/ansible/roles/gitea-runner/templates/runner-healthcheck.sh.j2 new file mode 100644 index 0000000..da2a3f4 --- /dev/null +++ b/ansible/roles/gitea-runner/templates/runner-healthcheck.sh.j2 @@ -0,0 +1,49 @@ +#!/bin/bash +# Health check for gitea-runner: verifies Docker daemon and runner service. +# Exits 0 if healthy, 1 if Docker is down (triggers restart), 2 if runner is down. +set -euo pipefail + +DOCKER_HOST="unix:///run/user/{{ gitea_runner_uid }}/docker.sock" +XDG_RUNTIME_DIR="/run/user/{{ gitea_runner_uid }}" +export DOCKER_HOST XDG_RUNTIME_DIR + +# 1. Check Docker daemon responsiveness +if ! docker info >/dev/null 2>&1; then + echo "ERROR: Docker daemon not responding at ${DOCKER_HOST}" + systemctl --user restart docker.service + sleep 3 + if ! docker info >/dev/null 2>&1; then + echo "CRITICAL: Docker daemon still down after restart" + exit 1 + fi + echo "RECOVERED: Docker daemon restarted successfully" +fi + +# 2. Check gitea-runner service is active +runner_state=$(systemctl --user is-active gitea-runner.service 2>/dev/null || true) +if [[ "$runner_state" != "active" ]]; then + echo "ERROR: gitea-runner service is ${runner_state}, restarting" + systemctl --user restart gitea-runner.service + sleep 2 + runner_state=$(systemctl --user is-active gitea-runner.service 2>/dev/null || true) + if [[ "$runner_state" != "active" ]]; then + echo "CRITICAL: gitea-runner service still down after restart" + exit 2 + fi + echo "RECOVERED: gitea-runner service restarted successfully" +fi + +# 3. Check disk space — prune aggressively if below threshold +disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}') +if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_threshold }} ]]; then + echo "WARN: Disk usage at ${disk_pct}%, pruning all runner resources" + docker system prune -af --filter "label={{ gitea_runner_prune_label }}" --filter "until=1h" || true + docker volume prune -af --filter "label={{ gitea_runner_prune_label }}" || true + # Also prune dangling images (no label) + docker image prune -af || true + disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}') + echo "INFO: Disk usage after prune: ${disk_pct}%" +fi + +echo "OK: runner healthy, disk at ${disk_pct}%" +exit 0 diff --git a/ansible/roles/gitea-runner/templates/runner-healthcheck.timer.j2 b/ansible/roles/gitea-runner/templates/runner-healthcheck.timer.j2 new file mode 100644 index 0000000..689c576 --- /dev/null +++ b/ansible/roles/gitea-runner/templates/runner-healthcheck.timer.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Periodic Gitea Runner health check + +[Timer] +OnBootSec={{ gitea_runner_healthcheck_boot_delay }} +OnUnitActiveSec={{ gitea_runner_healthcheck_interval }} +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/src/gitea_runner_manager/cli.py b/src/gitea_runner_manager/cli.py index 46fb293..b7ed86a 100644 --- a/src/gitea_runner_manager/cli.py +++ b/src/gitea_runner_manager/cli.py @@ -442,6 +442,44 @@ def _collect_become_pass(ask_become_pass: bool) -> str | None: return sys.stdin.readline().strip() or None +@cli.command(name="health", help=_("Run health check on one or all registered runners.")) +@click.argument("runner_name", required=False) +@_runner_options +@_handle_errors("Health check failed: {error}") +def health( + runner_name: str | None, + host: str | None, + user: str | None, + key: str | None, + ask_become_pass: bool, +) -> None: + """Check Docker, runner service, and disk health on remote hosts.""" + become_pass = _collect_become_pass(ask_become_pass) + manager = RunnerManager() + results = manager.health( + name=runner_name, + host=host, + user=user, + key=key, + ask_become_pass=ask_become_pass, + become_pass=become_pass, + become_password_file=_get_become_password_file(), + verbose=_get_verbose(), + ) + if not results: + click.echo(_("No runners registered. Use 'grm install' to add one.")) + return + click.echo(f"{_('NAME'):<18} {_('HOST'):<16} {_('HEALTHY'):<10} {_('MESSAGE')}") + click.echo("-" * 80) + all_healthy = True + for r in results: + if r["healthy"] != "yes": + all_healthy = False + click.echo(f"{r['name']:<18} {r['host']:<16} {r['healthy']:<10} {r['message']}") + if not all_healthy: + raise click.ClickException(_("One or more runners are unhealthy")) + + @cli.command(name="list", help=_("List all registered runners with live status.")) @click.option( "--ask-become-pass/--no-ask-become-pass", diff --git a/src/gitea_runner_manager/runner_manager.py b/src/gitea_runner_manager/runner_manager.py index 8998d1d..59b8d49 100644 --- a/src/gitea_runner_manager/runner_manager.py +++ b/src/gitea_runner_manager/runner_manager.py @@ -487,6 +487,68 @@ class RunnerManager: ) return result + def health( + self, + name: str | None = None, + host: str | None = None, + user: str | None = None, + key: str | None = None, + ask_become_pass: bool = False, + become_pass: str | None = None, + become_password_file: str | None = None, + verbose: bool = False, + ) -> list[dict[str, str]]: + """Run health check on one or all registered runners. + + When *name* is provided, checks only that runner. Otherwise, + checks all registered runners. Returns a list of dicts with + ``name``, ``host``, ``healthy`` (``"yes"``/``"no"``), and + ``message`` keys. + """ + if name: + actual_host, actual_user, actual_key, _gitea_url = self._resolve_runner(name, host, user, key) + entries = [(name, actual_host, actual_user, actual_key)] + else: + entries = [(n, info["host"], info["user"], info.get("key")) for n, info in self._registry.list().items()] + + results: list[dict[str, str]] = [] + for runner_name, r_host, r_user, r_key in entries: + say(_("Checking health of {name} on {host}", name=runner_name, host=r_host)) + healthy = "no" + message = "unknown" + try: + stdout = self._executor.run_ad_hoc( + r_host, + r_user, + r_key, + "shell", + f"sudo -u grm-{runner_name} " + f"XDG_RUNTIME_DIR=/run/user/$(id -u grm-{runner_name}) " + f"systemctl --user start runner-healthcheck.service && " + f"journalctl --user -u runner-healthcheck.service --no-pager -n 1", + become=True, + ask_become_pass=ask_become_pass or become_password_file is not None, + check=False, + become_pass=become_pass, + ) + if "OK:" in stdout: + healthy = "yes" + # Extract the OK line + for line in stdout.splitlines(): + if "OK:" in line: + message = line.split("OK:", 1)[1].strip() + break + else: + for line in stdout.splitlines(): + stripped = line.strip() + if stripped and "CHANGED" not in stripped and "WARNING" not in stripped: + message = stripped + break + except AnsibleError as e: + message = str(e) + results.append({"name": runner_name, "host": r_host, "healthy": healthy, "message": message}) + return results + @staticmethod def _parse_status(stdout: str) -> str: ansible_noise = (" | CHANGED | ", " | FAILED | ", " | UNREACHABLE | ", "[WARNING]", "ssh:", ">>") diff --git a/src/gitea_runner_manager/translations.json b/src/gitea_runner_manager/translations.json index 8b63c21..f842e92 100644 --- a/src/gitea_runner_manager/translations.json +++ b/src/gitea_runner_manager/translations.json @@ -31,6 +31,14 @@ "ru": "Проверить состояние зарегистрированного Gitea Runner.", "zh": "检查已注册的 Gitea Runner 状态。" }, + "Checking health of {name} on {host}": { + "bg": "Проверка на здравословното състояние на {name} на {host}", + "de": "Gesundheitsprüfung von {name} auf {host}", + "en": "Checking health of {name} on {host}", + "pl": "Sprawdzanie zdrowia {name} na {host}", + "ru": "Проверка здоровья {name} на {host}", + "zh": "正在检查 {host} 上 {name} 的健康状态" + }, "Checking status of Gitea Runner {name} on {host}": { "bg": "Проверка на състоянието на Gitea Runner {name} на {host}", "de": "Prüfe Status von Gitea Runner {name} auf {host}", @@ -175,6 +183,22 @@ "ru": "Токен админ API Gitea для интеграционного теста (env: CI_GITEA_TOKEN)", "zh": "Gitea 管理员 API 令牌，用于集成测试（环境变量: CI_GITEA_TOKEN）" }, + "HEALTHY": { + "bg": "ЗДРАВ", + "de": "GESUND", + "en": "HEALTHY", + "pl": "ZDROWY", + "ru": "ЗДОРОВ", + "zh": "健康" + }, + "Health check failed: {error}": { + "bg": "Проверката на здравословното състояние неуспешна: {error}", + "de": "Gesundheitsprüfung fehlgeschlagen: {error}", + "en": "Health check failed: {error}", + "pl": "Sprawdzanie zdrowia nie powiodło się: {error}", + "ru": "Проверка здоровья не удалась: {error}", + "zh": "健康检查失败: {error}" + }, "HOST": { "bg": "ХОСТ", "de": "HOST", @@ -239,6 +263,14 @@ "ru": "Ошибка списка: {error}", "zh": "列表失败: {error}" }, + "MESSAGE": { + "bg": "СЪОБЩЕНИЕ", + "de": "MELDUNG", + "en": "MESSAGE", + "pl": "WIADOMOŚĆ", + "ru": "СООБЩЕНИЕ", + "zh": "消息" + }, "NAME": { "bg": "ИМЕ", "de": "NAME", @@ -263,6 +295,14 @@ "ru": "Нет зарегистрированных runners. Используйте 'grm install' чтобы добавить.", "zh": "没有已注册的 runners。使用 'grm install' 添加一个。" }, + "One or more runners are unhealthy": { + "bg": "Един или повече runners са нездравословни", + "de": "Ein oder mehrere Runner sind fehlerhaft", + "en": "One or more runners are unhealthy", + "pl": "Jeden lub więcej runnerów jest w złym stanie", + "ru": "Один или несколько runners нездоровы", + "zh": "一个或多个 runners 不健康" + }, "Override SSH key from registry": { "bg": "Замяна на SSH ключа от регистъра", "de": "SSH-Schlüssel aus Registrierung überschreiben", @@ -423,6 +463,14 @@ "ru": "Выполнение Ansible playbook", "zh": "正在运行 Ansible playbook" }, + "Run health check on one or all registered runners.": { + "bg": "Проверка на здравословното състояние на един или всички регистрирани runners.", + "de": "Gesundheitsprüfung für einen oder alle registrierten Runner ausführen.", + "en": "Run health check on one or all registered runners.", + "pl": "Uruchom sprawdzanie zdrowia jednego lub wszystkich zarejestrowanych runnerów.", + "ru": "Проверить здоровье одного или всех зарегистрированных runners.", + "zh": "对一个或所有已注册 runners 运行健康检查。" + }, "SSH user (env: GITEA_RUNNER_USER)": { "bg": "SSH потребител (env: GITEA_RUNNER_USER)", "de": "SSH-Benutzer (env: GITEA_RUNNER_USER)", diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 9cc4a97..9b8f4eb 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -852,6 +852,75 @@ class TestCLI: assert result.exit_code != 0 assert "fail" in result.output + @patch("gitea_runner_manager.cli.RunnerManager") + def test_health_all_healthy(self, mock_manager_class: MagicMock) -> None: + mock_manager = MagicMock() + mock_manager.health.return_value = [ + {"name": "r1", "host": "10.0.0.1", "healthy": "yes", "message": "runner healthy, disk at 42%"}, + {"name": "r2", "host": "10.0.0.2", "healthy": "yes", "message": "runner healthy, disk at 50%"}, + ] + mock_manager_class.return_value = mock_manager + + runner = CliRunner() + result = runner.invoke(cli, ["health"], input="secret\n") + assert result.exit_code == 0 + assert "r1" in result.output + assert "r2" in result.output + assert "yes" in result.output + + @patch("gitea_runner_manager.cli.RunnerManager") + def test_health_with_unhealthy(self, mock_manager_class: MagicMock) -> None: + mock_manager = MagicMock() + mock_manager.health.return_value = [ + {"name": "r1", "host": "10.0.0.1", "healthy": "yes", "message": "runner healthy, disk at 42%"}, + {"name": "r2", "host": "10.0.0.2", "healthy": "no", "message": "Docker daemon down"}, + ] + mock_manager_class.return_value = mock_manager + + runner = CliRunner() + result = runner.invoke(cli, ["health"], input="secret\n") + assert result.exit_code != 0 + assert "unhealthy" in result.output.lower() + assert "r2" in result.output + + @patch("gitea_runner_manager.cli.RunnerManager") + def test_health_single_runner(self, mock_manager_class: MagicMock) -> None: + mock_manager = MagicMock() + mock_manager.health.return_value = [ + {"name": "r1", "host": "10.0.0.1", "healthy": "yes", "message": "runner healthy, disk at 42%"}, + ] + mock_manager_class.return_value = mock_manager + + runner = CliRunner() + result = runner.invoke(cli, ["health", "r1"], input="secret\n") + assert result.exit_code == 0 + assert "r1" in result.output + mock_manager.health.assert_called_once() + + @patch("gitea_runner_manager.cli.RunnerManager") + def test_health_empty(self, mock_manager_class: MagicMock) -> None: + mock_manager = MagicMock() + mock_manager.health.return_value = [] + mock_manager_class.return_value = mock_manager + + runner = CliRunner() + result = runner.invoke(cli, ["health"], input="secret\n") + assert result.exit_code == 0 + assert "No runners registered" in result.output + + @patch("gitea_runner_manager.cli.RunnerManager") + def test_health_error(self, mock_manager_class: MagicMock) -> None: + mock_manager = MagicMock() + from gitea_runner_manager.exceptions import AnsibleError + + mock_manager.health.side_effect = AnsibleError("fail") + mock_manager_class.return_value = mock_manager + + runner = CliRunner() + result = runner.invoke(cli, ["health"], input="secret\n") + assert result.exit_code != 0 + assert "fail" in result.output + @patch("gitea_runner_manager.cli.os.getlogin", side_effect=OSError("no tty")) @patch("gitea_runner_manager.cli.RunnerManager") def test_default_user_fallback_on_getlogin_error( diff --git a/tests/unit/test_runner_manager.py b/tests/unit/test_runner_manager.py index ebd3c33..b05a28f 100644 --- a/tests/unit/test_runner_manager.py +++ b/tests/unit/test_runner_manager.py @@ -625,6 +625,112 @@ class TestRunnerManager: assert manager.list_runners(no_status=True) == [] +class TestHealth: + """Tests for the ``health`` method.""" + + def test_health_single_runner_healthy(self) -> None: + mock_registry = MagicMock() + mock_registry.get.return_value = {"host": "10.0.0.1", "user": "ubuntu", "key": "/key"} + manager = RunnerManager(registry=mock_registry) + mock_executor = MagicMock() + mock_executor.run_ad_hoc.return_value = "OK: runner healthy, disk at 42%" + manager._executor = mock_executor + + results = manager.health(name="r1") + assert len(results) == 1 + assert results[0]["name"] == "r1" + assert results[0]["healthy"] == "yes" + assert "runner healthy" in results[0]["message"] + mock_executor.run_ad_hoc.assert_called_once() + + def test_health_single_runner_unhealthy(self) -> None: + mock_registry = MagicMock() + mock_registry.get.return_value = {"host": "10.0.0.1", "user": "ubuntu", "key": None} + manager = RunnerManager(registry=mock_registry) + mock_executor = MagicMock() + mock_executor.run_ad_hoc.return_value = "CRITICAL: Docker daemon still down after restart" + manager._executor = mock_executor + + results = manager.health(name="r1") + assert len(results) == 1 + assert results[0]["healthy"] == "no" + assert "Docker daemon still down" in results[0]["message"] + + def test_health_all_runners(self) -> None: + mock_registry = MagicMock() + mock_registry.list.return_value = { + "r1": {"host": "10.0.0.1", "user": "ubuntu", "key": None}, + "r2": {"host": "10.0.0.2", "user": "ubuntu", "key": None}, + } + manager = RunnerManager(registry=mock_registry) + mock_executor = MagicMock() + mock_executor.run_ad_hoc.side_effect = [ + "OK: runner healthy, disk at 42%", + "ERROR: gitea-runner service is inactive, restarting", + ] + manager._executor = mock_executor + + results = manager.health() + assert len(results) == 2 + assert results[0]["healthy"] == "yes" + assert results[1]["healthy"] == "no" + + def test_health_empty_registry(self) -> None: + mock_registry = MagicMock() + mock_registry.list.return_value = {} + manager = RunnerManager(registry=mock_registry) + results = manager.health() + assert results == [] + + def test_health_ansible_error(self) -> None: + mock_registry = MagicMock() + mock_registry.get.return_value = {"host": "10.0.0.1", "user": "ubuntu", "key": None} + manager = RunnerManager(registry=mock_registry) + mock_executor = MagicMock() + mock_executor.run_ad_hoc.side_effect = AnsibleError("ssh unreachable") + manager._executor = mock_executor + + results = manager.health(name="r1") + assert len(results) == 1 + assert results[0]["healthy"] == "no" + assert "ssh unreachable" in results[0]["message"] + + def test_health_with_host_override(self) -> None: + mock_registry = MagicMock() + mock_registry.get.return_value = {"host": "10.0.0.1", "user": "ubuntu", "key": None} + manager = RunnerManager(registry=mock_registry) + mock_executor = MagicMock() + mock_executor.run_ad_hoc.return_value = "OK: runner healthy, disk at 50%" + manager._executor = mock_executor + + results = manager.health(name="r1", host="10.0.0.99", user="root") + assert len(results) == 1 + assert results[0]["host"] == "10.0.0.99" + call_args = mock_executor.run_ad_hoc.call_args.args + assert call_args[0] == "10.0.0.99" + assert call_args[1] == "root" + + def test_health_runner_not_found(self) -> None: + mock_registry = MagicMock() + mock_registry.get.return_value = None + manager = RunnerManager(registry=mock_registry) + with pytest.raises(AnsibleError, match="not found in registry"): + manager.health(name="nonexistent") + + def test_health_passes_become_pass(self) -> None: + """become_pass is forwarded to run_ad_hoc for sudo authentication.""" + mock_registry = MagicMock() + mock_registry.get.return_value = {"host": "10.0.0.1", "user": "ubuntu", "key": None} + manager = RunnerManager(registry=mock_registry) + mock_executor = MagicMock() + mock_executor.run_ad_hoc.return_value = "OK: runner healthy, disk at 42%" + manager._executor = mock_executor + + manager.health(name="r1", become_pass="s3cr3t") + call_kwargs = mock_executor.run_ad_hoc.call_args.kwargs + assert call_kwargs["become_pass"] == "s3cr3t" + + class TestExtraVarsFile: """Tests for the ``_extra_vars_file`` context manager."""