GRM-156: fix: switch default network driver to slirp4netns (pasta TCP RST bug)
Co-authored-by: oblachno Admin <admin@oblachno.oblachno.fyi>
This commit is contained in:
Emil Simeonov
@@ -3,6 +3,13 @@ gitea_runner_version: "2.0.1"
|
||||
gitea_runner_labels: "docker,ubuntu-latest:docker://runner-images:ubuntu-26.04"
|
||||
gitea_runner_skip_registration: false
|
||||
|
||||
# Force re-registration even if .runner file exists.
|
||||
# Use this when Gitea no longer recognizes the runner (e.g., after a Gitea
|
||||
# server restore/reinstall or when the runner record was deleted from the
|
||||
# admin UI). The existing .runner file is removed and a new registration is
|
||||
# performed. Requires registration_token.
|
||||
gitea_runner_force_reregister: false
|
||||
|
||||
# Per-runner user (rootless isolation)
|
||||
gitea_runner_user_prefix: "grm-"
|
||||
gitea_runner_base_home: "/home"
|
||||
@@ -37,6 +44,18 @@ gitea_runner_healthcheck_boot_delay: "2min"
|
||||
gitea_runner_healthcheck_disk_threshold: 75
|
||||
gitea_runner_healthcheck_script_path: "{{ gitea_runner_config_dir }}/healthcheck.sh"
|
||||
|
||||
# Auto-recovery: when the healthcheck detects an unregistered runner, it
|
||||
# can automatically re-register if a Gitea API token is provided.
|
||||
# The token needs admin or org-level access to fetch registration tokens.
|
||||
# Stored in a file readable by the runner user (mode 0400).
|
||||
# Set to empty string to disable auto-recovery (manual re-registration required).
|
||||
gitea_runner_auto_recover_api_token: ""
|
||||
|
||||
# Cooldown file to prevent auto-recovery loops (e.g., if Gitea is down).
|
||||
# The healthcheck writes a timestamp to this file after a re-registration
|
||||
# attempt and skips further attempts for the cooldown period.
|
||||
gitea_runner_auto_recover_cooldown_sec: 300
|
||||
|
||||
# Docker daemon resilience settings (applied to daemon.json).
|
||||
# live-restore: containers survive daemon restarts — prevents stuck container
|
||||
# states when the healthcheck restarts a hung daemon.
|
||||
|
||||
@@ -7,6 +7,22 @@
|
||||
group: "{{ gitea_runner_service_user }}"
|
||||
mode: "0755"
|
||||
|
||||
- name: Write auto-recovery API token file
|
||||
ansible.builtin.copy:
|
||||
content: "{{ gitea_runner_auto_recover_api_token }}"
|
||||
dest: "{{ gitea_runner_config_dir }}/auto-recover.token"
|
||||
owner: "{{ gitea_runner_service_user }}"
|
||||
group: "{{ gitea_runner_service_user }}"
|
||||
mode: "0400"
|
||||
no_log: true
|
||||
when: gitea_runner_auto_recover_api_token | length > 0
|
||||
|
||||
- name: Remove stale auto-recovery token file (if auto-recovery disabled)
|
||||
ansible.builtin.file:
|
||||
path: "{{ gitea_runner_config_dir }}/auto-recover.token"
|
||||
state: absent
|
||||
when: gitea_runner_auto_recover_api_token | length == 0
|
||||
|
||||
- name: Create healthcheck user service file
|
||||
ansible.builtin.template:
|
||||
src: runner-healthcheck.service.j2
|
||||
|
||||
@@ -7,11 +7,20 @@
|
||||
group: "{{ gitea_runner_service_user }}"
|
||||
mode: "0755"
|
||||
|
||||
- name: Check if runner is already registered
|
||||
- name: Check if runner registration file exists
|
||||
ansible.builtin.stat:
|
||||
path: "{{ gitea_runner_data_dir }}/.runner"
|
||||
register: gitea_runner_registered
|
||||
|
||||
- name: Remove stale runner registration file
|
||||
ansible.builtin.file:
|
||||
path: "{{ gitea_runner_data_dir }}/.runner"
|
||||
state: absent
|
||||
when:
|
||||
- gitea_runner_registered.stat.exists
|
||||
- gitea_runner_force_reregister | bool
|
||||
register: gitea_runner_registration_removed
|
||||
|
||||
- name: Register runner with Gitea
|
||||
ansible.builtin.command: >
|
||||
{{ gitea_runner_binary_path }} register
|
||||
@@ -28,7 +37,23 @@
|
||||
XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid | default(0) }}"
|
||||
DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/{{ gitea_runner_uid | default(0) }}/bus"
|
||||
DOCKER_HOST: "unix:///run/user/{{ gitea_runner_uid | default(0) }}/docker.sock"
|
||||
when: not gitea_runner_registered.stat.exists
|
||||
when: not gitea_runner_registered.stat.exists or gitea_runner_force_reregister | bool
|
||||
register: gitea_runner_register_output
|
||||
changed_when: "'already exists' not in gitea_runner_register_output.stdout | default('')"
|
||||
changed_when: >-
|
||||
gitea_runner_register_output.rc == 0 and
|
||||
('already exists' not in gitea_runner_register_output.stdout | default(''))
|
||||
timeout: 60
|
||||
|
||||
- name: Ensure runner service is running after registration
|
||||
ansible.builtin.command: systemctl --user start gitea-runner
|
||||
become: true
|
||||
become_user: "{{ gitea_runner_service_user }}"
|
||||
environment:
|
||||
XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}"
|
||||
DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/{{ gitea_runner_uid | default(0) }}/bus"
|
||||
changed_when: true
|
||||
when:
|
||||
- gitea_runner_systemd_available.stat.exists
|
||||
- gitea_runner_docker_rootless_setup
|
||||
- gitea_runner_register_output is defined
|
||||
- gitea_runner_register_output.rc | default(1) == 0
|
||||
|
||||
@@ -53,72 +53,6 @@
|
||||
register: gitea_runner_docker_install
|
||||
when: ansible_facts['os_family'] == 'Debian'
|
||||
|
||||
# Docker 28.x vendors containerd v2.1.x. containerd.io >= 2.3 ships a shim that
|
||||
# returns a protobuf BootstrapResult the vendored 2.1.x code cannot parse, causing
|
||||
# "failed to create TTRPC connection: unsupported protocol" on every container start.
|
||||
# Detect the mismatch and downgrade containerd.io to the latest compatible 2.2.x.
|
||||
- name: Check installed Docker CE version (Debian/Ubuntu)
|
||||
ansible.builtin.command: dpkg-query -W -f='${Version}' docker-ce
|
||||
register: gitea_runner_docker_version_check
|
||||
changed_when: false
|
||||
when: ansible_facts['os_family'] == 'Debian'
|
||||
|
||||
- name: Check installed containerd.io version (Debian/Ubuntu)
|
||||
ansible.builtin.shell: "set -o pipefail; dpkg-query -W -f='${Version}' containerd.io 2>/dev/null | cut -d: -f2 | cut -d- -f1"
|
||||
args:
|
||||
executable: /bin/bash
|
||||
register: gitea_runner_containerd_version_check
|
||||
changed_when: false
|
||||
when: ansible_facts['os_family'] == 'Debian'
|
||||
|
||||
- name: Determine if containerd.io is incompatible with installed Docker
|
||||
ansible.builtin.set_fact:
|
||||
gitea_runner_containerd_needs_downgrade: >-
|
||||
{{
|
||||
gitea_runner_containerd_version_check.stdout.split('.')[0] | int > gitea_runner_containerd_max_compatible_major
|
||||
or (
|
||||
gitea_runner_containerd_version_check.stdout.split('.')[0] | int == gitea_runner_containerd_max_compatible_major
|
||||
and gitea_runner_containerd_version_check.stdout.split('.')[1] | int > gitea_runner_containerd_max_compatible_minor
|
||||
)
|
||||
}}
|
||||
gitea_runner_docker_major: "{{ gitea_runner_docker_version_check.stdout.split('.')[0] | default('0') | int }}"
|
||||
when: ansible_facts['os_family'] == 'Debian'
|
||||
|
||||
- name: Find latest compatible containerd.io version (Debian/Ubuntu)
|
||||
ansible.builtin.shell: |
|
||||
set -o pipefail
|
||||
apt-cache madison containerd.io \
|
||||
| awk -F'|' '{print $2}' \
|
||||
| tr -d ' ' \
|
||||
| grep -E '^{{ gitea_runner_containerd_max_compatible_major }}\.{{ gitea_runner_containerd_max_compatible_minor }}\.' \
|
||||
| head -1
|
||||
args:
|
||||
executable: /bin/bash
|
||||
register: gitea_runner_containerd_compatible_version
|
||||
changed_when: false
|
||||
when:
|
||||
- ansible_facts['os_family'] == 'Debian'
|
||||
- gitea_runner_containerd_needs_downgrade | default(false)
|
||||
- gitea_runner_docker_major | int < 29
|
||||
|
||||
- name: Downgrade containerd.io to compatible version (Debian/Ubuntu)
|
||||
ansible.builtin.apt:
|
||||
name: "containerd.io={{ gitea_runner_containerd_compatible_version.stdout }}"
|
||||
state: present
|
||||
allow_downgrades: true
|
||||
register: gitea_runner_containerd_downgrade
|
||||
when:
|
||||
- ansible_facts['os_family'] == 'Debian'
|
||||
- gitea_runner_containerd_needs_downgrade | default(false)
|
||||
- gitea_runner_docker_major | int < 29
|
||||
- gitea_runner_containerd_compatible_version.stdout | length > 0
|
||||
|
||||
- name: Hold containerd.io package to prevent auto-upgrade (Debian/Ubuntu)
|
||||
ansible.builtin.dpkg_selections:
|
||||
name: containerd.io
|
||||
selection: hold
|
||||
when: ansible_facts['os_family'] == 'Debian'
|
||||
|
||||
- name: Update pacman cache (Arch Linux)
|
||||
community.general.pacman:
|
||||
update_cache: true
|
||||
|
||||
@@ -44,6 +44,117 @@ if [[ "$runner_state" != "active" ]]; then
|
||||
echo "RECOVERED: gitea-runner service restarted successfully"
|
||||
fi
|
||||
|
||||
# 2b. Detect unregistered runner state. When Gitea no longer recognizes the
|
||||
# runner (e.g., server restore, runner record deleted, Gitea restart with
|
||||
# token salt change), the runner logs "unregistered runner" every few seconds.
|
||||
# A service restart will not fix this; re-registration is required.
|
||||
{% if gitea_runner_auto_recover_api_token %}
|
||||
# Auto-recovery is enabled: fetch a new registration token from the Gitea API
|
||||
# and re-register the runner automatically. A cooldown prevents infinite loops.
|
||||
GITEA_API_TOKEN_FILE="{{ gitea_runner_config_dir }}/auto-recover.token"
|
||||
COOLDOWN_FILE="{{ gitea_runner_data_dir }}/auto-recover.cooldown"
|
||||
COOLDOWN_SEC={{ gitea_runner_auto_recover_cooldown_sec }}
|
||||
GITEA_URL="{{ gitea_url }}"
|
||||
RUNNER_NAME="{{ gitea_runner_name }}"
|
||||
RUNNER_LABELS="{{ gitea_runner_labels }}"
|
||||
BINARY="{{ gitea_runner_binary_path }}"
|
||||
{% endif %}
|
||||
recent_errors=$(journalctl --user -u gitea-runner.service --since "5 minutes ago" --no-pager -q 2>/dev/null | grep -c "unregistered runner" || true)
|
||||
if [[ "$recent_errors" -ge 3 ]]; then
|
||||
echo "CRITICAL: runner is unregistered in Gitea (re-login failed $recent_errors times in 5 minutes)."
|
||||
{% if gitea_runner_auto_recover_api_token %}
|
||||
# Check cooldown — skip if we recently attempted recovery
|
||||
if [[ -f "$COOLDOWN_FILE" ]]; then
|
||||
last_attempt=$(cat "$COOLDOWN_FILE" 2>/dev/null || echo 0)
|
||||
now=$(date +%s)
|
||||
elapsed=$((now - last_attempt))
|
||||
if [[ "$elapsed" -lt "$COOLDOWN_SEC" ]]; then
|
||||
echo "SKIP: auto-recovery cooldown active (${elapsed}s < ${COOLDOWN_SEC}s). Will retry later."
|
||||
exit 3
|
||||
fi
|
||||
fi
|
||||
|
||||
# Mark attempt time BEFORE trying (so failures also get cooldown)
|
||||
date +%s > "$COOLDOWN_FILE" 2>/dev/null || true
|
||||
|
||||
# Read the API token
|
||||
if [[ ! -f "$GITEA_API_TOKEN_FILE" ]]; then
|
||||
echo "ERROR: auto-recover token file not found at $GITEA_API_TOKEN_FILE. Manual re-registration required."
|
||||
exit 3
|
||||
fi
|
||||
API_TOKEN=$(cat "$GITEA_API_TOKEN_FILE" 2>/dev/null || true)
|
||||
if [[ -z "$API_TOKEN" ]]; then
|
||||
echo "ERROR: auto-recover token file is empty. Manual re-registration required."
|
||||
exit 3
|
||||
fi
|
||||
|
||||
echo "ATTEMPT: auto-recovering by fetching new registration token and re-registering..."
|
||||
|
||||
# Fetch a new registration token from the Gitea API
|
||||
# Try org-level first (for org-scoped runners), then instance-level
|
||||
REG_TOKEN=""
|
||||
for endpoint in \
|
||||
"api/v1/orgs/{{ gitea_runner_org | default('oblachno') }}/actions/runners/registration-token" \
|
||||
"api/v1/admin/actions/runners/registration-token"; do
|
||||
REG_TOKEN=$(curl -sf --connect-timeout 5 --max-time 10 -X POST \
|
||||
-H "Authorization: token $API_TOKEN" \
|
||||
"${GITEA_URL}/${endpoint}" 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin).get('token',''))" 2>/dev/null || true)
|
||||
if [[ -n "$REG_TOKEN" ]]; then
|
||||
echo "INFO: fetched registration token from ${endpoint}"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ -z "$REG_TOKEN" ]]; then
|
||||
echo "ERROR: failed to fetch registration token from Gitea API. Is Gitea reachable?"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
# Stop the runner service
|
||||
systemctl --user stop gitea-runner.service 2>/dev/null || true
|
||||
sleep 1
|
||||
|
||||
# Remove the stale .runner file
|
||||
rm -f "{{ gitea_runner_data_dir }}/.runner" 2>/dev/null || true
|
||||
|
||||
# Re-register
|
||||
cd "{{ gitea_runner_data_dir }}"
|
||||
if "$BINARY" register \
|
||||
--token "$REG_TOKEN" \
|
||||
--name "$RUNNER_NAME" \
|
||||
--instance "$GITEA_URL" \
|
||||
--labels "$RUNNER_LABELS" \
|
||||
--no-interactive 2>&1; then
|
||||
echo "RECOVERED: runner re-registered successfully"
|
||||
else
|
||||
echo "ERROR: re-registration failed. Manual intervention required."
|
||||
exit 3
|
||||
fi
|
||||
|
||||
# Start the runner service
|
||||
systemctl --user start gitea-runner.service
|
||||
sleep 3
|
||||
|
||||
# Verify recovery — check if unregistered errors stopped
|
||||
new_errors=$(journalctl --user -u gitea-runner.service --since "10 seconds ago" --no-pager -q 2>/dev/null | grep -c "unregistered runner" || true)
|
||||
if [[ "$new_errors" -eq 0 ]]; then
|
||||
echo "OK: runner recovered and no longer reporting unregistered errors"
|
||||
# Clear cooldown on success
|
||||
rm -f "$COOLDOWN_FILE" 2>/dev/null || true
|
||||
else
|
||||
echo "WARN: runner still showing unregistered errors after re-registration. Will retry after cooldown."
|
||||
exit 3
|
||||
fi
|
||||
{% else %}
|
||||
echo "Manual re-registration required: re-run gitea_runner role with gitea_runner_force_reregister=true."
|
||||
# Restart the service once in case it is a transient token refresh issue,
|
||||
# but this cannot recover an unregistered runner without re-registration.
|
||||
systemctl --user restart gitea-runner.service
|
||||
sleep 2
|
||||
exit 3
|
||||
{% endif %}
|
||||
fi
|
||||
|
||||
# 3. Check disk space — prune aggressively if below threshold
|
||||
disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
|
||||
if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_threshold }} ]]; then
|
||||
|
||||
Reference in New Issue
Block a user