From d16f53dfd7292f29fcedc2d62276068e0a6f4f3e Mon Sep 17 00:00:00 2001 From: Emil Simeonov Date: Sun, 23 Aug 2026 20:20:36 +0200 Subject: [PATCH 1/5] fix: use runuser for systemctl --user tasks in gitea_runner role The become_user pattern with sudo doesn't reliably connect to the user's D-Bus session, causing systemd user timers (docker-prune, runner-healthcheck) to silently fail to enable. This led to stale Docker images accumulating and filling the disk on CI runners. Switch all systemctl --user invocations to use runuser with explicit XDG_RUNTIME_DIR and DBUS_SESSION_BUS_ADDRESS environment variables, which correctly connects to the user's systemd session. Affected tasks: prune timer, healthcheck timer, gitea-runner service, rootless Docker daemon start/enable/restart, runner service restart. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../roles/gitea_runner/tasks/healthcheck.yml | 22 +++++----- ansible/roles/gitea_runner/tasks/prune.yml | 22 +++++----- ansible/roles/gitea_runner/tasks/register.yml | 17 ++++--- .../gitea_runner/tasks/rootless_docker.yml | 44 +++++++++---------- ansible/roles/gitea_runner/tasks/service.yml | 33 +++++++------- 5 files changed, 65 insertions(+), 73 deletions(-) diff --git a/ansible/roles/gitea_runner/tasks/healthcheck.yml b/ansible/roles/gitea_runner/tasks/healthcheck.yml index 30bd8c7..a99374d 100644 --- a/ansible/roles/gitea_runner/tasks/healthcheck.yml +++ b/ansible/roles/gitea_runner/tasks/healthcheck.yml @@ -40,24 +40,22 @@ mode: "0644" - name: Reload systemd user daemon for healthcheck timer - ansible.builtin.command: systemctl --user daemon-reload - become: true - become_user: "{{ gitea_runner_service_user }}" - environment: - XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}" - DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/{{ gitea_runner_uid | default(0) }}/bus" + ansible.builtin.command: > + runuser -u {{ gitea_runner_service_user }} -- + env XDG_RUNTIME_DIR=/run/user/{{ gitea_runner_uid }} + DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/{{ gitea_runner_uid | default(0) }}/bus + systemctl --user daemon-reload changed_when: true when: - gitea_runner_systemd_available.stat.exists - gitea_runner_docker_rootless_setup - name: Enable and start healthcheck user timer - ansible.builtin.command: systemctl --user enable --now runner-healthcheck.timer - become: true - become_user: "{{ gitea_runner_service_user }}" - environment: - XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}" - DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/{{ gitea_runner_uid | default(0) }}/bus" + ansible.builtin.command: > + runuser -u {{ gitea_runner_service_user }} -- + env XDG_RUNTIME_DIR=/run/user/{{ gitea_runner_uid }} + DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/{{ gitea_runner_uid | default(0) }}/bus + systemctl --user enable --now runner-healthcheck.timer changed_when: true when: - gitea_runner_systemd_available.stat.exists diff --git a/ansible/roles/gitea_runner/tasks/prune.yml b/ansible/roles/gitea_runner/tasks/prune.yml index 245b17c..a5a81ea 100644 --- a/ansible/roles/gitea_runner/tasks/prune.yml +++ b/ansible/roles/gitea_runner/tasks/prune.yml @@ -18,12 +18,11 @@ register: gitea_runner_prune_timer - name: Reload systemd user daemon for prune timer - ansible.builtin.command: systemctl --user daemon-reload - become: true - become_user: "{{ gitea_runner_service_user }}" - environment: - XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}" - DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/{{ gitea_runner_uid | default(0) }}/bus" + ansible.builtin.command: > + runuser -u {{ gitea_runner_service_user }} -- + env XDG_RUNTIME_DIR=/run/user/{{ gitea_runner_uid }} + DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/{{ gitea_runner_uid | default(0) }}/bus + systemctl --user daemon-reload changed_when: true when: - gitea_runner_systemd_available.stat.exists @@ -31,12 +30,11 @@ - gitea_runner_prune_service is changed or gitea_runner_prune_timer is changed - name: Enable and start docker-prune user timer - ansible.builtin.command: systemctl --user enable --now docker-prune.timer - become: true - become_user: "{{ gitea_runner_service_user }}" - environment: - XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}" - DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/{{ gitea_runner_uid | default(0) }}/bus" + ansible.builtin.command: > + runuser -u {{ gitea_runner_service_user }} -- + env XDG_RUNTIME_DIR=/run/user/{{ gitea_runner_uid }} + DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/{{ gitea_runner_uid | default(0) }}/bus + systemctl --user enable --now docker-prune.timer changed_when: true when: - gitea_runner_systemd_available.stat.exists diff --git a/ansible/roles/gitea_runner/tasks/register.yml b/ansible/roles/gitea_runner/tasks/register.yml index d0a9248..9ae4b77 100644 --- a/ansible/roles/gitea_runner/tasks/register.yml +++ b/ansible/roles/gitea_runner/tasks/register.yml @@ -44,13 +44,16 @@ ('already exists' not in gitea_runner_register_output.stdout | default('')) timeout: 60 -- name: Ensure runner service is running after registration - ansible.builtin.command: systemctl --user start gitea-runner - become: true - become_user: "{{ gitea_runner_service_user }}" - environment: - XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}" - DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/{{ gitea_runner_uid | default(0) }}/bus" +# Note: service start is handled by service.yml (included after register.yml +# in install_runner.yml). Starting here fails because the systemd unit file +# has not been created yet. + +- name: Restart runner service after (re-)registration + ansible.builtin.command: > + runuser -u {{ gitea_runner_service_user }} -- + env XDG_RUNTIME_DIR=/run/user/{{ gitea_runner_uid }} + DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/{{ gitea_runner_uid | default(0) }}/bus + systemctl --user restart gitea-runner changed_when: true when: - gitea_runner_systemd_available.stat.exists diff --git a/ansible/roles/gitea_runner/tasks/rootless_docker.yml b/ansible/roles/gitea_runner/tasks/rootless_docker.yml index 8f087dc..760d7e7 100644 --- a/ansible/roles/gitea_runner/tasks/rootless_docker.yml +++ b/ansible/roles/gitea_runner/tasks/rootless_docker.yml @@ -212,22 +212,20 @@ - not gitea_runner_rootless_docker_check.stat.exists - name: Start rootless Docker daemon (systemd user service) - ansible.builtin.command: systemctl --user start docker - become: true - become_user: "{{ gitea_runner_service_user }}" - environment: - XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}" - DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/{{ gitea_runner_uid }}/bus" + ansible.builtin.command: > + runuser -u {{ gitea_runner_service_user }} -- + env XDG_RUNTIME_DIR=/run/user/{{ gitea_runner_uid }} + DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/{{ gitea_runner_uid | default(0) }}/bus + systemctl --user start docker changed_when: true when: gitea_runner_docker_rootless_setup - name: Enable rootless Docker daemon (systemd user service) - ansible.builtin.command: systemctl --user enable docker - become: true - become_user: "{{ gitea_runner_service_user }}" - environment: - XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}" - DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/{{ gitea_runner_uid }}/bus" + ansible.builtin.command: > + runuser -u {{ gitea_runner_service_user }} -- + env XDG_RUNTIME_DIR=/run/user/{{ gitea_runner_uid }} + DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/{{ gitea_runner_uid | default(0) }}/bus + systemctl --user enable docker changed_when: true when: gitea_runner_docker_rootless_setup @@ -257,12 +255,11 @@ when: gitea_runner_docker_rootless_setup - name: Reload systemd user daemon if network config changed - ansible.builtin.command: systemctl --user daemon-reload - become: true - become_user: "{{ gitea_runner_service_user }}" - environment: - XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}" - DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/{{ gitea_runner_uid }}/bus" + ansible.builtin.command: > + runuser -u {{ gitea_runner_service_user }} -- + env XDG_RUNTIME_DIR=/run/user/{{ gitea_runner_uid }} + DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/{{ gitea_runner_uid | default(0) }}/bus + systemctl --user daemon-reload changed_when: true when: - gitea_runner_docker_rootless_setup @@ -300,12 +297,11 @@ when: gitea_runner_docker_rootless_setup - name: Restart rootless Docker if config changed - ansible.builtin.command: systemctl --user restart docker - become: true - become_user: "{{ gitea_runner_service_user }}" - environment: - XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}" - DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/{{ gitea_runner_uid }}/bus" + ansible.builtin.command: > + runuser -u {{ gitea_runner_service_user }} -- + env XDG_RUNTIME_DIR=/run/user/{{ gitea_runner_uid }} + DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/{{ gitea_runner_uid | default(0) }}/bus + systemctl --user restart docker changed_when: true when: - gitea_runner_docker_rootless_setup diff --git a/ansible/roles/gitea_runner/tasks/service.yml b/ansible/roles/gitea_runner/tasks/service.yml index bc18c8a..b47ed62 100644 --- a/ansible/roles/gitea_runner/tasks/service.yml +++ b/ansible/roles/gitea_runner/tasks/service.yml @@ -9,12 +9,11 @@ register: gitea_runner_service_file - name: Reload systemd user daemon - ansible.builtin.command: systemctl --user daemon-reload - become: true - become_user: "{{ gitea_runner_service_user }}" - environment: - XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}" - DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/{{ gitea_runner_uid | default(0) }}/bus" + ansible.builtin.command: > + runuser -u {{ gitea_runner_service_user }} -- + env XDG_RUNTIME_DIR=/run/user/{{ gitea_runner_uid }} + DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/{{ gitea_runner_uid | default(0) }}/bus + systemctl --user daemon-reload changed_when: true when: - gitea_runner_systemd_available.stat.exists @@ -22,12 +21,11 @@ - gitea_runner_service_file is changed - name: Restart gitea-runner if service file changed - ansible.builtin.command: systemctl --user restart gitea-runner - become: true - become_user: "{{ gitea_runner_service_user }}" - environment: - XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}" - DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/{{ gitea_runner_uid | default(0) }}/bus" + ansible.builtin.command: > + runuser -u {{ gitea_runner_service_user }} -- + env XDG_RUNTIME_DIR=/run/user/{{ gitea_runner_uid }} + DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/{{ gitea_runner_uid | default(0) }}/bus + systemctl --user restart gitea-runner changed_when: true when: - gitea_runner_systemd_available.stat.exists @@ -35,12 +33,11 @@ - gitea_runner_service_file is changed - name: Enable and start gitea-runner user service - ansible.builtin.command: systemctl --user enable --now gitea-runner - become: true - become_user: "{{ gitea_runner_service_user }}" - environment: - XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}" - DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/{{ gitea_runner_uid | default(0) }}/bus" + ansible.builtin.command: > + runuser -u {{ gitea_runner_service_user }} -- + env XDG_RUNTIME_DIR=/run/user/{{ gitea_runner_uid }} + DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/{{ gitea_runner_uid | default(0) }}/bus + systemctl --user enable --now gitea-runner changed_when: true when: - gitea_runner_systemd_available.stat.exists -- 2.54.0 From fe9f66afb5e59a9b0a78a8f9e0e784f19418aeff Mon Sep 17 00:00:00 2001 From: Emil Simeonov Date: Mon, 17 Aug 2026 18:45:39 +0200 Subject: [PATCH 2/5] =?UTF-8?q?fix:=20runner=20self-healing=20=E2=80=94=20?= =?UTF-8?q?fix=20service=20startup=20order,=20add=20Docker=20restart=20ove?= =?UTF-8?q?rride?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three fixes for runner reliability: 1. Fix service startup order (install_runner.yml) - Move service.yml include BEFORE register.yml - Previously, register.yml tried to start gitea-runner.service before service.yml created the systemd unit file, causing "Unit gitea-runner.service not found" on first install - This only failed when registration actually happened (new runner); on idempotent re-runs registration was skipped so the bug was masked 2. Restart runner after (re-)registration (register.yml) - Add "Restart runner service after (re-)registration" task - When a runner is re-registered (force-reregister or auto-recovery), the service must be restarted to pick up the new .runner file - The old premature start task was already removed; this adds it back in the correct position (after service.yml has created the unit) 3. Docker daemon restart override (rootless_docker.yml) - Add Restart=always, RestartSec=5, StartLimitIntervalSec=300, StartLimitBurst=10 to both Docker service override blocks - Upstream dockerd-rootless-setuptool.sh defaults to StartLimitBurst=3 in 60s — too aggressive for OOM-prone hosts - If Docker crashes 3 times in a minute, it stops restarting permanently, taking the runner down with it - New settings allow 10 restarts in 5 minutes, matching the runner service's own restart policy Note: Auto-recovery (--auto-recover-token) already exists in the CLI and healthcheck script but defaults to disabled. Operators should pass --auto-recover-token during grm install to enable automatic re-registration when runners become unregistered. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ansible/roles/gitea_runner/tasks/install_runner.yml | 6 +++--- ansible/roles/gitea_runner/tasks/rootless_docker.yml | 8 ++++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/ansible/roles/gitea_runner/tasks/install_runner.yml b/ansible/roles/gitea_runner/tasks/install_runner.yml index b977c45..82d4c73 100644 --- a/ansible/roles/gitea_runner/tasks/install_runner.yml +++ b/ansible/roles/gitea_runner/tasks/install_runner.yml @@ -13,9 +13,9 @@ - name: Include validation ansible.builtin.include_tasks: validate.yml +- name: Include service setup + ansible.builtin.include_tasks: service.yml + - name: Include registration ansible.builtin.include_tasks: register.yml when: not gitea_runner_skip_registration - -- name: Include service setup - ansible.builtin.include_tasks: service.yml diff --git a/ansible/roles/gitea_runner/tasks/rootless_docker.yml b/ansible/roles/gitea_runner/tasks/rootless_docker.yml index 760d7e7..ae01c8c 100644 --- a/ansible/roles/gitea_runner/tasks/rootless_docker.yml +++ b/ansible/roles/gitea_runner/tasks/rootless_docker.yml @@ -143,6 +143,10 @@ {% if gitea_runner_docker_rootless_net_driver == 'pasta' %} Environment="DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS=--ipv6" {% endif %} + Restart=always + RestartSec=5 + StartLimitIntervalSec=300 + StartLimitBurst=10 mode: "0644" owner: "{{ gitea_runner_service_user }}" group: "{{ gitea_runner_service_user }}" @@ -248,6 +252,10 @@ {% if gitea_runner_docker_rootless_net_driver == 'pasta' %} Environment="DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS=--ipv6" {% endif %} + Restart=always + RestartSec=5 + StartLimitIntervalSec=300 + StartLimitBurst=10 mode: "0644" owner: "{{ gitea_runner_service_user }}" group: "{{ gitea_runner_service_user }}" -- 2.54.0 From 150d3267d178b9184530268edeee49959e044de4 Mon Sep 17 00:00:00 2001 From: Emil Simeonov Date: Mon, 24 Aug 2026 02:11:52 +0200 Subject: [PATCH 3/5] fix: pin devx to v0.50.1 (v0.50.5 tag does not exist) --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3ef3fcb..cacdfb8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ ci = [ "build==1.5.1", "twine==6.2.0", # Reusable CI/CD and dev tools (auto-merge, pr-review, pre-push checks, etc.) - "devx @ git+https://git.oblachno.oblachno.fyi/oblachno-oss/devx.git@v0.50.0", + "devx @ git+https://git.oblachno.oblachno.fyi/oblachno-oss/devx.git@v0.50.1", ] # Lint and type-checking tools (validate job) lint = [ @@ -56,7 +56,7 @@ molecule = [ dev = [ "grm[ci,lint,molecule]", # Reusable CI/CD and dev tools (pre-push hooks, create-task, create-pr) - "devx @ git+https://git.oblachno.oblachno.fyi/oblachno-oss/devx.git@v0.50.0", + "devx @ git+https://git.oblachno.oblachno.fyi/oblachno-oss/devx.git@v0.50.1", # Non-Python dev dependency: checkmake (Makefile linter) # Install via: go install github.com/checkmake/checkmake/cmd/checkmake@latest ] -- 2.54.0 From 4083835ca366449d95002f5f1670c7ac46415cbf Mon Sep 17 00:00:00 2001 From: Emil Simeonov Date: Mon, 24 Aug 2026 04:50:15 +0200 Subject: [PATCH 4/5] chore: re-trigger auto-merge -- 2.54.0 From 8b751e4eff22210c0d87f8236e0e5f67b3a1843f Mon Sep 17 00:00:00 2001 From: Emil Simeonov Date: Mon, 24 Aug 2026 05:03:51 +0200 Subject: [PATCH 5/5] fix: wait for molecule tests before auto-merge attempt --- .gitea/workflows/ci.yml | 43 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 55e3453..6a9f76f 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -295,6 +295,49 @@ jobs: --checklist-confirmed \ --checklist-categories 1,2,3,4,5,6,7,8,9,10,11,12,13 \ --body "Auto-approved: all CI checks passed (validate, molecule-tests)." + - name: Wait for molecule tests to complete + env: + CI_GITEA_API_TOKEN: ${{ secrets.CI_GITEA_API_TOKEN }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + . .venv/bin/activate 2>/dev/null || true + # Poll commit status until all required checks pass or fail + MAX_WAIT=600 # 10 minutes + ELAPSED=0 + while [ $ELAPSED -lt $MAX_WAIT ]; do + STATUS=$(curl -s -H "Authorization: token $CI_GITEA_API_TOKEN" \ + "https://git.oblachno.oblachno.fyi/api/v1/repos/${{ github.repository }}/commits/$HEAD_SHA/status" \ + | python3 -c " + import sys,json + d=json.load(sys.stdin) + statuses={s['context']:s['status'] for s in d.get('statuses',[])} + # Check if all molecule-tests contexts are terminal (success/failure) + mol_contexts=[k for k in statuses if 'molecule-tests' in k] + if not mol_contexts: + print('pending') + elif all(statuses[k] in ('success','failure') for k in mol_contexts): + if any(statuses[k]=='failure' for k in mol_contexts): + print('failure') + else: + print('success') + else: + print('pending') + ") + echo "Molecule tests status: $STATUS (elapsed: ${ELAPSED}s)" + if [ "$STATUS" = "success" ]; then + echo "All molecule tests passed." + break + elif [ "$STATUS" = "failure" ]; then + echo "ERROR: Molecule tests failed. Aborting auto-merge." + exit 1 + fi + sleep 30 + ELAPSED=$((ELAPSED + 30)) + done + if [ $ELAPSED -ge $MAX_WAIT ]; then + echo "ERROR: Timed out waiting for molecule tests." + exit 1 + fi - name: Squash merge with task ID env: CI_GITEA_API_TOKEN: ${{ secrets.CI_GITEA_API_TOKEN }} -- 2.54.0