diff --git a/ansible/roles/gitea_runner/defaults/main.yml b/ansible/roles/gitea_runner/defaults/main.yml index 5a5fe71..c47c305 100644 --- a/ansible/roles/gitea_runner/defaults/main.yml +++ b/ansible/roles/gitea_runner/defaults/main.yml @@ -90,6 +90,22 @@ gitea_runner_remove_user: true gitea_runner_log_level: "info" gitea_runner_container_label: "gitea-runner=true" gitea_runner_file: ".runner" +# force_pull: when false (default), the runner reuses locally cached images +# instead of pulling on every job. Pre-cached images (via the pre-cache timer +# or pre_pull_images task) eliminate registry thundering-herd when all runners +# start jobs simultaneously. +gitea_runner_force_pull: false +# Container options passed to `docker run` for CI job containers. +# Mounts the host rootless Docker socket as /run/host-docker.sock so +# start_docker.py inside the container can detect and use the host daemon +# (full disk, no nested DinD) instead of starting an inner dockerd. +gitea_runner_container_options: "-v /run/user/{{ gitea_runner_uid }}/docker.sock:/run/host-docker.sock" +# Volumes allowed in CI job containers (validated by the runner against +# container.options and job-level volumes). Must include the host Docker +# socket mount target. +gitea_runner_valid_volumes: + - "/run/host-docker.sock" + - "/run/user/{{ gitea_runner_uid }}/docker.sock" # Containerd version pinning — Docker 28.x vendors containerd v2.1.x internally. # containerd.io >= 2.3 ships a shim that returns a protobuf BootstrapResult which @@ -135,3 +151,11 @@ gitea_runner_docker_ipv6_cidr: "fd00:dead:beef::/48" # disk-space prune only removes dangling images, so pre-pulled tagged images persist. # Set to [] to skip pre-pulling. Images are pulled as the runner user via rootless Docker. gitea_runner_pre_pull_images: [] + +# Pre-cache timer: periodically pulls the runner container image so it stays +# fresh in the local Docker cache. This prevents thundering-herd registry +# timeouts when all runners start CI jobs simultaneously with empty caches. +# Runs every 6 hours (aligned with prune schedule). Set to empty string to +# disable the timer. +gitea_runner_pre_cache_schedule: "*-*-* 00/6:30:00" +gitea_runner_pre_cache_images: "{{ gitea_runner_pre_pull_images }}" diff --git a/ansible/roles/gitea_runner/tasks/main.yml b/ansible/roles/gitea_runner/tasks/main.yml index d887d03..caf6f9c 100644 --- a/ansible/roles/gitea_runner/tasks/main.yml +++ b/ansible/roles/gitea_runner/tasks/main.yml @@ -20,6 +20,9 @@ - name: Include pre-pull images ansible.builtin.include_tasks: pre_pull_images.yml +- name: Include pre-cache timer + ansible.builtin.include_tasks: pre_cache.yml + - name: Include integration test ansible.builtin.include_tasks: integration_test.yml when: not gitea_runner_skip_registration diff --git a/ansible/roles/gitea_runner/tasks/pre_cache.yml b/ansible/roles/gitea_runner/tasks/pre_cache.yml new file mode 100644 index 0000000..5f6713f --- /dev/null +++ b/ansible/roles/gitea_runner/tasks/pre_cache.yml @@ -0,0 +1,67 @@ +--- +# Periodic timer that pre-pulls CI runner images into the local Docker cache. +# Prevents thundering-herd registry timeouts when all runners start jobs +# simultaneously with empty/stale caches. Runs every 6 hours (configurable). +# The prune timer removes dangling images but NOT tagged ones, so pre-pulled +# images persist between runs. + +- name: Create docker-pull-images user service file + ansible.builtin.template: + src: docker-pull-images.service.j2 + dest: "{{ gitea_runner_home }}/.config/systemd/user/docker-pull-images.service" + owner: "{{ gitea_runner_service_user }}" + group: "{{ gitea_runner_service_user }}" + mode: "0644" + register: gitea_runner_pre_cache_service + +- name: Create docker-pull-images user timer file + ansible.builtin.template: + src: docker-pull-images.timer.j2 + dest: "{{ gitea_runner_home }}/.config/systemd/user/docker-pull-images.timer" + owner: "{{ gitea_runner_service_user }}" + group: "{{ gitea_runner_service_user }}" + mode: "0644" + register: gitea_runner_pre_cache_timer + +- name: Reload systemd user daemon for pre-cache timer + ansible.builtin.command: systemctl --user daemon-reload + become: true + become_user: "{{ gitea_runner_service_user }}" + environment: + XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}" + DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/{{ gitea_runner_uid | default(0) }}/bus" + changed_when: true + when: + - gitea_runner_systemd_available.stat.exists + - gitea_runner_docker_rootless_setup + - gitea_runner_pre_cache_service is changed or gitea_runner_pre_cache_timer is changed + - gitea_runner_pre_cache_schedule | length > 0 + - gitea_runner_pre_cache_images | length > 0 + +- name: Enable and start docker-pull-images user timer + ansible.builtin.command: systemctl --user enable --now docker-pull-images.timer + become: true + become_user: "{{ gitea_runner_service_user }}" + environment: + XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}" + DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/{{ gitea_runner_uid | default(0) }}/bus" + changed_when: true + when: + - gitea_runner_systemd_available.stat.exists + - gitea_runner_docker_rootless_setup + - gitea_runner_pre_cache_schedule | length > 0 + - gitea_runner_pre_cache_images | length > 0 + +- name: Disable and stop docker-pull-images timer (no images or schedule) + ansible.builtin.command: systemctl --user disable --now docker-pull-images.timer + become: true + become_user: "{{ gitea_runner_service_user }}" + environment: + XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}" + DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/{{ gitea_runner_uid | default(0) }}/bus" + changed_when: true + failed_when: false + when: + - gitea_runner_systemd_available.stat.exists + - gitea_runner_docker_rootless_setup + - gitea_runner_pre_cache_schedule | length == 0 or gitea_runner_pre_cache_images | length == 0 diff --git a/ansible/roles/gitea_runner/templates/docker-pull-images.service.j2 b/ansible/roles/gitea_runner/templates/docker-pull-images.service.j2 new file mode 100644 index 0000000..5226970 --- /dev/null +++ b/ansible/roles/gitea_runner/templates/docker-pull-images.service.j2 @@ -0,0 +1,14 @@ +[Unit] +Description=Pre-pull Docker images for CI runner cache +After=docker.service +Wants=docker.service + +[Service] +Type=oneshot +Environment=DOCKER_HOST=unix:///run/user/{{ gitea_runner_uid }}/docker.sock +Environment=XDG_RUNTIME_DIR=/run/user/{{ gitea_runner_uid }} +# Pull each image quietly. docker pull exits 0 if image is already up-to-date, +# so this is idempotent. Errors are non-fatal (image may already be cached). +{% for image in gitea_runner_pre_cache_images %} +ExecStart=/usr/bin/docker pull -q {{ image }} +{% endfor %} diff --git a/ansible/roles/gitea_runner/templates/docker-pull-images.timer.j2 b/ansible/roles/gitea_runner/templates/docker-pull-images.timer.j2 new file mode 100644 index 0000000..95c3391 --- /dev/null +++ b/ansible/roles/gitea_runner/templates/docker-pull-images.timer.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Periodic Docker image pre-cache for CI runner + +[Timer] +OnCalendar={{ gitea_runner_pre_cache_schedule }} +Persistent=true +RandomizedDelaySec=300 + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/gitea_runner/templates/gitea-runner-config.yaml.j2 b/ansible/roles/gitea_runner/templates/gitea-runner-config.yaml.j2 index a5adca5..d79e65c 100644 --- a/ansible/roles/gitea_runner/templates/gitea-runner-config.yaml.j2 +++ b/ansible/roles/gitea_runner/templates/gitea-runner-config.yaml.j2 @@ -9,3 +9,13 @@ runner: container: label: "{{ gitea_runner_container_label }}" docker_host: "unix:///run/user/{{ gitea_runner_uid }}/docker.sock" + force_pull: {{ gitea_runner_force_pull | lower }} +{% if gitea_runner_container_options | length > 0 %} + options: "{{ gitea_runner_container_options }}" +{% endif %} +{% if gitea_runner_valid_volumes | length > 0 %} + valid_volumes: +{% for volume in gitea_runner_valid_volumes %} + - "{{ volume }}" +{% endfor %} +{% endif %} diff --git a/pyproject.toml b/pyproject.toml index 3ef3fcb..cacdfb8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ ci = [ "build==1.5.1", "twine==6.2.0", # Reusable CI/CD and dev tools (auto-merge, pr-review, pre-push checks, etc.) - "devx @ git+https://git.oblachno.oblachno.fyi/oblachno-oss/devx.git@v0.50.0", + "devx @ git+https://git.oblachno.oblachno.fyi/oblachno-oss/devx.git@v0.50.1", ] # Lint and type-checking tools (validate job) lint = [ @@ -56,7 +56,7 @@ molecule = [ dev = [ "grm[ci,lint,molecule]", # Reusable CI/CD and dev tools (pre-push hooks, create-task, create-pr) - "devx @ git+https://git.oblachno.oblachno.fyi/oblachno-oss/devx.git@v0.50.0", + "devx @ git+https://git.oblachno.oblachno.fyi/oblachno-oss/devx.git@v0.50.1", # Non-Python dev dependency: checkmake (Makefile linter) # Install via: go install github.com/checkmake/checkmake/cmd/checkmake@latest ]