diff --git a/ansible/roles/gitea_runner/defaults/main.yml b/ansible/roles/gitea_runner/defaults/main.yml index 62d3070..9221d9f 100644 --- a/ansible/roles/gitea_runner/defaults/main.yml +++ b/ansible/roles/gitea_runner/defaults/main.yml @@ -31,6 +31,13 @@ gitea_runner_prune_until: "24h" # container operations). gitea_runner_prune_schedule: "*-*-* 00/6:00:00" gitea_runner_prune_label: "gitea-runner=true" +# Shared scoped cleanup script (runner-cleanup.sh) used by the prune timer +# and the healthcheck disk-pressure tiers (GRM-173). +gitea_runner_cleanup_script_path: "{{ gitea_runner_config_dir }}/cleanup.sh" +# Regex alternation of image refs never removed by cleanup — warm base +# layers stay warm even under critical disk pressure. +gitea_runner_keep_images: + - "runner-images/" # Service configuration gitea_runner_service_restart_sec: "5" @@ -42,13 +49,20 @@ gitea_runner_service_restart_sec: "5" gitea_runner_healthcheck_interval: "2min" gitea_runner_healthcheck_boot_delay: "2min" gitea_runner_healthcheck_disk_threshold: 70 -# When disk reaches this level, prune EVERYTHING (no until-filter) — the -# runner is dangerously full and the gentle until=1h prune isn't enough. -# This removes all stopped containers and unused images regardless of age. -# At 75%+, molecule containers fail with "container is not running" because -# overlay2 runs out of space under parallel DinD load. +# At this level the cleanup script drops age limits — the runner is +# dangerously full and the gentle until=1h prune isn't enough. Leases, +# keep-images, running containers and CI job containers are still honored +# (GRM-173). At 75%+, molecule containers fail with "container is not +# running" because overlay2 runs out of space under parallel DinD load. gitea_runner_healthcheck_disk_critical: 75 gitea_runner_healthcheck_script_path: "{{ gitea_runner_config_dir }}/healthcheck.sh" +# Disk-pressure admission control (GRM-173): at critical disk usage the +# healthcheck stops gitea-runner.service (no new jobs are fetched) once no +# CI job container is running, and resumes it automatically after recovery. +gitea_runner_disk_admission_enabled: true +# Physical-host admission (GRM-173): act_runner capacity — max parallel +# tasks per runner. Declared explicitly (upstream default is 1). +gitea_runner_capacity: 1 # CI job containers older than this many minutes get an exec-responsiveness # probe; a timeout writes one diagnostics bundle per container for @@ -121,6 +135,13 @@ gitea_runner_valid_volumes: gitea_runner_containerd_max_compatible_major: 2 gitea_runner_containerd_max_compatible_minor: 2 +# Production-host exclusion (GRM-173): the role fails when the target is a +# production host — via this flag or the marker file — unless +# allow_production_host explicitly overrides. +gitea_runner_on_production_host: false +gitea_runner_allow_production_host: false +gitea_runner_production_marker_path: "/etc/oblachno/production-host" + # Docker installation (for rootless dependencies) gitea_runner_docker_gpg_key_path: "/etc/apt/keyrings/docker.gpg" gitea_runner_docker_apt_arch: "{{ 'amd64' if ansible_facts['architecture'] == 'x86_64' else ansible_facts['architecture'] }}" diff --git a/ansible/roles/gitea_runner/molecule/template-content/verify.yml b/ansible/roles/gitea_runner/molecule/template-content/verify.yml index aec3ea7..2f3021a 100644 --- a/ansible/roles/gitea_runner/molecule/template-content/verify.yml +++ b/ansible/roles/gitea_runner/molecule/template-content/verify.yml @@ -47,14 +47,39 @@ ansible.builtin.assert: that: - "'Type=oneshot' in prune_service.content | b64decode" - - "'docker rm -f' in prune_service.content | b64decode" - - "'status=exited' in prune_service.content | b64decode" - "'GITEA-ACTIONS-TASK' in prune_service.content | b64decode" - - "'docker system prune -af' in prune_service.content | b64decode" - - "'docker network prune' in prune_service.content | b64decode" - - "'docker builder prune' in prune_service.content | b64decode" + - "(gitea_runner_cleanup_script_path ~ ' --tier routine') in prune_service.content | b64decode" fail_msg: "Prune service template is missing expected directives" + - name: Read rendered cleanup script + ansible.builtin.slurp: + src: "{{ gitea_runner_cleanup_script_path }}" + register: cleanup_script + + - name: Assert cleanup script honors leases and tiers + ansible.builtin.assert: + that: + - "'org.oblachno.lease-until' in cleanup_script.content | b64decode" + - "'lease_active' in cleanup_script.content | b64decode" + - "'GITEA-ACTIONS-TASK' in cleanup_script.content | b64decode" + - "'runner-images/' in cleanup_script.content | b64decode" + - "'docker image inspect' in cleanup_script.content | b64decode" + - "'label!=' in cleanup_script.content | b64decode" + - "'docker system prune' not in cleanup_script.content | b64decode" + - "'critical)' in cleanup_script.content | b64decode" + fail_msg: "Cleanup script template is missing expected content" + + - name: Read rendered runner config + ansible.builtin.slurp: + src: "{{ gitea_runner_config_dir }}/config.yaml" + register: runner_config + + - name: Assert runner config declares capacity + ansible.builtin.assert: + that: + - "('capacity: ' ~ gitea_runner_capacity) in runner_config.content | b64decode" + fail_msg: "Runner config is missing capacity declaration" + - name: Read rendered prune timer template ansible.builtin.slurp: src: "{{ gitea_runner_home }}/.config/systemd/user/docker-prune.timer" @@ -108,12 +133,13 @@ - "'systemctl --user restart gitea-runner.service' in healthcheck_script.content | b64decode" - "'docker rm -f' in healthcheck_script.content | b64decode" - "'GITEA-ACTIONS-TASK' in healthcheck_script.content | b64decode" - - "'docker system prune -af' in healthcheck_script.content | b64decode" - - "'docker network prune' in healthcheck_script.content | b64decode" + - "'--tier critical' in healthcheck_script.content | b64decode" + - "'--tier pressure' in healthcheck_script.content | b64decode" + - "'disk-admission-block' in healthcheck_script.content | b64decode" + - "'systemctl --user stop gitea-runner.service' in healthcheck_script.content | b64decode" + - "'docker system prune' not in healthcheck_script.content | b64decode" - "'status=removing' in healthcheck_script.content | b64decode" - "'status=stopping' in healthcheck_script.content | b64decode" - - "'status=exited' in healthcheck_script.content | b64decode" - - "'status=dead' in healthcheck_script.content | b64decode" - "gitea_runner_healthcheck_disk_threshold | string in healthcheck_script.content | b64decode" - "gitea_runner_healthcheck_disk_critical | string in healthcheck_script.content | b64decode" fail_msg: "Healthcheck script template is missing expected content" diff --git a/ansible/roles/gitea_runner/tasks/main.yml b/ansible/roles/gitea_runner/tasks/main.yml index caf6f9c..caa906b 100644 --- a/ansible/roles/gitea_runner/tasks/main.yml +++ b/ansible/roles/gitea_runner/tasks/main.yml @@ -1,4 +1,23 @@ --- +# Implements: REQ-6 (GRM-173) — a CI runner must never be installed on a +# production host (production workloads must not share hardware with +# arbitrary CI jobs, and runner cleanup logic assumes a dedicated host). +- name: Check for production-host marker + ansible.builtin.stat: + path: "{{ gitea_runner_production_marker_path }}" + register: gitea_runner_production_marker + +- name: Fail on production hosts + ansible.builtin.fail: + msg: >- + Refusing to install a CI runner on a production host + (marker: {{ gitea_runner_production_marker_path }} present or + gitea_runner_on_production_host=true). Set + gitea_runner_allow_production_host=true to override. + when: + - not gitea_runner_allow_production_host + - gitea_runner_on_production_host or gitea_runner_production_marker.stat.exists + - name: Include systemd availability check ansible.builtin.include_tasks: systemd_check.yml diff --git a/ansible/roles/gitea_runner/tasks/prune.yml b/ansible/roles/gitea_runner/tasks/prune.yml index a5a81ea..fe9dfaf 100644 --- a/ansible/roles/gitea_runner/tasks/prune.yml +++ b/ansible/roles/gitea_runner/tasks/prune.yml @@ -1,4 +1,14 @@ --- +# Implements: REQ-2 (GRM-173) — shared scoped cleanup script used by both +# the prune timer and the healthcheck disk-pressure tiers. +- name: Create runner cleanup script + ansible.builtin.template: + src: runner-cleanup.sh.j2 + dest: "{{ gitea_runner_cleanup_script_path }}" + owner: "{{ gitea_runner_service_user }}" + group: "{{ gitea_runner_service_user }}" + mode: "0755" + - name: Create docker-prune user service file ansible.builtin.template: src: docker-prune.service.j2 diff --git a/ansible/roles/gitea_runner/templates/docker-prune.service.j2 b/ansible/roles/gitea_runner/templates/docker-prune.service.j2 index a794097..c57f18e 100644 --- a/ansible/roles/gitea_runner/templates/docker-prune.service.j2 +++ b/ansible/roles/gitea_runner/templates/docker-prune.service.j2 @@ -5,21 +5,10 @@ Description=Docker prune for Gitea runner resources Type=oneshot Environment=DOCKER_HOST=unix:///run/user/{{ gitea_runner_uid }}/docker.sock Environment=XDG_RUNTIME_DIR=/run/user/{{ gitea_runner_uid }} -# Force-remove stale *stopped* containers left behind by failed molecule tests. -# Implements: REQ-1 (GRM-166) — only containers with status=exited are -# eligible. RunningFor measures creation time, so a stale molecule instance -# (e.g. ubuntu-2604) that a new run restarts still looks ">1h old"; removing -# running containers kills active converges with "No such container" -# (infra nightly run 5710). Running leftovers are instead reused or destroyed -# by the next molecule create/destroy cycle. -# Exclude CI job containers (name starts with GITEA-ACTIONS-TASK) — removing -# them kills the active CI job and causes "RWLayer is unexpectedly nil" errors. -# Only remove containers older than 1 hour (grep for "hour/day/week/month/year -# ago" in RunningFor) to avoid removing containers a job just created. -ExecStart=/bin/sh -c 'docker ps -a --filter "status=exited" --format "{% raw %}{{.ID}} {{.Names}} {{.RunningFor}}{% endraw %}" 2>/dev/null | grep -v "GITEA-ACTIONS-TASK" | grep -E "(hour|day|week|month|year)s? ago" | awk "{print $1}" | xargs -r docker rm -f 2>/dev/null || true' -ExecStart=/usr/bin/docker system prune -af --filter "until={{ gitea_runner_prune_until }}" --volumes -# Prune networks older than the prune-until threshold to avoid removing -# networks that molecule tests are actively creating (e.g. 'traefik' network -# created during molecule create phase before containers are attached). -ExecStart=/usr/bin/docker network prune -f --filter "until={{ gitea_runner_prune_until }}" -ExecStart=/usr/bin/docker builder prune -f +# Implements: REQ-1/REQ-2 (GRM-173) — all cleanup goes through the shared +# scoped cleanup script: only stopped containers, CI job containers excluded +# (GITEA-ACTIONS-TASK prefix), valid `org.oblachno.lease-until` leases never +# removed, keep-images retained. The historical inline logic here killed +# active molecule converges ("No such container", infra nightly run 5710) +# and CI jobs ("RWLayer is unexpectedly nil"). +ExecStart={{ gitea_runner_cleanup_script_path }} --tier routine diff --git a/ansible/roles/gitea_runner/templates/gitea-runner-config.yaml.j2 b/ansible/roles/gitea_runner/templates/gitea-runner-config.yaml.j2 index d79e65c..884310c 100644 --- a/ansible/roles/gitea_runner/templates/gitea-runner-config.yaml.j2 +++ b/ansible/roles/gitea_runner/templates/gitea-runner-config.yaml.j2 @@ -3,6 +3,9 @@ log: runner: file: "{{ gitea_runner_file }}" + # Implements: REQ-5 (GRM-173) — physical-host admission: declared capacity + # limits parallel tasks instead of relying on labels alone. + capacity: {{ gitea_runner_capacity }} fetch_timeout: 50s fetch_interval: 2s diff --git a/ansible/roles/gitea_runner/templates/runner-cleanup.sh.j2 b/ansible/roles/gitea_runner/templates/runner-cleanup.sh.j2 new file mode 100644 index 0000000..cce3edb --- /dev/null +++ b/ansible/roles/gitea_runner/templates/runner-cleanup.sh.j2 @@ -0,0 +1,125 @@ +#!/bin/bash +# Scoped Docker cleanup for gitea-runner hosts. +# Implements: REQ-1..REQ-3 (GRM-173) — ownership leases, tiered watermarks, +# keep-images. Single entry point shared by docker-prune.service (routine) +# and runner-healthcheck.sh (pressure/critical). +# No `set -e`: a failing prune must not abort the remaining cleanup. +set -uo pipefail + +DOCKER_HOST="unix:///run/user/{{ gitea_runner_uid }}/docker.sock" +XDG_RUNTIME_DIR="/run/user/{{ gitea_runner_uid }}" +export DOCKER_HOST XDG_RUNTIME_DIR + +# REQ-1 label contract: `org.oblachno.lease-until` (epoch) protects an object +# while in the future; `org.oblachno.owner` records the owning run. +TIER="${1:-routine}" +LEASE_UNTIL_LABEL="org.oblachno.lease-until" +KEEP_IMAGES_RE="{{ gitea_runner_keep_images | join('|') }}" +now_epoch=$(date +%s) + +# Implements: REQ-1 — a lease whose `lease-until` epoch lies in the future +# protects its object from every removal path in this script. +lease_active() { + local until="$1" + [[ -n "$until" && "$until" =~ ^[0-9]+$ && "$until" -gt "$now_epoch" ]] +} + +# Remove stopped containers. $1 = "aged" (only >1h, RunningFor heuristic) +# or "all". CI job containers and valid leases are never removed. +remove_stopped_containers() { + local mode="$1" + # Implements: REQ-1/REQ-3 — pipe-separated fields; RunningFor contains + # spaces, so whitespace-splitting would break the age gate. + { timeout 30 docker ps -a --filter "status=exited" --filter "status=dead" \ + --format '{% raw %}{{.ID}}|{{.Names}}|{{.RunningFor}}|{{.Label "org.oblachno.lease-until"}}{% endraw %}' \ + 2>/dev/null || true; } \ + | while IFS='|' read -r cid cname running_for lease_until; do + [[ -z "$cid" ]] && continue + case "$cname" in GITEA-ACTIONS-TASK*) continue ;; esac + lease_active "$lease_until" && continue + if [[ "$mode" != "all" ]] \ + && ! grep -qE '(hour|day|week|month|year)s? ago' <<<"$running_for"; then + continue + fi + docker rm -f "$cid" >/dev/null 2>&1 || true + done +} + +# Remove unused images older than $1 ("all" = no age limit). The keep-list +# (warm base layers) and leased images are never removed; images referenced +# by any container are refused by the daemon anyway. `docker images` has no +# label formatter, so the lease is checked via inspect per candidate. +remove_old_images() { + local until="$1" + docker image prune -f >/dev/null 2>&1 || true + local filters=(--filter "dangling=false") + [[ "$until" != "all" ]] && filters+=(--filter "until=${until}") + { timeout 30 docker images "${filters[@]}" \ + --format '{% raw %}{{.ID}}|{{.Repository}}:{{.Tag}}{% endraw %}' \ + 2>/dev/null || true; } \ + | while IFS='|' read -r iid ref; do + [[ -z "$iid" || "$ref" == *""* ]] && continue + [[ -n "$KEEP_IMAGES_RE" && "$ref" =~ $KEEP_IMAGES_RE ]] && continue + local lease_until + lease_until=$(docker image inspect "$iid" \ + --format '{% raw %}{{index .Config.Labels "org.oblachno.lease-until"}}{% endraw %}' \ + 2>/dev/null || true) + lease_active "$lease_until" && continue + docker image rm "$iid" >/dev/null 2>&1 || true + done +} + +# Reclaim volumes/networks whose lease expired. The prune filters below +# skip every leased object (label!=); this pass removes the expired ones. +reclaim_expired_leases() { + timeout 20 docker volume ls -q --filter "label=${LEASE_UNTIL_LABEL}" 2>/dev/null \ + | while read -r vol; do + lease_until=$(docker volume inspect "$vol" \ + --format '{% raw %}{{index .Labels "org.oblachno.lease-until"}}{% endraw %}' \ + 2>/dev/null || true) + lease_active "$lease_until" || docker volume rm "$vol" >/dev/null 2>&1 || true + done + timeout 20 docker network ls -q --filter "label=${LEASE_UNTIL_LABEL}" 2>/dev/null \ + | while read -r net; do + lease_until=$(docker network inspect "$net" \ + --format '{% raw %}{{index .Labels "org.oblachno.lease-until"}}{% endraw %}' \ + 2>/dev/null || true) + lease_active "$lease_until" || docker network rm "$net" >/dev/null 2>&1 || true + done +} + +# Anonymous volumes only at routine/pressure tiers — a named volume may +# belong to a job between create/attach steps. Critical removes all unused. +prune_volumes_networks() { + local vol_all="$1" net_until="$2" + docker volume prune ${vol_all:+$vol_all} -f --filter "label!=${LEASE_UNTIL_LABEL}" >/dev/null 2>&1 || true + docker network prune -f --filter "label!=${LEASE_UNTIL_LABEL}" ${net_until:+--filter until=${net_until}} >/dev/null 2>&1 || true +} + +case "$TIER" in + routine) + remove_stopped_containers aged + remove_old_images "{{ gitea_runner_prune_until }}" + prune_volumes_networks "" "{{ gitea_runner_prune_until }}" + docker builder prune -f --filter "until=24h" >/dev/null 2>&1 || true + ;; + pressure) + remove_stopped_containers aged + remove_old_images "1h" + prune_volumes_networks "" "1h" + docker builder prune -f --filter "until=24h" >/dev/null 2>&1 || true + ;; + critical) + # Implements: REQ-3 — age limits dropped, ownership still honored. + remove_stopped_containers all + remove_old_images all + prune_volumes_networks "-a" "" + docker builder prune -af >/dev/null 2>&1 || true + ;; + *) + echo "ERROR: unknown cleanup tier '$TIER' (expected routine|pressure|critical)" >&2 + exit 2 + ;; +esac + +reclaim_expired_leases diff --git a/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 b/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 index a8d728d..ec2c71d 100644 --- a/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 +++ b/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2 @@ -265,54 +265,56 @@ except Exception: {% endif %} fi -# 3. Check disk space — prune aggressively if below threshold +# 3. Check disk space — scoped tiered cleanup via the shared cleanup script. +# Implements: REQ-2/REQ-3 (GRM-173) — honors org.oblachno.lease-until leases, +# keep-images, and the GITEA-ACTIONS-TASK exclusion; never removes running +# containers. No unfiltered prune remains (the old `system prune -af +# --volumes` could wipe a job's freshly created volumes mid-run). disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}') +ADMISSION_MARKER="{{ gitea_runner_config_dir }}/disk-admission-block" +CLEANUP_SCRIPT="{{ gitea_runner_cleanup_script_path }}" + if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_critical }} ]]; then - echo "CRITICAL: Disk usage at ${disk_pct}% (>= {{ gitea_runner_healthcheck_disk_critical }}%), full prune" - # Critical level: remove ALL stopped containers (no age filter) and ALL - # unused images/volumes. The until=1h gentle prune is insufficient here. - # Implements: REQ-1 (GRM-167) — only exited/dead containers are removed. - # Running molecule instances are never killed: RunningFor counts creation - # time, so an adopted stale instance looks old; and a running container's - # writable layer is tiny — images/volumes are what actually fills the disk. - docker ps -a --filter "status=exited" --filter "status=dead" \ - --format '{% raw %}{{.ID}} {{.Names}}{% endraw %}' 2>/dev/null \ - | grep -v 'GITEA-ACTIONS-TASK' \ - | awk '{print $1}' \ - | xargs -r docker rm -f 2>/dev/null || true - docker system prune -af --volumes || true - docker network prune -f || true - docker builder prune -af || true + echo "CRITICAL: Disk usage at ${disk_pct}% (>= {{ gitea_runner_healthcheck_disk_critical }}%), critical cleanup" + "$CLEANUP_SCRIPT" --tier critical || true disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}') - echo "INFO: Disk usage after full prune: ${disk_pct}%" + echo "INFO: Disk usage after critical cleanup: ${disk_pct}%" + + # Implements: REQ-4 — stop admitting new jobs while critically full, + # but only when no CI job is in flight (stopping the runner service + # mid-job would kill it). A later healthcheck resumes the service once + # disk drops below the warn threshold. +{% if gitea_runner_disk_admission_enabled %} + in_flight=$(timeout 15 docker ps --filter "name=GITEA-ACTIONS-TASK" \ + --format '{% raw %}{{.ID}}{% endraw %}' 2>/dev/null | wc -l || echo 0) + if [[ "$in_flight" -eq 0 ]] \ + && systemctl --user is-active --quiet gitea-runner.service; then + echo "ADMISSION: disk critical, no jobs in flight — stopping runner service" + date +%s > "$ADMISSION_MARKER" 2>/dev/null || true + systemctl --user stop gitea-runner.service || true + elif [[ "$in_flight" -gt 0 ]]; then + echo "ADMISSION: disk critical but ${in_flight} job(s) in flight — runner left running" + fi +{% endif %} elif [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_threshold }} ]]; then - echo "WARN: Disk usage at ${disk_pct}%, pruning runner resources (until=1h)" - # Force-remove stale stopped containers older than 1 hour. - # Implements: REQ-1 (GRM-167) — only exited/dead containers are removed. - # A running molecule instance must never be janitor-killed: RunningFor - # measures creation time, so a stale instance restarted by an active run - # looks ">1h old" and would die mid-converge ("No such container", - # infra nightly run 5710). Running leftovers are reused or destroyed by - # the next molecule create/destroy cycle. - # Exclude CI job containers (name starts with GITEA-ACTIONS-TASK). - docker ps -a --filter "status=exited" --filter "status=dead" \ - --format '{% raw %}{{.ID}} {{.Names}} {{.RunningFor}}{% endraw %}' 2>/dev/null \ - | grep -v 'GITEA-ACTIONS-TASK' \ - | grep -E '(hour|day|week|month|year)s? ago' \ - | awk '{print $1}' \ - | xargs -r docker rm -f 2>/dev/null || true - # Prune images and containers older than 1h (until filter is NOT - # supported with --volumes, so prune volumes separately without a filter). - docker image prune -af --filter "until=1h" 2>/dev/null || true - docker container prune -f --filter "until=1h" 2>/dev/null || true - docker volume prune -f 2>/dev/null || true - # Prune networks older than 1 hour to avoid removing networks that - # molecule tests are actively creating (e.g. 'traefik' network created - # during molecule create phase before containers are attached). - docker network prune -f --filter "until=1h" || true + echo "WARN: Disk usage at ${disk_pct}%, pressure cleanup (until=1h)" + "$CLEANUP_SCRIPT" --tier pressure || true disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}') - echo "INFO: Disk usage after prune: ${disk_pct}%" + echo "INFO: Disk usage after cleanup: ${disk_pct}%" fi +# Implements: REQ-4 — resume admission once pressure has cleared. +{% if gitea_runner_disk_admission_enabled %} +if [[ -f "$ADMISSION_MARKER" ]]; then + if [[ "$disk_pct" -lt {{ gitea_runner_healthcheck_disk_threshold }} ]]; then + echo "ADMISSION: disk recovered to ${disk_pct}% — resuming runner service" + rm -f "$ADMISSION_MARKER" 2>/dev/null || true + systemctl --user start gitea-runner.service || true + else + echo "ADMISSION: still blocked (disk ${disk_pct}%), runner stays stopped" + fi +fi +{% endif %} + echo "OK: runner healthy, disk at ${disk_pct}%" exit 0 diff --git a/docs/specs/GRM-173-skills-historical.md b/docs/specs/GRM-173-skills-historical.md new file mode 100644 index 0000000..ad59558 --- /dev/null +++ b/docs/specs/GRM-173-skills-historical.md @@ -0,0 +1,46 @@ +# GRM-173: Add dependency-graph, deployment-coordination, and skill-creation skills + +## Problem +Agents working across the oblachno ecosystem lack shared, written context +for three recurring struggles: (1) knowing which repo produces what and +the correct order for cross-repo changes, (2) coordinating grm releases +with the downstream infra dependency PR, and (3) creating and validating +new Devin skills consistently. Without these skills, agents repeatedly +make mistakes such as deploying infra before the grm dependency PR is +merged, or writing skills that fail the validator. + +## Approach +Add three skill files under `.devin/skills/`. Two are shared skills +(`dependency-graph`, `skill-creation`) that must be identical across +repos; one is grm-specific (`deployment-coordination`). All three +follow the standard skill structure (H1 title, When to Invoke, +Prerequisites, core content) and reference real make targets, file +paths, and API endpoints. + +REQ-1: Add `.devin/skills/dependency-graph/SKILL.md` — shared skill mapping the oblachno ecosystem (repos, produces/consumers, dependency chain, correct change order, state verification) +REQ-2: Add `.devin/skills/deployment-coordination/SKILL.md` — grm-specific skill covering release flow, downstream consumer, coordinating a grm change, and common mistakes +REQ-3: Add `.devin/skills/skill-creation/SKILL.md` — shared skill for creating, validating, and maintaining skills (structure, quality standards, scope rules, automated validation, checklist) + +## Files Affected +- `.devin/skills/dependency-graph/SKILL.md` (new) +- `.devin/skills/deployment-coordination/SKILL.md` (new) +- `.devin/skills/skill-creation/SKILL.md` (new) +- `docs/specs/GRM-173.md` (new) + +## Test Plan +- Verify all three SKILL.md files follow the required structure (H1, When to Invoke, Prerequisites) +- Verify referenced make targets and file paths are accurate +- Run `make pytest-cov` to confirm no test regressions (skills are docs-only, no code changes) +- Confirm shared skills (`dependency-graph`, `skill-creation`) are ready for cross-repo sync + +## Deploy Plan +- Merge to master via auto-merge workflow +- No runtime changes; documentation-only (`.devin/**` is infrastructure path, no release triggered) + +## Rollback Plan +- Revert the merge commit; skill files are removed, no functional impact + +## Acceptance Criteria +- [x] REQ-1: `.devin/skills/dependency-graph/SKILL.md` exists with ecosystem map, dependency chain, correct change order, and state verification sections +- [x] REQ-2: `.devin/skills/deployment-coordination/SKILL.md` exists with release flow, downstream consumer table, coordination steps, and common mistakes +- [x] REQ-3: `.devin/skills/skill-creation/SKILL.md` exists with skill structure template, quality standards, scope rules, automated validation, and creation checklist diff --git a/docs/specs/GRM-173.md b/docs/specs/GRM-173.md index ad59558..49febce 100644 --- a/docs/specs/GRM-173.md +++ b/docs/specs/GRM-173.md @@ -1,46 +1,88 @@ -# GRM-173: Add dependency-graph, deployment-coordination, and skill-creation skills +# GRM-173: Runner resource leases and scoped disk cleanup ## Problem -Agents working across the oblachno ecosystem lack shared, written context -for three recurring struggles: (1) knowing which repo produces what and -the correct order for cross-repo changes, (2) coordinating grm releases -with the downstream infra dependency PR, and (3) creating and validating -new Devin skills consistently. Without these skills, agents repeatedly -make mistakes such as deploying infra before the grm dependency PR is -merged, or writing skills that fail the validator. + +Runner hosts protect in-flight work only via name-prefix and age heuristics: + +- The healthcheck critical tier runs unfiltered `docker system prune -af --volumes` + and `docker volume prune -f` — a job's momentarily unused volume/network can be + wiped mid-run, and warm base images are destroyed exactly when needed most. +- Molecule containers owned by a live job are protected only by the + `GITEA-ACTIONS-TASK` naming convention, not by an ownership claim. +- Cleanup logic is duplicated between `docker-prune.service` and the healthcheck. +- No admission control: under disk pressure the runner keeps accepting jobs + while cleanup races in-flight work. +- `capacity` is never declared; nothing prevents installing on a production host. ## Approach -Add three skill files under `.devin/skills/`. Two are shared skills -(`dependency-graph`, `skill-creation`) that must be identical across -repos; one is grm-specific (`deployment-coordination`). All three -follow the standard skill structure (H1 title, When to Invoke, -Prerequisites, core content) and reference real make targets, file -paths, and API endpoints. -REQ-1: Add `.devin/skills/dependency-graph/SKILL.md` — shared skill mapping the oblachno ecosystem (repos, produces/consumers, dependency chain, correct change order, state verification) -REQ-2: Add `.devin/skills/deployment-coordination/SKILL.md` — grm-specific skill covering release flow, downstream consumer, coordinating a grm change, and common mistakes -REQ-3: Add `.devin/skills/skill-creation/SKILL.md` — shared skill for creating, validating, and maintaining skills (structure, quality standards, scope rules, automated validation, checklist) +REQ-1: Define an ownership-lease label contract. Producers tag containers, +images, volumes and networks with `org.oblachno.lease-until` (epoch seconds) +and `org.oblachno.owner` (free-form run/job id). All cleanup paths must never +remove an object whose `lease-until` is in the future; expired leases are +reclaimable. Existing `GITEA-ACTIONS-TASK` name-prefix and `status=exited` +guards are retained for unlabeled objects. -## Files Affected -- `.devin/skills/dependency-graph/SKILL.md` (new) -- `.devin/skills/deployment-coordination/SKILL.md` (new) -- `.devin/skills/skill-creation/SKILL.md` (new) -- `docs/specs/GRM-173.md` (new) +REQ-2: Introduce a single shared cleanup script (`runner-cleanup.sh`, +templated next to the healthcheck script) invoked with +`--tier routine|pressure|critical`, replacing all inline prune logic in +`docker-prune.service` and the healthcheck. Every prune is scoped (leases, +`until=` where supported); images matching `gitea_runner_keep_images` are +never removed, so warm base layers survive critical pressure. + +REQ-3: Watermark-tiered behavior: `routine` (timer) prunes aged resources; +`pressure` (disk >= warn) prunes unowned resources older than 1h; `critical` +(disk >= critical) drops age limits but still honors leases, keep-images, and +never removes running or `GITEA-ACTIONS-TASK` containers. + +REQ-4: Admission control under disk pressure. When disk is >= critical and no +`GITEA-ACTIONS-TASK` container is running, the healthcheck writes a marker file +and stops `gitea-runner.service` (the runner stops fetching jobs). A later +healthcheck restarts it once disk drops below warn. In-flight jobs are never +killed. Controlled by `gitea_runner_disk_admission_enabled`. + +REQ-5: Declare physical-host capacity explicitly: `runner.capacity: +{{ gitea_runner_capacity }}` in the act_runner config (default 1 = upstream). + +REQ-6: Production-host exclusion. The role fails early when the target carries +the marker file `/etc/oblachno/production-host` or +`gitea_runner_on_production_host` is true, unless +`gitea_runner_allow_production_host` overrides. Infra-side marker provisioning +is a follow-up task. + +Historical spec for the colliding task ID: +[GRM-173-skills-historical](GRM-173-skills-historical.md). ## Test Plan -- Verify all three SKILL.md files follow the required structure (H1, When to Invoke, Prerequisites) -- Verify referenced make targets and file paths are accurate -- Run `make pytest-cov` to confirm no test regressions (skills are docs-only, no code changes) -- Confirm shared skills (`dependency-graph`, `skill-creation`) are ready for cross-repo sync + +- `template-content` molecule scenario: prune service calls `runner-cleanup.sh`; + lease filters, keep-images and `capacity:` render correctly. +- `default` scenario: cleanup script installed and executable. +- `bash -n` on rendered templates; `make lint-all`, `make pytest-cov`, fast + molecule for the changed role. ## Deploy Plan -- Merge to master via auto-merge workflow -- No runtime changes; documentation-only (`.devin/**` is infrastructure path, no release triggered) + +- Merge via auto-merge; post-merge publishes the package and auto-creates the + infra dependency-bump PR. Runner hosts pick up the change on the next + `grm install`/update run — no manual host action. +- Producer-side lease emission (molecule/CI jobs) is a separate devx change; + until then the guards degrade to the existing name-prefix/age behavior. ## Rollback Plan -- Revert the merge commit; skill files are removed, no functional impact + +- Revert the merge commit and re-run `grm install` to redeploy the previous + prune/healthcheck units. No persistent state or migration. ## Acceptance Criteria -- [x] REQ-1: `.devin/skills/dependency-graph/SKILL.md` exists with ecosystem map, dependency chain, correct change order, and state verification sections -- [x] REQ-2: `.devin/skills/deployment-coordination/SKILL.md` exists with release flow, downstream consumer table, coordination steps, and common mistakes -- [x] REQ-3: `.devin/skills/skill-creation/SKILL.md` exists with skill structure template, quality standards, scope rules, automated validation, and creation checklist + +- [x] REQ-1: `org.oblachno.lease-until`/`org.oblachno.owner` labels are honored + by every cleanup path; valid leases are never removed, expired leases are. +- [x] REQ-2: single shared `runner-cleanup.sh` used by prune service and + healthcheck; no unfiltered `system prune --volumes`, `volume prune`, or + `network prune` remains; `gitea_runner_keep_images` never removed. +- [x] REQ-3: three tiers behave as specified (routine/pressure/critical). +- [x] REQ-4: critical pressure with zero in-flight job containers stops + admission via marker + service stop; recovery resumes automatically. +- [x] REQ-5: `runner.capacity` rendered in `config.yaml`. +- [x] REQ-6: role fails on production-marked hosts unless explicitly allowed.