Public Access
feat: sync missing features from v0.49.x line to master
The v0.49.x tag line diverged from origin/master, leaving many features only accessible via tags but not on the master branch. New modules: - ci/cancel_superseded_runs.py — cancel superseded CI runs - ci/check_workflow_artifact_deps.py — validate artifact deps - ci/check_workflow_tofu_init.py — validate tofu init steps - tools/check_alert_rules.py — validate Prometheus alert rules - tools/check_ansible_set_fact_to_json.py — lint set_fact usage - tools/check_docker_init.py — validate Docker init scripts - utils/jinja.py — Jinja2 template utilities - utils/ui.py — UI/console utilities Modified modules: - distribute_molecule.py: add --include-roles/--exclude-roles - utils/api.py: add container.credentials for private registry auth - install_tools.py: retry ansible-galaxy on transient timeouts - setup_image.py: skip dep resolution with --no-deps - cli.py: register new commands - i18n.py: add new translation keys Also removes accidentally committed .vale/styles/Google/ files. Test results: 2195 passed, 100% coverage. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
parent
bd4530094e
commit
5cacbdec09
@@ -1,3 +1,3 @@
|
||||
"""devx — reusable development and CI/CD tools for oblachno-oss projects."""
|
||||
|
||||
__version__ = "0.48.1"
|
||||
__version__ = "0.49.5"
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
"""Cancel superseded CI runs for the same PR.
|
||||
|
||||
When a new push to a PR branch triggers a new CI run, any in-flight
|
||||
runs for the same PR are wasting runner time. This script cancels
|
||||
all but the latest running CI run for each PR branch.
|
||||
|
||||
Uses the Gitea Actions API:
|
||||
GET /repos/{owner}/{repo}/actions/runs?status=in_progress&event=pull_request
|
||||
POST /repos/{owner}/{repo}/actions/runs/{run_id}/cancel
|
||||
|
||||
Usage::
|
||||
|
||||
# CI (cancels superseded runs for the current PR):
|
||||
python -m devx.ci.cancel_superseded_runs \\
|
||||
--repo "$REPOSITORY" \\
|
||||
--current-run-id "$GITHUB_RUN_ID" \\
|
||||
--head-branch "$HEAD_REF"
|
||||
|
||||
# Dry-run (lists what would be cancelled without cancelling):
|
||||
python -m devx.ci.cancel_superseded_runs \\
|
||||
--repo "$REPOSITORY" \\
|
||||
--current-run-id "$GITHUB_RUN_ID" \\
|
||||
--head-branch "$HEAD_REF" \\
|
||||
--dry-run
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
_HTTP_NO_CONTENT = 204
|
||||
_HTTP_NOT_FOUND = 404
|
||||
_HTTP_BAD_REQUEST = 400
|
||||
_PAGE_SIZE = 50
|
||||
|
||||
|
||||
def _log(msg: str) -> None:
|
||||
"""Log to stderr."""
|
||||
print(f"[cancel-superseded] {msg}", file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
def _api_request(
|
||||
method: str,
|
||||
path: str,
|
||||
token: str,
|
||||
base_url: str,
|
||||
body: dict | None = None,
|
||||
) -> dict | list:
|
||||
"""Make a Gitea API request."""
|
||||
url = f"{base_url}/api/v1{path}"
|
||||
headers = {
|
||||
"Authorization": f"token {token}",
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json",
|
||||
}
|
||||
data = json.dumps(body).encode() if body else None
|
||||
req = urllib.request.Request(url, data=data, headers=headers, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp: # nosec B310 — authenticated API request to known Gitea instance
|
||||
if resp.status == _HTTP_NO_CONTENT:
|
||||
return {}
|
||||
return json.loads(resp.read().decode())
|
||||
except urllib.error.HTTPError as e:
|
||||
_log(f"API error {e.code} on {method} {path}: {e.read().decode()[:200]}")
|
||||
raise
|
||||
except urllib.error.URLError as e:
|
||||
_log(f"URL error on {method} {path}: {e}")
|
||||
raise
|
||||
|
||||
|
||||
def list_running_runs(repo: str, token: str, base_url: str) -> list[dict]:
|
||||
"""List all running CI runs for pull_request events."""
|
||||
runs: list[dict] = []
|
||||
page = 1
|
||||
while True:
|
||||
result = _api_request(
|
||||
"GET",
|
||||
f"/repos/{repo}/actions/runs?status=in_progress&event=pull_request&page={page}&limit=50",
|
||||
token,
|
||||
base_url,
|
||||
)
|
||||
# Gitea returns {"workflow_runs": [...], "total_count": N}
|
||||
page_runs = result["workflow_runs"] if isinstance(result, dict) else result
|
||||
if not page_runs:
|
||||
break
|
||||
runs.extend(page_runs)
|
||||
if len(page_runs) < _PAGE_SIZE:
|
||||
break
|
||||
page += 1
|
||||
return runs
|
||||
|
||||
|
||||
def cancel_run(repo: str, run_id: int, token: str, base_url: str) -> bool:
|
||||
"""Cancel a CI run. Returns True on success."""
|
||||
try:
|
||||
_api_request(
|
||||
"POST",
|
||||
f"/repos/{repo}/actions/runs/{run_id}/cancel",
|
||||
token,
|
||||
base_url,
|
||||
)
|
||||
except (urllib.error.HTTPError, urllib.error.URLError):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Cancel superseded CI runs for the same PR.")
|
||||
parser.add_argument("--repo", required=True, help="owner/repo")
|
||||
parser.add_argument("--current-run-id", required=True, help="Current run ID (not cancelled)")
|
||||
parser.add_argument("--head-branch", required=True, help="PR head branch name")
|
||||
parser.add_argument("--dry-run", action="store_true", help="List without cancelling")
|
||||
parser.add_argument(
|
||||
"--base-url",
|
||||
default=os.environ.get("GITEA_API_URL", "https://git.oblachno.oblachno.fyi"),
|
||||
help="Gitea base URL",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
token = os.environ.get("CI_GITEA_API_TOKEN") or os.environ.get("CI_GITEA_TOKEN")
|
||||
if not token:
|
||||
_log("No CI_GITEA_API_TOKEN or CI_GITEA_TOKEN set — skipping")
|
||||
return 0
|
||||
|
||||
current_run_id = int(args.current_run_id)
|
||||
|
||||
_log(f"Listing running PR runs for {args.repo}...")
|
||||
try:
|
||||
runs = list_running_runs(args.repo, token, args.base_url)
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code in (_HTTP_NOT_FOUND, _HTTP_BAD_REQUEST):
|
||||
_log(
|
||||
f"Actions runs API not usable (HTTP {e.code}) — "
|
||||
f"Gitea {args.base_url} may not support this endpoint or status filter. "
|
||||
f"Skipping cancel-superseded (non-fatal)."
|
||||
)
|
||||
return 0
|
||||
raise
|
||||
_log(f"Found {len(runs)} running PR runs")
|
||||
|
||||
# Group by head_branch — only cancel runs for the SAME branch
|
||||
# that are older than the current run
|
||||
same_branch_runs = [
|
||||
r
|
||||
for r in runs
|
||||
if r.get("head_branch") == args.head_branch
|
||||
and int(r.get("id", 0)) != current_run_id
|
||||
and int(r.get("id", 0)) < current_run_id
|
||||
]
|
||||
|
||||
if not same_branch_runs:
|
||||
_log(f"No superseded runs for branch {args.head_branch}")
|
||||
return 0
|
||||
|
||||
_log(f"Found {len(same_branch_runs)} superseded run(s) for branch {args.head_branch}:")
|
||||
for r in same_branch_runs:
|
||||
run_id = r.get("id")
|
||||
created = r.get("created_at", "?")
|
||||
_log(f" Run #{run_id} (created: {created})")
|
||||
|
||||
if args.dry_run:
|
||||
_log("[dry-run] Would cancel the above runs")
|
||||
return 0
|
||||
|
||||
cancelled = 0
|
||||
for r in same_branch_runs:
|
||||
run_id = int(r["id"])
|
||||
_log(f"Cancelling run #{run_id}...")
|
||||
if cancel_run(args.repo, run_id, token, args.base_url):
|
||||
cancelled += 1
|
||||
_log(f" Cancelled run #{run_id}")
|
||||
else:
|
||||
_log(f" Failed to cancel run #{run_id}")
|
||||
|
||||
_log(f"Cancelled {cancelled}/{len(same_branch_runs)} superseded runs")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,163 @@
|
||||
"""Check that workflow jobs downloading artifacts depend on the uploading job.
|
||||
|
||||
This prevents the class of bug where a job downloads an artifact produced by
|
||||
another job but does not declare that job in its ``needs`` list. When both
|
||||
jobs run in parallel, the download fails because the artifact hasn't been
|
||||
uploaded yet.
|
||||
|
||||
The check scans all workflow YAML files for:
|
||||
- ``gitea-upload-artifact`` / ``actions/upload-artifact`` steps
|
||||
- ``gitea-download-artifact`` / ``actions/download-artifact`` steps
|
||||
|
||||
For each download, it finds the job(s) that upload an artifact with a
|
||||
matching name and verifies that at least one uploading job is in the
|
||||
downloading job's ``needs`` list.
|
||||
|
||||
Artifact names with ``${{ ... }}`` expressions are matched literally
|
||||
(both sides use the same expression, so they resolve to the same value
|
||||
at runtime).
|
||||
|
||||
Usage::
|
||||
|
||||
python -m devx.ci.check_workflow_artifact_deps
|
||||
python -m devx.ci.check_workflow_artifact_deps --workflow .gitea/workflows/ci.yml
|
||||
|
||||
Exit code 0 if all artifact dependencies are satisfied, 1 otherwise.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
import yaml
|
||||
|
||||
REPO_ROOT = Path.cwd()
|
||||
WORKFLOWS_DIR = REPO_ROOT / ".gitea" / "workflows"
|
||||
|
||||
UPLOAD_ACTIONS = ("upload-artifact",)
|
||||
DOWNLOAD_ACTIONS = ("download-artifact",)
|
||||
|
||||
|
||||
def _is_artifact_action(uses: str, action_types: tuple[str, ...]) -> bool:
|
||||
"""Check if a step's ``uses`` field references an artifact action."""
|
||||
if not uses:
|
||||
return False
|
||||
uses_lower = uses.lower()
|
||||
return any(action in uses_lower for action in action_types)
|
||||
|
||||
|
||||
def _extract_artifact_info(workflow: dict) -> tuple[dict[str, list[str]], list[tuple[str, str, str]]]:
|
||||
"""Extract artifact upload and download info from a workflow.
|
||||
|
||||
Returns:
|
||||
uploads: Mapping of artifact_name → list of job names that upload it.
|
||||
downloads: List of (job_name, artifact_name, step_name) tuples.
|
||||
"""
|
||||
uploads: dict[str, list[str]] = {}
|
||||
downloads: list[tuple[str, str, str]] = []
|
||||
|
||||
jobs = workflow.get("jobs", {})
|
||||
for job_name, job_def in jobs.items():
|
||||
for step in job_def.get("steps", []):
|
||||
uses = step.get("uses", "")
|
||||
with_data = step.get("with", {})
|
||||
artifact_name = with_data.get("name", "")
|
||||
step_name = step.get("name", "")
|
||||
|
||||
if _is_artifact_action(uses, UPLOAD_ACTIONS):
|
||||
if artifact_name:
|
||||
uploads.setdefault(artifact_name, []).append(job_name)
|
||||
elif _is_artifact_action(uses, DOWNLOAD_ACTIONS) and artifact_name:
|
||||
downloads.append((job_name, artifact_name, step_name))
|
||||
|
||||
return uploads, downloads
|
||||
|
||||
|
||||
def _check_workflow(filepath: Path) -> list[str]:
|
||||
"""Check a single workflow file for missing artifact dependencies.
|
||||
|
||||
Returns a list of error messages (empty if all OK).
|
||||
"""
|
||||
errors: list[str] = []
|
||||
content = filepath.read_text(encoding="utf-8")
|
||||
try:
|
||||
workflow = yaml.safe_load(content)
|
||||
except yaml.YAMLError as exc:
|
||||
return [f"{filepath}: cannot parse YAML: {exc}"]
|
||||
|
||||
if not isinstance(workflow, dict):
|
||||
return [f"{filepath}: not a valid workflow (expected dict)"]
|
||||
|
||||
uploads, downloads = _extract_artifact_info(workflow)
|
||||
jobs = workflow.get("jobs", {})
|
||||
|
||||
for dl_job, artifact_name, step_name in downloads:
|
||||
uploading_jobs = uploads.get(artifact_name, [])
|
||||
if not uploading_jobs:
|
||||
# Artifact not uploaded in this workflow — may come from an
|
||||
# external source (e.g., S3). Skip.
|
||||
continue
|
||||
|
||||
dl_job_def = jobs.get(dl_job, {})
|
||||
needs_raw = dl_job_def.get("needs", [])
|
||||
needs = {needs_raw} if isinstance(needs_raw, str) else set(needs_raw or [])
|
||||
|
||||
# Check if any uploading job is in the download job's needs
|
||||
if not any(uploader in needs for uploader in uploading_jobs):
|
||||
# Check if the download step has continue-on-error: true
|
||||
# (valid guard when the uploading job may be skipped due to
|
||||
# Gitea Actions' needs skip behavior — the download will
|
||||
# fail gracefully if the artifact doesn't exist).
|
||||
dl_steps = dl_job_def.get("steps", [])
|
||||
step_def = next((s for s in dl_steps if s.get("name", "") == step_name), {})
|
||||
if step_def.get("continue-on-error") is True:
|
||||
continue
|
||||
|
||||
uploaders_str = ", ".join(sorted(uploading_jobs))
|
||||
errors.append(
|
||||
f"{filepath.name}::{dl_job}: step '{step_name}' downloads "
|
||||
f"artifact '{artifact_name}' produced by job(s) "
|
||||
f"[{uploaders_str}] but none are in its 'needs' list "
|
||||
f"(current needs: {sorted(needs) or 'none'}). "
|
||||
f"Add the uploading job to 'needs' or guard the download "
|
||||
f"with an if: condition checking the upload job's result."
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option(
|
||||
"--workflow",
|
||||
type=click.Path(exists=True, path_type=Path),
|
||||
help="Check a specific workflow file (default: all in .gitea/workflows/).",
|
||||
)
|
||||
@click.option(
|
||||
"--workflows-dir",
|
||||
type=click.Path(exists=True, path_type=Path),
|
||||
default=None,
|
||||
help="Override the workflows directory (default: .gitea/workflows/).",
|
||||
)
|
||||
def main(workflow: Path | None, workflows_dir: Path | None) -> None:
|
||||
"""Check that artifact download jobs depend on upload jobs."""
|
||||
wdir = workflows_dir or WORKFLOWS_DIR
|
||||
files = [workflow] if workflow else sorted(wdir.glob("*.yml"))
|
||||
|
||||
all_errors: list[str] = []
|
||||
for f in files:
|
||||
errors = _check_workflow(f)
|
||||
all_errors.extend(errors)
|
||||
|
||||
if all_errors:
|
||||
click.echo("[check-workflow-artifact-deps] FAIL: missing artifact dependencies found:")
|
||||
for err in all_errors:
|
||||
click.echo(f" - {err}")
|
||||
sys.exit(1)
|
||||
else:
|
||||
click.echo("[check-workflow-artifact-deps] OK: all artifact downloads have upload jobs in needs.")
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
main()
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Check that workflow jobs using tofu state have a tofu-init step.
|
||||
|
||||
This prevents the class of bug where a job runs ``tofu output`` or calls
|
||||
a script that uses tofu state without first running ``tofu init``,
|
||||
causing "Required plugins are not installed" errors.
|
||||
|
||||
The check scans all workflow YAML files for jobs that:
|
||||
- Call scripts that use ``tofu output`` (configurable via --state-scripts)
|
||||
- Call ``tofu output`` directly
|
||||
- Call ``tofu plan`` or ``tofu apply`` directly
|
||||
|
||||
For each such job, it verifies the same job has a ``tofu-init`` step,
|
||||
either:
|
||||
- Directly via ``tofu init`` in a step's run command
|
||||
- Via ``create_staging_deployment.py --phase tofu-init``
|
||||
- Via ``create_production_deployment.py --phase tofu-init``
|
||||
|
||||
Usage::
|
||||
|
||||
python -m devx.ci.check_workflow_tofu_init
|
||||
python -m devx.ci.check_workflow_tofu_init --workflow .gitea/workflows/deploy.yml
|
||||
|
||||
Exit code 0 if all jobs have tofu-init, 1 otherwise.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
import yaml
|
||||
|
||||
REPO_ROOT = Path.cwd()
|
||||
WORKFLOWS_DIR = REPO_ROOT / ".gitea" / "workflows"
|
||||
|
||||
# Scripts that call `tofu output`, `tofu plan`, or `tofu apply` internally.
|
||||
# If a job calls any of these, it must have a tofu-init step.
|
||||
# NOTE: destroy_orphans.py reads terraform.tfstate directly from disk
|
||||
# (does not invoke `tofu output`), so it does NOT need tofu-init.
|
||||
DEFAULT_TOFU_STATE_SCRIPTS: set[str] = {
|
||||
"preflight_deploy.py",
|
||||
}
|
||||
|
||||
# Commands that directly use tofu state (must be preceded by tofu init).
|
||||
TOFU_STATE_COMMANDS = ("tofu output", "tofu plan", "tofu apply", "tofu show")
|
||||
|
||||
# Commands that initialize tofu (counted as tofu-init steps).
|
||||
TOFU_INIT_COMMANDS = (
|
||||
"tofu init",
|
||||
"--phase tofu-init",
|
||||
"tofu-init",
|
||||
)
|
||||
|
||||
|
||||
def _check_workflow(filepath: Path, state_scripts: set[str]) -> list[str]:
|
||||
"""Check a single workflow file for missing tofu-init steps.
|
||||
|
||||
Returns a list of error messages (empty if all OK).
|
||||
"""
|
||||
errors: list[str] = []
|
||||
content = filepath.read_text(encoding="utf-8")
|
||||
try:
|
||||
workflow = yaml.safe_load(content)
|
||||
except yaml.YAMLError as exc:
|
||||
return [f"{filepath}: cannot parse YAML: {exc}"]
|
||||
|
||||
jobs = workflow.get("jobs", {})
|
||||
for job_name, job_def in jobs.items():
|
||||
steps = job_def.get("steps", [])
|
||||
if not steps:
|
||||
continue
|
||||
|
||||
uses_tofu_state = False
|
||||
has_tofu_init = False
|
||||
|
||||
for step in steps:
|
||||
run_cmd = step.get("run", "")
|
||||
if not run_cmd:
|
||||
continue
|
||||
# Check if this step uses tofu state
|
||||
for script in state_scripts:
|
||||
if script in run_cmd:
|
||||
uses_tofu_state = True
|
||||
for cmd in TOFU_STATE_COMMANDS:
|
||||
if cmd in run_cmd:
|
||||
uses_tofu_state = True
|
||||
# Check if this step initializes tofu
|
||||
for cmd in TOFU_INIT_COMMANDS:
|
||||
if cmd in run_cmd:
|
||||
has_tofu_init = True
|
||||
|
||||
if uses_tofu_state and not has_tofu_init:
|
||||
errors.append(
|
||||
f"{filepath.name}::{job_name}: uses tofu state "
|
||||
f"(tofu output/plan/apply or {state_scripts}) "
|
||||
f"but has no tofu-init step. Add a step running "
|
||||
f"'create_*_deployment.py --phase tofu-init' before "
|
||||
f"the first tofu state access."
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option(
|
||||
"--workflow",
|
||||
type=click.Path(exists=True, path_type=Path),
|
||||
help="Check a specific workflow file (default: all in .gitea/workflows/).",
|
||||
)
|
||||
@click.option(
|
||||
"--workflows-dir",
|
||||
type=click.Path(exists=True, path_type=Path),
|
||||
default=None,
|
||||
help="Override the workflows directory (default: .gitea/workflows/).",
|
||||
)
|
||||
@click.option(
|
||||
"--state-script",
|
||||
"state_scripts",
|
||||
multiple=True,
|
||||
default=None,
|
||||
help="Add a script name that uses tofu state (can be repeated). Overrides the default list if any are specified.",
|
||||
)
|
||||
def main(workflow: Path | None, workflows_dir: Path | None, state_scripts: tuple[str, ...]) -> None:
|
||||
"""Check that workflow jobs using tofu state have a tofu-init step."""
|
||||
scripts = set(state_scripts) if state_scripts else DEFAULT_TOFU_STATE_SCRIPTS
|
||||
wdir = workflows_dir or WORKFLOWS_DIR
|
||||
files = [workflow] if workflow else sorted(wdir.glob("*.yml"))
|
||||
|
||||
all_errors: list[str] = []
|
||||
for f in files:
|
||||
errors = _check_workflow(f, scripts)
|
||||
all_errors.extend(errors)
|
||||
|
||||
if all_errors:
|
||||
click.echo("[check-workflow-tofu-init] FAIL: missing tofu-init steps found:")
|
||||
for err in all_errors:
|
||||
click.echo(f" - {err}")
|
||||
sys.exit(1)
|
||||
else:
|
||||
click.echo("[check-workflow-tofu-init] OK: all tofu-state jobs have tofu-init.")
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
main()
|
||||
@@ -52,6 +52,7 @@ REQUIRED_SCRIPTS = [
|
||||
"detect_release_commit.py",
|
||||
"push_badges.py",
|
||||
"distribute_molecule.py",
|
||||
"molecule_ci_guard.py",
|
||||
"validate_commit_msg.py",
|
||||
]
|
||||
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run integration tests with cross-runner failure detection.
|
||||
|
||||
Wraps ``pytest`` with Gitea API polling. If any other integration-tests
|
||||
matrix runner reports failure, the current pytest subprocess is killed
|
||||
and this runner exits early with code 1.
|
||||
Wraps ``pytest`` with the same Gitea API polling mechanism used by
|
||||
``molecule_ci_guard``. If any other integration-tests matrix runner
|
||||
reports failure, the current pytest subprocess is killed and this runner
|
||||
exits early with code 1.
|
||||
|
||||
Usage::
|
||||
|
||||
@@ -34,62 +35,17 @@ import threading
|
||||
import time
|
||||
|
||||
import click
|
||||
import requests
|
||||
|
||||
from devx.config import REPO_NAME, REPO_OWNER
|
||||
from devx.i18n import _
|
||||
from devx.molecule.molecule_ci_guard import (
|
||||
poll_for_other_failures,
|
||||
)
|
||||
from devx.tokens import get_ci_token
|
||||
|
||||
POLL_INTERVAL = 10
|
||||
|
||||
|
||||
def get_running_jobs(gitea_url: str, owner: str, repo: str, token: str, run_id: int) -> list[dict]:
|
||||
"""Return jobs for the given workflow run."""
|
||||
url = f"{gitea_url}/api/v1/repos/{owner}/{repo}/actions/runs/{run_id}/jobs"
|
||||
headers = {"Authorization": f"token {token}"}
|
||||
response = requests.get(url, headers=headers, timeout=30)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return data.get("jobs", [])
|
||||
|
||||
|
||||
def any_other_runner_failed(jobs: list[dict], current_job_name: str, current_index: int) -> bool:
|
||||
"""Return True if any other matrix job has failed."""
|
||||
for job in jobs:
|
||||
name = job.get("name", "")
|
||||
if not name.startswith(current_job_name):
|
||||
continue
|
||||
if name == f"{current_job_name} ({current_index})" or name == current_job_name:
|
||||
continue
|
||||
if job.get("conclusion") == "failure":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def poll_for_other_failures(
|
||||
gitea_url: str,
|
||||
owner: str,
|
||||
repo: str,
|
||||
token: str,
|
||||
run_id: int,
|
||||
job_name: str,
|
||||
current_index: int,
|
||||
stop_event: threading.Event,
|
||||
failed_event: threading.Event,
|
||||
) -> None:
|
||||
"""Background thread: poll API and signal if another runner fails."""
|
||||
while not stop_event.is_set():
|
||||
try:
|
||||
jobs = get_running_jobs(gitea_url, owner, repo, token, run_id)
|
||||
if any_other_runner_failed(jobs, job_name, current_index):
|
||||
click.echo(_("Another runner failed. Stopping this runner early."))
|
||||
failed_event.set()
|
||||
return
|
||||
except requests.RequestException as exc:
|
||||
click.echo(_("API poll warning: {exc}", exc=exc))
|
||||
stop_event.wait(POLL_INTERVAL)
|
||||
|
||||
|
||||
@click.command(context_settings={"ignore_unknown_options": True})
|
||||
@click.argument("pytest_args", nargs=-1, type=click.UNPROCESSED, required=True)
|
||||
def cli(pytest_args: tuple[str, ...]) -> None:
|
||||
|
||||
@@ -172,6 +172,27 @@ def ci_integration_guard(args: tuple[str, ...]) -> None:
|
||||
_run_module("devx.ci.integration_guard", list(args))
|
||||
|
||||
|
||||
@ci.command("cancel-superseded-runs")
|
||||
@click.argument("args", nargs=-1)
|
||||
def ci_cancel_superseded_runs(args: tuple[str, ...]) -> None:
|
||||
"""Cancel superseded CI runs for the same PR branch."""
|
||||
_run_module("devx.ci.cancel_superseded_runs", list(args))
|
||||
|
||||
|
||||
@ci.command("check-workflow-artifact-deps")
|
||||
@click.argument("args", nargs=-1)
|
||||
def ci_check_workflow_artifact_deps(args: tuple[str, ...]) -> None:
|
||||
"""Check that artifact download jobs depend on upload jobs."""
|
||||
_run_module("devx.ci.check_workflow_artifact_deps", list(args))
|
||||
|
||||
|
||||
@ci.command("check-workflow-tofu-init")
|
||||
@click.argument("args", nargs=-1)
|
||||
def ci_check_workflow_tofu_init(args: tuple[str, ...]) -> None:
|
||||
"""Check that workflow jobs using tofu state have a tofu-init step."""
|
||||
_run_module("devx.ci.check_workflow_tofu_init", list(args))
|
||||
|
||||
|
||||
@cli.group()
|
||||
def tools() -> None:
|
||||
"""Development tool commands."""
|
||||
@@ -240,6 +261,27 @@ def tools_pr_rebase(args: tuple[str, ...]) -> None:
|
||||
_run_module("devx.tools.pr_rebase", list(args))
|
||||
|
||||
|
||||
@tools.command("check-docker-init")
|
||||
@click.argument("args", nargs=-1)
|
||||
def tools_check_docker_init(args: tuple[str, ...]) -> None:
|
||||
"""Check that Docker Compose services with healthchecks have init: true."""
|
||||
_run_module("devx.tools.check_docker_init", list(args))
|
||||
|
||||
|
||||
@tools.command("check-ansible-set-fact-to-json")
|
||||
@click.argument("args", nargs=-1)
|
||||
def tools_check_ansible_set_fact_to_json(args: tuple[str, ...]) -> None:
|
||||
"""Check that Ansible set_fact tasks don't misuse to_json."""
|
||||
_run_module("devx.tools.check_ansible_set_fact_to_json", list(args))
|
||||
|
||||
|
||||
@tools.command("check-alert-rules")
|
||||
@click.argument("args", nargs=-1)
|
||||
def tools_check_alert_rules(args: tuple[str, ...]) -> None:
|
||||
"""Validate rendered Prometheus alert rules with promtool."""
|
||||
_run_module("devx.tools.check_alert_rules", list(args))
|
||||
|
||||
|
||||
@cli.group()
|
||||
def molecule() -> None:
|
||||
"""Molecule testing commands (requires devx[molecule])."""
|
||||
@@ -259,6 +301,13 @@ def molecule_discover_runners(args: tuple[str, ...]) -> None:
|
||||
_run_module("devx.molecule.discover_runners", list(args))
|
||||
|
||||
|
||||
@molecule.command("guard")
|
||||
@click.argument("args", nargs=-1)
|
||||
def molecule_guard(args: tuple[str, ...]) -> None:
|
||||
"""Run molecule tests sequentially with CI failure polling."""
|
||||
_run_module("devx.molecule.molecule_ci_guard", list(args))
|
||||
|
||||
|
||||
@molecule.command("all")
|
||||
@click.argument("args", nargs=-1)
|
||||
def molecule_all(args: tuple[str, ...]) -> None:
|
||||
|
||||
+34
-5
@@ -6,6 +6,10 @@ Supported: en, bg, de, ru, zh, pl.
|
||||
Projects can extend translations by setting DEVX_TRANSLATIONS_PATH to a
|
||||
JSON file with additional keys. Keys from the project's file are merged
|
||||
on top of devx's built-in translations.
|
||||
|
||||
Projects that use different env var names (e.g. GRM_LANG instead of
|
||||
DEVX_LANG) can call :func:`configure_i18n` at import time to override
|
||||
the defaults.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -14,15 +18,39 @@ import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
# Configurable env var names — projects can override via configure_i18n()
|
||||
_lang_env_var = "DEVX_LANG"
|
||||
_translations_path_env_var = "DEVX_TRANSLATIONS_PATH"
|
||||
|
||||
# Load built-in translations
|
||||
_BUILTIN_TRANSLATIONS: dict[str, dict[str, str]] = json.loads(
|
||||
(Path(__file__).parent / "translations.json").read_text(encoding="utf-8")
|
||||
)
|
||||
|
||||
|
||||
def configure_i18n(
|
||||
*,
|
||||
lang_env_var: str = "DEVX_LANG",
|
||||
translations_path_env_var: str = "DEVX_TRANSLATIONS_PATH",
|
||||
) -> None:
|
||||
"""Override the env var names used for language and translations path.
|
||||
|
||||
This allows downstream projects (e.g. grm) to use their own env var
|
||||
names (e.g. ``GRM_LANG``) while still using devx's i18n system.
|
||||
|
||||
Args:
|
||||
lang_env_var: Environment variable name for language selection.
|
||||
translations_path_env_var: Environment variable name for the
|
||||
path to a JSON file with project-specific translations.
|
||||
"""
|
||||
global _lang_env_var, _translations_path_env_var
|
||||
_lang_env_var = lang_env_var
|
||||
_translations_path_env_var = translations_path_env_var
|
||||
|
||||
|
||||
def _load_project_translations() -> dict[str, dict[str, str]]:
|
||||
"""Load project-specific translations from DEVX_TRANSLATIONS_PATH if set."""
|
||||
path = os.getenv("DEVX_TRANSLATIONS_PATH")
|
||||
"""Load project-specific translations from the configured env var if set."""
|
||||
path = os.getenv(_translations_path_env_var)
|
||||
if not path:
|
||||
return {}
|
||||
p = Path(path)
|
||||
@@ -41,10 +69,11 @@ TRANSLATIONS: dict[str, dict[str, str]] = {**_BUILTIN_TRANSLATIONS, **_load_proj
|
||||
def _(key: str, **kwargs: object) -> str:
|
||||
"""Return a translated string for the given key.
|
||||
|
||||
Translation is opt-in via the ``DEVX_LANG`` environment variable.
|
||||
If unset, English is always returned regardless of system locale.
|
||||
Translation is opt-in via the configured language environment variable
|
||||
(default ``DEVX_LANG``). If unset, English is always returned regardless
|
||||
of system locale.
|
||||
"""
|
||||
lang = os.getenv("DEVX_LANG", "en")
|
||||
lang = os.getenv(_lang_env_var, "en")
|
||||
if lang not in ("en", "bg", "de", "ru", "zh", "pl"):
|
||||
lang = "en"
|
||||
template = TRANSLATIONS.get(key, {}).get(lang, key)
|
||||
|
||||
@@ -104,21 +104,36 @@ def discover_scenarios(root: Path | None = None) -> list[str]:
|
||||
return sorted(scenarios)
|
||||
|
||||
|
||||
def discover_multi_role_scenarios(roles_root: Path | None = None) -> list[tuple[str, str]]:
|
||||
def discover_multi_role_scenarios(
|
||||
roles_root: Path | None = None,
|
||||
include_roles: list[str] | None = None,
|
||||
exclude_roles: list[str] | None = None,
|
||||
) -> list[tuple[str, str]]:
|
||||
"""Discover (role, scenario) pairs across all roles under *roles_root*.
|
||||
|
||||
Scans ``roles_root/*/molecule/*/`` for scenario directories, skipping
|
||||
``common`` and directories starting with ``_``. Returns a sorted list of
|
||||
``(role_name, scenario_name)`` tuples.
|
||||
|
||||
If *include_roles* is given, only roles whose name is in the list are
|
||||
returned. If *exclude_roles* is given, roles whose name is in the list
|
||||
are skipped. Both filters are case-insensitive.
|
||||
"""
|
||||
if roles_root is None:
|
||||
roles_root = DEFAULT_ROLES_ROOT
|
||||
if not roles_root.is_dir():
|
||||
raise click.ClickException(_("Roles directory not found: {path}", path=str(roles_root)))
|
||||
include_set = {r.lower() for r in include_roles} if include_roles else None
|
||||
exclude_set = {r.lower() for r in exclude_roles} if exclude_roles else None
|
||||
pairs: list[tuple[str, str]] = []
|
||||
for role_dir in sorted(roles_root.iterdir()):
|
||||
if not role_dir.is_dir():
|
||||
continue
|
||||
role_name = role_dir.name
|
||||
if include_set is not None and role_name.lower() not in include_set:
|
||||
continue
|
||||
if exclude_set is not None and role_name.lower() in exclude_set:
|
||||
continue
|
||||
mol_dir = role_dir / "molecule"
|
||||
if not mol_dir.is_dir():
|
||||
continue
|
||||
@@ -348,6 +363,24 @@ def _write_github_env(key: str, value: str) -> None:
|
||||
help="JSON file with custom platform list (each entry: name, image, command). "
|
||||
"Overrides the default platform matrix. Useful for projects with custom test images.",
|
||||
)
|
||||
@click.option(
|
||||
"--include-roles",
|
||||
"include_roles",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Comma-separated list of role names to include (multi-role mode only). "
|
||||
"Only scenarios from these roles are distributed. Case-insensitive. "
|
||||
"Example: --include-roles docker_base,crowdsec,disk_cleanup,app_hardening",
|
||||
)
|
||||
@click.option(
|
||||
"--exclude-roles",
|
||||
"exclude_roles",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Comma-separated list of role names to exclude (multi-role mode only). "
|
||||
"Scenarios from these roles are skipped. Case-insensitive. "
|
||||
"Example: --exclude-roles docker_base,crowdsec,disk_cleanup,app_hardening",
|
||||
)
|
||||
def cli(
|
||||
runner_index: int | None,
|
||||
max_runners: int,
|
||||
@@ -358,11 +391,18 @@ def cli(
|
||||
molecule_root: Path | None,
|
||||
roles_root: Path | None,
|
||||
platforms_file: Path | None,
|
||||
include_roles: str | None,
|
||||
exclude_roles: str | None,
|
||||
) -> None:
|
||||
platforms = load_platforms(platforms_file)
|
||||
# Parse role filters
|
||||
include_list = [r.strip() for r in include_roles.split(",")] if include_roles else None
|
||||
exclude_list = [r.strip() for r in exclude_roles.split(",")] if exclude_roles else None
|
||||
# Multi-role mode: discover (role, scenario) pairs across all roles
|
||||
if roles_root is not None:
|
||||
role_scenarios = discover_multi_role_scenarios(roles_root)
|
||||
role_scenarios = discover_multi_role_scenarios(
|
||||
roles_root, include_roles=include_list, exclude_roles=exclude_list
|
||||
)
|
||||
if list_all:
|
||||
for role, scenario in role_scenarios:
|
||||
click.echo(f"{role}|{scenario}")
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Validate Prometheus alert rules with promtool check rules.
|
||||
|
||||
Renders an alert-rules Jinja2 template with test values and validates
|
||||
the output with ``promtool check rules``. Exits 0 if valid, non-zero
|
||||
otherwise. Skips (exits 0) if promtool is not on PATH.
|
||||
|
||||
Usage::
|
||||
|
||||
python -m devx.tools.check_alert_rules \\
|
||||
--template-path ansible/roles/observability/templates \\
|
||||
--template-name alert-rules.yml.j2
|
||||
|
||||
# With extra template variables:
|
||||
python -m devx.tools.check_alert_rules \\
|
||||
--template-path ansible/roles/observability/templates \\
|
||||
--template-name alert-rules.yml.j2 \\
|
||||
--var grafana_base_url=https://grafana.test.example.com
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import subprocess # nosec B404 — used to run promtool, a trusted binary
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
|
||||
from devx.utils.jinja import make_env, render_template
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option(
|
||||
"--template-path",
|
||||
type=click.Path(exists=True, path_type=Path),
|
||||
required=True,
|
||||
help="Path to the directory containing the Jinja2 template.",
|
||||
)
|
||||
@click.option(
|
||||
"--template-name",
|
||||
default="alert-rules.yml.j2",
|
||||
help="Name of the Jinja2 template file to render.",
|
||||
)
|
||||
@click.option(
|
||||
"--var",
|
||||
"template_vars",
|
||||
multiple=True,
|
||||
help="Template variables in key=value format (can be repeated). "
|
||||
"Example: --var grafana_base_url=https://grafana.example.com",
|
||||
)
|
||||
def main(template_path: Path, template_name: str, template_vars: tuple[str, ...]) -> None:
|
||||
"""Validate rendered alert rules with promtool."""
|
||||
if not shutil.which("promtool"):
|
||||
click.echo("promtool not found in PATH — skipping alert rules validation")
|
||||
return
|
||||
|
||||
# Parse template variables
|
||||
kwargs: dict[str, str] = {}
|
||||
for v in template_vars:
|
||||
if "=" in v:
|
||||
key, value = v.split("=", 1)
|
||||
kwargs[key] = value
|
||||
|
||||
env = make_env(str(template_path))
|
||||
output = render_template(env, template_name, **kwargs)
|
||||
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".yml", delete=False) as f:
|
||||
f.write(output)
|
||||
tmp_path = f.name
|
||||
|
||||
click.echo("[check-alert-rules] Validating rendered rules with promtool...")
|
||||
result = subprocess.run( # nosec
|
||||
["promtool", "check", "rules", tmp_path],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
click.echo(result.stdout, nl=False)
|
||||
if result.returncode != 0:
|
||||
click.echo(result.stderr, nl=False, err=True)
|
||||
sys.exit(result.returncode)
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
main()
|
||||
@@ -0,0 +1,196 @@
|
||||
"""Check that Ansible ``set_fact`` tasks don't misuse ``| to_json``.
|
||||
|
||||
This prevents the class of bug where ``set_fact`` tasks use
|
||||
``{{ targets | to_json }}`` to store Python lists, but ``to_json``
|
||||
converts native types to JSON strings. Ansible then stored the result
|
||||
as a string, so iterating over the fact yielded individual characters
|
||||
instead of list items, causing ``object of type 'str' has no attribute
|
||||
'ip'`` errors.
|
||||
|
||||
The check scans all Ansible task files (playbooks and role tasks) for
|
||||
``set_fact`` tasks where any value uses ``| to_json`` or ``| to_nice_json``
|
||||
and flags them as potential bugs.
|
||||
|
||||
``| to_json`` is legitimate in Jinja2 templates (e.g., rendering JSON
|
||||
config files) but almost never correct in ``set_fact`` — the fact should
|
||||
store the native Python type so downstream tasks can iterate/index it.
|
||||
|
||||
Usage::
|
||||
|
||||
python -m devx.tools.check_ansible_set_fact_to_json
|
||||
python -m devx.tools.check_ansible_set_fact_to_json --path ansible/playbooks/deploy.yml
|
||||
|
||||
Exit code 0 if no misuses found, 1 otherwise.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
import yaml
|
||||
|
||||
REPO_ROOT = Path.cwd()
|
||||
DEFAULT_ANSIBLE_DIRS: list[Path] = [
|
||||
REPO_ROOT / "ansible" / "playbooks",
|
||||
REPO_ROOT / "ansible" / "roles",
|
||||
]
|
||||
|
||||
TO_JSON_FILTERS = ("| to_json", "| to_nice_json", "|to_json", "|to_nice_json")
|
||||
|
||||
|
||||
def _find_task_files(base: Path) -> list[Path]:
|
||||
"""Find all YAML task files under a base directory."""
|
||||
if base.is_file() and base.suffix in (".yml", ".yaml"):
|
||||
return [base]
|
||||
if not base.is_dir():
|
||||
return []
|
||||
return sorted(base.rglob("*.yml")) + sorted(base.rglob("*.yaml"))
|
||||
|
||||
|
||||
def _check_file(filepath: Path, repo_root: Path) -> list[str]:
|
||||
"""Check a single YAML file for set_fact + to_json misuse.
|
||||
|
||||
Returns a list of error messages (empty if all OK).
|
||||
"""
|
||||
errors: list[str] = []
|
||||
content = filepath.read_text(encoding="utf-8")
|
||||
|
||||
# Multi-document YAML (--- separators) is common in playbooks
|
||||
try:
|
||||
docs = list(yaml.safe_load_all(content))
|
||||
except yaml.YAMLError as exc:
|
||||
return [f"{filepath}: cannot parse YAML: {exc}"]
|
||||
|
||||
for doc in docs:
|
||||
if isinstance(doc, list):
|
||||
# Could be a playbook (list of plays) or a role tasks file (list of tasks)
|
||||
for item in doc:
|
||||
if isinstance(item, dict):
|
||||
if any(k in item for k in ("tasks", "pre_tasks", "post_tasks", "handlers", "roles")):
|
||||
# It's a play
|
||||
_check_tasks(item, filepath, errors, repo_root)
|
||||
else:
|
||||
# It's a bare task (role tasks file)
|
||||
_check_task(item, filepath, errors, repo_root)
|
||||
block = item.get("block")
|
||||
if isinstance(block, list):
|
||||
_check_task_list(block, filepath, errors, repo_root)
|
||||
elif isinstance(doc, dict):
|
||||
# Role tasks file or single play — _check_tasks handles all task sections
|
||||
_check_tasks(doc, filepath, errors, repo_root)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def _check_tasks(doc: dict, filepath: Path, errors: list[str], repo_root: Path) -> None:
|
||||
"""Check top-level tasks and nested task sections in a playbook doc."""
|
||||
tasks = doc.get("tasks")
|
||||
if isinstance(tasks, list):
|
||||
_check_task_list(tasks, filepath, errors, repo_root)
|
||||
for role_key in ("pre_tasks", "post_tasks", "handlers"):
|
||||
section = doc.get(role_key)
|
||||
if isinstance(section, list):
|
||||
_check_task_list(section, filepath, errors, repo_root)
|
||||
# Check tasks in roles imported via `roles:` key
|
||||
roles = doc.get("roles")
|
||||
if isinstance(roles, list):
|
||||
for role_entry in roles:
|
||||
if isinstance(role_entry, dict):
|
||||
role_tasks = role_entry.get("tasks")
|
||||
if isinstance(role_tasks, list):
|
||||
_check_task_list(role_tasks, filepath, errors, repo_root)
|
||||
|
||||
|
||||
def _check_task_list(tasks: list, filepath: Path, errors: list[str], repo_root: Path) -> None:
|
||||
"""Check a list of task definitions for set_fact + to_json."""
|
||||
for task in tasks:
|
||||
if not isinstance(task, dict):
|
||||
continue
|
||||
_check_task(task, filepath, errors, repo_root)
|
||||
# Check nested block tasks
|
||||
block = task.get("block")
|
||||
if isinstance(block, list):
|
||||
_check_task_list(block, filepath, errors, repo_root)
|
||||
|
||||
|
||||
def _check_task(task: dict, filepath: Path, errors: list[str], repo_root: Path) -> None:
|
||||
"""Check a single task for set_fact + to_json misuse."""
|
||||
# Detect set_fact — could be a module name key or ansible.builtin.set_fact
|
||||
has_set_fact = False
|
||||
for key in task:
|
||||
if key in {"set_fact", "ansible.builtin.set_fact"}:
|
||||
has_set_fact = True
|
||||
break
|
||||
|
||||
if not has_set_fact:
|
||||
return
|
||||
|
||||
set_fact_body = task.get("set_fact") or task.get("ansible.builtin.set_fact")
|
||||
if not isinstance(set_fact_body, dict):
|
||||
return
|
||||
|
||||
task_name = task.get("name", "(unnamed)")
|
||||
|
||||
for fact_name, fact_value in set_fact_body.items():
|
||||
if fact_name in ("cacheable",):
|
||||
continue
|
||||
value_str = str(fact_value)
|
||||
for filter_pattern in TO_JSON_FILTERS:
|
||||
if filter_pattern in value_str:
|
||||
try:
|
||||
display_path = filepath.relative_to(repo_root)
|
||||
except ValueError:
|
||||
display_path = filepath
|
||||
errors.append(
|
||||
f"{display_path}: task '{task_name}' "
|
||||
f"sets fact '{fact_name}' with '{filter_pattern.strip()}' "
|
||||
f"— this converts native Python types to JSON strings. "
|
||||
f"Remove the filter to preserve the native type, or use "
|
||||
f"'| from_json' in the consuming task if the string "
|
||||
f"representation is intentional."
|
||||
)
|
||||
break # One error per fact is enough
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option(
|
||||
"--path",
|
||||
type=click.Path(exists=True, path_type=Path),
|
||||
help="Check a specific file or directory (default: ansible/playbooks + ansible/roles).",
|
||||
)
|
||||
@click.option(
|
||||
"--ansible-dir",
|
||||
"ansible_dirs",
|
||||
type=click.Path(exists=True, path_type=Path),
|
||||
multiple=True,
|
||||
default=None,
|
||||
help="Override the default ansible directories (can be repeated). Defaults to ansible/playbooks and ansible/roles.",
|
||||
)
|
||||
def main(path: Path | None, ansible_dirs: tuple[Path, ...]) -> None:
|
||||
"""Check that set_fact tasks don't misuse to_json."""
|
||||
dirs = list(ansible_dirs) if ansible_dirs else DEFAULT_ANSIBLE_DIRS
|
||||
if path:
|
||||
files = _find_task_files(path)
|
||||
else:
|
||||
files: list[Path] = []
|
||||
for d in dirs:
|
||||
files.extend(_find_task_files(d))
|
||||
|
||||
all_errors: list[str] = []
|
||||
for f in files:
|
||||
errors = _check_file(f, REPO_ROOT)
|
||||
all_errors.extend(errors)
|
||||
|
||||
if all_errors:
|
||||
click.echo("[check-ansible-set-fact-to-json] FAIL: set_fact with to_json found:")
|
||||
for err in all_errors:
|
||||
click.echo(f" - {err}")
|
||||
sys.exit(1)
|
||||
else:
|
||||
click.echo("[check-ansible-set-fact-to-json] OK: no set_fact tasks misuse to_json.")
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
main()
|
||||
@@ -0,0 +1,166 @@
|
||||
"""Check that Docker Compose services with healthchecks have ``init: true``.
|
||||
|
||||
This prevents zombie process accumulation on production VMs. Without
|
||||
``init: true``, Docker uses the container's PID 1 process to reap
|
||||
child processes. Many images (especially those using CMD-SHELL
|
||||
healthchecks with ``wget``) don't call ``wait()`` on children, causing
|
||||
zombies to accumulate.
|
||||
|
||||
The check scans all Jinja2 docker-compose templates for services that
|
||||
have a ``healthcheck:`` key but no ``init: true`` key. Since the
|
||||
templates use Jinja2 syntax (not pure YAML), the check uses text-based
|
||||
parsing to identify service blocks and their properties.
|
||||
|
||||
Usage::
|
||||
|
||||
python -m devx.tools.check_docker_init
|
||||
python -m devx.tools.check_docker_init --path ansible/roles/observability/templates/docker-compose.yml.j2
|
||||
|
||||
Exit code 0 if all services with healthchecks have init: true, 1 otherwise.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
|
||||
REPO_ROOT = Path.cwd()
|
||||
DEFAULT_TEMPLATES_DIR = REPO_ROOT / "ansible" / "roles"
|
||||
|
||||
|
||||
def _find_compose_templates(base: Path) -> list[Path]:
|
||||
"""Find all Jinja2 docker-compose templates under a base directory."""
|
||||
if base.is_file():
|
||||
return [base]
|
||||
if not base.is_dir():
|
||||
return []
|
||||
results: list[Path] = []
|
||||
for pattern in ("*docker-compose*", "*compose*"):
|
||||
results.extend(base.rglob(f"{pattern}.yml.j2"))
|
||||
results.extend(base.rglob(f"{pattern}.yaml.j2"))
|
||||
# Also check exporters-compose
|
||||
results.extend(base.rglob("exporters-compose*.j2"))
|
||||
# Deduplicate while preserving order
|
||||
seen: set[Path] = set()
|
||||
unique: list[Path] = []
|
||||
for p in sorted(results):
|
||||
if p not in seen:
|
||||
seen.add(p)
|
||||
unique.append(p)
|
||||
return unique
|
||||
|
||||
|
||||
def _parse_services(content: str) -> dict[str, list[str]]:
|
||||
"""Parse service blocks from a docker-compose Jinja2 template.
|
||||
|
||||
Returns a mapping of service_name → list of lines in that service block.
|
||||
"""
|
||||
lines = content.splitlines()
|
||||
in_services = False
|
||||
services: dict[str, list[str]] = {}
|
||||
current_svc: str | None = None
|
||||
current_lines: list[str] = []
|
||||
|
||||
for line in lines:
|
||||
if line.startswith("services:"):
|
||||
in_services = True
|
||||
continue
|
||||
if not in_services:
|
||||
continue
|
||||
# Top-level keys (networks:, volumes:) end the services section
|
||||
if re.match(r"^(networks|volumes):\s*$", line):
|
||||
if current_svc is not None:
|
||||
services[current_svc] = current_lines
|
||||
current_svc = None
|
||||
in_services = False
|
||||
continue
|
||||
# Service definition: exactly 2-space indent, ends with :
|
||||
# Service names can contain Jinja2 variables like {{ app_name }}
|
||||
# or {{ app_name }}-db. Match: 2-space indent + non-whitespace
|
||||
# chars (including {{ }}, -, _, .) + optional spaces inside {{ }} + :
|
||||
m = re.match(r"^ (\{\{.*?\}\}[a-zA-Z0-9_-]*|[a-zA-Z0-9_().-]+):\s*$", line)
|
||||
if m:
|
||||
if current_svc is not None:
|
||||
services[current_svc] = current_lines
|
||||
current_svc = m.group(1)
|
||||
current_lines = []
|
||||
elif current_svc is not None:
|
||||
current_lines.append(line)
|
||||
|
||||
if current_svc is not None:
|
||||
services[current_svc] = current_lines
|
||||
|
||||
return services
|
||||
|
||||
|
||||
def _check_template(filepath: Path, repo_root: Path) -> list[str]:
|
||||
"""Check a single docker-compose template for missing init: true.
|
||||
|
||||
Returns a list of error messages (empty if all OK).
|
||||
"""
|
||||
errors: list[str] = []
|
||||
content = filepath.read_text(encoding="utf-8")
|
||||
|
||||
if "services:" not in content:
|
||||
return errors
|
||||
|
||||
services = _parse_services(content)
|
||||
|
||||
for svc_name, svc_lines in services.items():
|
||||
svc_text = "\n".join(svc_lines)
|
||||
has_init = "init: true" in svc_text
|
||||
has_healthcheck = "healthcheck:" in svc_text
|
||||
# Skip services that are conditionally included (Jinja2 if blocks)
|
||||
# but still check them — the healthcheck is inside the conditional
|
||||
if has_healthcheck and not has_init:
|
||||
try:
|
||||
display_path = filepath.relative_to(repo_root)
|
||||
except ValueError:
|
||||
display_path = filepath
|
||||
errors.append(
|
||||
f"{display_path}: service '{svc_name}' has a healthcheck "
|
||||
f"but no 'init: true'. Without init: true, CMD-SHELL "
|
||||
f"healthchecks (wget, pgrep) spawn children that become "
|
||||
f"zombies when PID 1 doesn't reap them. Add 'init: true' "
|
||||
f"to enable Docker's built-in tini as PID 1."
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option(
|
||||
"--path",
|
||||
type=click.Path(exists=True, path_type=Path),
|
||||
help="Check a specific file or directory (default: ansible/roles/).",
|
||||
)
|
||||
@click.option(
|
||||
"--templates-dir",
|
||||
type=click.Path(exists=True, path_type=Path),
|
||||
default=None,
|
||||
help="Override the default templates directory (default: ansible/roles/).",
|
||||
)
|
||||
def main(path: Path | None, templates_dir: Path | None) -> None:
|
||||
"""Check that Docker Compose services with healthchecks have init: true."""
|
||||
tdir = templates_dir or DEFAULT_TEMPLATES_DIR
|
||||
files = _find_compose_templates(path) if path else _find_compose_templates(tdir)
|
||||
|
||||
all_errors: list[str] = []
|
||||
for f in files:
|
||||
errors = _check_template(f, tdir)
|
||||
all_errors.extend(errors)
|
||||
|
||||
if all_errors:
|
||||
click.echo("[check-docker-init] FAIL: services with healthchecks missing init: true:")
|
||||
for err in all_errors:
|
||||
click.echo(f" - {err}")
|
||||
sys.exit(1)
|
||||
else:
|
||||
click.echo("[check-docker-init] OK: all services with healthchecks have init: true.")
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
main()
|
||||
@@ -75,6 +75,25 @@ def _download(url: str, dest: Path) -> None:
|
||||
shutil.copyfileobj(resp, f)
|
||||
|
||||
|
||||
def _download_with_fallback(urls: list[str], binary_name: str) -> Path:
|
||||
"""Try downloading a binary from a list of URLs, falling back on failure.
|
||||
|
||||
Returns the path to the installed binary. Raises if all URLs fail.
|
||||
"""
|
||||
target_dir = _ensure_target_dir()
|
||||
dest = target_dir / binary_name
|
||||
errors: list[str] = []
|
||||
for url in urls:
|
||||
try:
|
||||
_download(url, dest)
|
||||
dest.chmod(0o755)
|
||||
return dest
|
||||
except Exception as exc: # noqa: BLE001
|
||||
errors.append(f"{url}: {exc}")
|
||||
click.echo(f" {binary_name}: retrying — {exc}")
|
||||
raise click.ClickException(f"Failed to download {binary_name} from all URLs: {'; '.join(errors)}")
|
||||
|
||||
|
||||
def _download_and_extract_tarball(url: str, binary_name: str) -> Path:
|
||||
"""Download a tarball, extract the binary, and install it to TARGET_DIR.
|
||||
|
||||
@@ -169,8 +188,13 @@ def install_tea() -> bool:
|
||||
click.echo("tea: already installed")
|
||||
return True
|
||||
arch = _arch()
|
||||
url = f"https://dl.gitea.com/tea/{TEA_VERSION}/tea-{TEA_VERSION}-linux-{arch}"
|
||||
dest = _download_binary(url, "tea")
|
||||
# dl.gitea.com is the primary CDN, but it can return 403 from some networks.
|
||||
# Fall back to the gitea.com release downloads URL.
|
||||
urls = [
|
||||
f"https://dl.gitea.com/tea/{TEA_VERSION}/tea-{TEA_VERSION}-linux-{arch}",
|
||||
f"https://gitea.com/gitea/tea/releases/download/v{TEA_VERSION}/tea-{TEA_VERSION}-linux-{arch}",
|
||||
]
|
||||
dest = _download_with_fallback(urls, "tea")
|
||||
click.echo(f"tea: installed to {dest}")
|
||||
return True
|
||||
|
||||
|
||||
@@ -59,12 +59,6 @@ def _install_pre_commit_hooks(bin_dir: str) -> None:
|
||||
def _install_ansible_collections(bin_dir: str) -> None:
|
||||
"""Install required Ansible Galaxy collections if requirements exist.
|
||||
|
||||
If the requirements file uses ``type: url`` entries pointing to the
|
||||
Gitea package registry, downloads them with authentication (using
|
||||
``CI_GITEA_TOKEN`` / ``CI_GITEA_API_TOKEN``) and installs from local
|
||||
files with ``--offline``. Falls back to direct galaxy install if the
|
||||
mirror download fails or no token is available.
|
||||
|
||||
Retries up to 3 times with exponential backoff to handle transient
|
||||
network timeouts when contacting galaxy.ansible.com.
|
||||
"""
|
||||
@@ -74,11 +68,6 @@ def _install_ansible_collections(bin_dir: str) -> None:
|
||||
click.echo(" ansible/requirements.yml not found — skipping collections.")
|
||||
return
|
||||
|
||||
# Try Gitea mirror first if requirements use type: url
|
||||
if _try_gitea_mirror_install(galaxy, requirements):
|
||||
return
|
||||
|
||||
# Fall back to direct galaxy install with retries
|
||||
@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=2, min=2, max=10), reraise=True)
|
||||
def _do_install() -> None:
|
||||
_run([galaxy, "collection", "install", "-r", str(requirements)])
|
||||
@@ -86,89 +75,6 @@ def _install_ansible_collections(bin_dir: str) -> None:
|
||||
_do_install()
|
||||
|
||||
|
||||
def _try_gitea_mirror_install(galaxy: str, requirements: Path) -> bool:
|
||||
"""Download ``type: url`` entries from Gitea with auth and install locally.
|
||||
|
||||
Returns ``True`` if the mirror install succeeded, ``False`` to fall back
|
||||
to direct galaxy install.
|
||||
"""
|
||||
import tempfile
|
||||
import urllib.request # noqa: PTH123 # nosec B404
|
||||
|
||||
import yaml # pyright: ignore[reportMissingImports]
|
||||
|
||||
try:
|
||||
data = yaml.safe_load(requirements.read_text())
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
collections = data.get("collections", []) if data else []
|
||||
url_entries = [c for c in collections if c.get("type") == "url"]
|
||||
if not url_entries:
|
||||
return False
|
||||
|
||||
# Resolve Gitea token for authenticated downloads
|
||||
token = os.environ.get("CI_GITEA_API_TOKEN", "").strip()
|
||||
if not token:
|
||||
token = os.environ.get("CI_GITEA_TOKEN", "").strip()
|
||||
if not token:
|
||||
token = os.environ.get("DEVELOPER_GITEA_API_TOKEN", "").strip()
|
||||
if not token:
|
||||
click.echo(" No Gitea token found — falling back to galaxy.ansible.com")
|
||||
return False
|
||||
|
||||
# Download each tarball with auth
|
||||
tmpdir = Path(tempfile.mkdtemp(prefix="ansible-collections-"))
|
||||
local_entries = []
|
||||
try:
|
||||
for entry in url_entries:
|
||||
source = entry.get("source", "")
|
||||
if "/api/packages/" not in source:
|
||||
local_entries.append(entry)
|
||||
continue
|
||||
filename = source.rsplit("/", 1)[-1]
|
||||
dest = tmpdir / filename
|
||||
click.echo(f" Downloading {entry.get('name', filename)} from Gitea mirror...")
|
||||
req = urllib.request.Request(source) # nosec B310
|
||||
req.add_header("Authorization", f"token {token}")
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp: # noqa: PTH123 # nosec B310
|
||||
dest.write_bytes(resp.read())
|
||||
except Exception as e:
|
||||
click.echo(f" WARN: mirror download failed for {entry.get('name')}: {e}")
|
||||
click.echo(" Falling back to galaxy.ansible.com")
|
||||
return False
|
||||
# Extract version from filename (e.g. ansible-posix-2.2.2.tar.gz)
|
||||
import re
|
||||
|
||||
ver_match = re.search(r"(\d+\.\d+\.\d+)", filename)
|
||||
local_entries.append(
|
||||
{
|
||||
"name": entry["name"],
|
||||
"version": ver_match.group(1) if ver_match else entry.get("version"),
|
||||
"type": "file",
|
||||
"source": str(dest),
|
||||
}
|
||||
)
|
||||
|
||||
# Add non-url entries as-is
|
||||
for entry in collections:
|
||||
if entry.get("type") != "url":
|
||||
local_entries.append(entry)
|
||||
|
||||
# Write local requirements file
|
||||
local_req = tmpdir / "requirements.yml"
|
||||
local_req.write_text(yaml.dump({"collections": local_entries}))
|
||||
|
||||
click.echo(" Installing collections from Gitea mirror (offline)...")
|
||||
_run([galaxy, "collection", "install", "-r", str(local_req), "--offline"])
|
||||
return True
|
||||
finally:
|
||||
import shutil as _shutil
|
||||
|
||||
_shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
|
||||
|
||||
def _configure_tea_login() -> None:
|
||||
"""Configure tea CLI login from .env if a Gitea token is set.
|
||||
|
||||
|
||||
@@ -64,9 +64,11 @@ def _install_in_image(
|
||||
link.symlink_to(opt_venv)
|
||||
|
||||
# Build pip install command
|
||||
# --no-deps: the CI image already has all dependencies pre-installed.
|
||||
# We only need to install the project itself in editable mode.
|
||||
spec = f".[{extras}]" if extras else "."
|
||||
pip_bin = str(Path(venv_link) / "bin" / "pip")
|
||||
cmd = [pip_bin, "install", "--no-cache-dir", "-e", spec]
|
||||
cmd = [pip_bin, "install", "--no-cache-dir", "--no-deps", "-e", spec]
|
||||
|
||||
env = os.environ.copy()
|
||||
try:
|
||||
|
||||
@@ -783,6 +783,14 @@
|
||||
"ru": "Additional directory to scan (default: scripts, tests). Can be repeated.",
|
||||
"zh": "Additional directory to scan (default: scripts, tests). Can be repeated."
|
||||
},
|
||||
"All molecule tests passed.": {
|
||||
"bg": "All molecule tests passed.",
|
||||
"de": "All molecule tests passed.",
|
||||
"en": "All molecule tests passed.",
|
||||
"pl": "Wszystkie testy molecule zakończone pomyślnie.",
|
||||
"ru": "All molecule tests passed.",
|
||||
"zh": "All molecule tests passed."
|
||||
},
|
||||
"Allow empty tag (PR mode where SHA is concrete).": {
|
||||
"bg": "Позволи празен таг (PR режим, където SHA е конкретен).",
|
||||
"de": "Leeren Tag zulassen (PR-Modus, in dem SHA konkret ist).",
|
||||
@@ -791,13 +799,13 @@
|
||||
"ru": "Разрешить пустой тег (режим PR, где SHA конкретен).",
|
||||
"zh": "允许空标签(SHA 为具体值的 PR 模式)。"
|
||||
},
|
||||
"Another runner failed. Stopping this runner early.": {
|
||||
"bg": "Друг runner се провали. Спиране на този runner по-рано.",
|
||||
"de": "Ein anderer Runner ist fehlgeschlagen. Dieser Runner wird vorzeitig gestoppt.",
|
||||
"en": "Another runner failed. Stopping this runner early.",
|
||||
"pl": "Inny runner zakończył się niepowodzeniem. Wczesne zatrzymanie tego runnera.",
|
||||
"ru": "Другой runner завершился с ошибкой. Останавливаю этот runner досрочно.",
|
||||
"zh": "另一个 runner 失败。提前停止此 runner。"
|
||||
"Another molecule runner failed. Stopping this runner early.": {
|
||||
"bg": "Another molecule runner failed. Stopping this runner early.",
|
||||
"de": "Another molecule runner failed. Stopping this runner early.",
|
||||
"en": "Another molecule runner failed. Stopping this runner early.",
|
||||
"pl": "Inny runner molecule zakończył się niepowodzeniem. Wczesne zatrzymanie tego runnera.",
|
||||
"ru": "Another molecule runner failed. Stopping this runner early.",
|
||||
"zh": "Another molecule runner failed. Stopping this runner early."
|
||||
},
|
||||
"Assigned {count} files to runner {runner_index}": {
|
||||
"bg": "Assigned {count} files to runner {runner_index}",
|
||||
@@ -1487,6 +1495,14 @@
|
||||
"ru": "FAILED: {count} undocumented dependency/ies",
|
||||
"zh": "FAILED: {count} undocumented dependency/ies"
|
||||
},
|
||||
"FAILED: {pair} exited with code {code}": {
|
||||
"bg": "FAILED: {pair} exited with code {code}",
|
||||
"de": "FAILED: {pair} exited with code {code}",
|
||||
"en": "FAILED: {pair} exited with code {code}",
|
||||
"pl": "NIEUDANE: {pair} zakończone kodem {code}",
|
||||
"ru": "FAILED: {pair} exited with code {code}",
|
||||
"zh": "FAILED: {pair} exited with code {code}"
|
||||
},
|
||||
"Failed images: {names}": {
|
||||
"bg": "Failed images: {names}",
|
||||
"de": "Failed images: {names}",
|
||||
@@ -2247,6 +2263,14 @@
|
||||
"ru": "PASS: All documentation checks passed!",
|
||||
"zh": "PASS: All documentation checks passed!"
|
||||
},
|
||||
"PASSED: {pair}": {
|
||||
"bg": "PASSED: {pair}",
|
||||
"de": "PASSED: {pair}",
|
||||
"en": "PASSED: {pair}",
|
||||
"pl": "UDANE: {pair}",
|
||||
"ru": "PASSED: {pair}",
|
||||
"zh": "PASSED: {pair}"
|
||||
},
|
||||
"PR #{pr} rebased successfully. A new CI run will start automatically.\nIf auto-merge is enabled (ready-to-merge label), the next CI run\nwill attempt to merge this PR.": {
|
||||
"bg": "PR #{pr} rebased successfully. A new CI run will start automatically.\nIf auto-merge is enabled (ready-to-merge label), the next CI run\nwill attempt to merge this PR.",
|
||||
"de": "PR #{pr} rebased successfully. A new CI run will start automatically.\nIf auto-merge is enabled (ready-to-merge label), the next CI run\nwill attempt to merge this PR.",
|
||||
@@ -2719,6 +2743,14 @@
|
||||
"ru": "Running: {cmd}",
|
||||
"zh": "Running: {cmd}"
|
||||
},
|
||||
"Running: {scenario} on {platform}": {
|
||||
"bg": "Running: {scenario} on {platform}",
|
||||
"de": "Running: {scenario} on {platform}",
|
||||
"en": "Running: {scenario} on {platform}",
|
||||
"pl": "Uruchamianie: {scenario} na {platform}",
|
||||
"ru": "Running: {scenario} on {platform}",
|
||||
"zh": "Running: {scenario} on {platform}"
|
||||
},
|
||||
"SSH key set up successfully": {
|
||||
"bg": "SSH ключът е настроен успешно",
|
||||
"de": "SSH-Schlüssel erfolgreich eingerichtet",
|
||||
@@ -3798,5 +3830,13 @@
|
||||
"pl": "[check-test-speed] CI environment detected — scaling limits by {factor}x (total: {orig}s → {eff}s, per-test: {orig_s}s → {eff_s}s)",
|
||||
"ru": "[check-test-speed] CI environment detected — scaling limits by {factor}x (total: {orig}s → {eff}s, per-test: {orig_s}s → {eff_s}s)",
|
||||
"zh": "[check-test-speed] CI environment detected — scaling limits by {factor}x (total: {orig}s → {eff}s, per-test: {orig_s}s → {eff_s}s)"
|
||||
},
|
||||
"Cleaning up: running molecule destroy for {scenario}": {
|
||||
"en": "Cleaning up: running molecule destroy for {scenario}",
|
||||
"bg": "Изчистване: изпълнение на molecule destroy за {scenario}",
|
||||
"de": "Aufräumen: molecule destroy wird ausgeführt für {scenario}",
|
||||
"pl": "Czyszczenie: uruchamianie molecule destroy dla {scenario}",
|
||||
"ru": "Очистка: запуск molecule destroy для {scenario}",
|
||||
"zh": "清理:正在为 {scenario} 运行 molecule destroy"
|
||||
}
|
||||
}
|
||||
|
||||
+94
-6
@@ -1,14 +1,26 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Utilities for handling API response values.
|
||||
"""Utilities for handling API response values and base HTTP API client.
|
||||
|
||||
Many APIs return boolean values as strings (``"true"``, ``"false"``)
|
||||
rather than native JSON booleans. The Mattermost ``/api/v4/config/client``
|
||||
endpoint is a notable example. These helpers handle both string and
|
||||
boolean responses safely.
|
||||
This module provides two categories of utilities:
|
||||
|
||||
1. **Response helpers** — :func:`is_truthy` and :func:`is_falsy` handle
|
||||
APIs that return boolean values as strings (``"true"``, ``"false"``)
|
||||
rather than native JSON booleans.
|
||||
|
||||
2. **Base API client** — :class:`APIClient` provides a reusable base
|
||||
class for HTTP API clients with consistent timeout handling, header
|
||||
propagation, and automatic raising on 4xx/5xx responses.
|
||||
|
||||
Usage::
|
||||
|
||||
from devx.utils.api import is_truthy, is_falsy
|
||||
from devx.utils.api import APIClient, is_truthy
|
||||
|
||||
class MyClient(APIClient):
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
base_url="https://api.example.com",
|
||||
headers={"Authorization": "Bearer token"},
|
||||
)
|
||||
|
||||
if not is_truthy(config.get("EnableOpenServer")):
|
||||
raise ValueError("EnableOpenServer not enabled")
|
||||
@@ -16,6 +28,82 @@ Usage::
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
class APIClient:
|
||||
"""Base class for HTTP API clients.
|
||||
|
||||
Subclasses set ``base_url``, ``headers``, and optionally ``auth`` in
|
||||
their constructor, then use :meth:`_request` or the convenience
|
||||
methods (:meth:`get`, :meth:`post`, etc.) to make requests.
|
||||
|
||||
All requests raise :class:`requests.HTTPError` on 4xx/5xx responses
|
||||
via :meth:`requests.Response.raise_for_status`.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
base_url: str,
|
||||
headers: dict,
|
||||
timeout: int = 30,
|
||||
verify: bool = True,
|
||||
auth: tuple[str, str] | None = None,
|
||||
) -> None:
|
||||
"""Initialize the API client.
|
||||
|
||||
Args:
|
||||
base_url: Base URL for the API (trailing slash stripped).
|
||||
headers: Default headers sent with every request.
|
||||
timeout: Request timeout in seconds.
|
||||
verify: Whether to verify TLS certificates.
|
||||
auth: Optional ``(username, password)`` tuple for basic auth.
|
||||
"""
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.headers = headers
|
||||
self.timeout = timeout
|
||||
self.verify = verify
|
||||
self.auth = auth
|
||||
|
||||
def _request(self, method: str, path: str, **kwargs) -> requests.Response:
|
||||
"""Execute an HTTP request against the API.
|
||||
|
||||
The URL is constructed as ``{base_url}{path}``. Default timeout,
|
||||
verify, auth, and headers are applied but can be overridden via
|
||||
``kwargs``.
|
||||
|
||||
Raises:
|
||||
requests.HTTPError: On 4xx/5xx response status codes.
|
||||
"""
|
||||
url = f"{self.base_url}{path}"
|
||||
kwargs.setdefault("timeout", self.timeout)
|
||||
kwargs.setdefault("verify", self.verify)
|
||||
if self.auth is not None:
|
||||
kwargs.setdefault("auth", self.auth)
|
||||
resp = requests.request(method, url, headers=self.headers, **kwargs) # noqa: S113
|
||||
resp.raise_for_status()
|
||||
return resp
|
||||
|
||||
def get(self, path: str, **kwargs) -> requests.Response:
|
||||
"""Send a GET request."""
|
||||
return self._request("GET", path, **kwargs)
|
||||
|
||||
def post(self, path: str, **kwargs) -> requests.Response:
|
||||
"""Send a POST request."""
|
||||
return self._request("POST", path, **kwargs)
|
||||
|
||||
def put(self, path: str, **kwargs) -> requests.Response:
|
||||
"""Send a PUT request."""
|
||||
return self._request("PUT", path, **kwargs)
|
||||
|
||||
def delete(self, path: str, **kwargs) -> requests.Response:
|
||||
"""Send a DELETE request."""
|
||||
return self._request("DELETE", path, **kwargs)
|
||||
|
||||
def patch(self, path: str, **kwargs) -> requests.Response:
|
||||
"""Send a PATCH request."""
|
||||
return self._request("PATCH", path, **kwargs)
|
||||
|
||||
|
||||
def is_truthy(value: str | bool | None) -> bool:
|
||||
"""Check if an API config value is truthy.
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
"""Shared Jinja2 environment helpers for unit tests and template rendering.
|
||||
|
||||
Creating a Jinja2 Environment is expensive (filesystem scanning, template
|
||||
compilation). These helpers create cached environments with
|
||||
``auto_reload=False`` to skip stat() calls on every ``get_template``,
|
||||
which is the single biggest speedup for template-heavy test suites.
|
||||
|
||||
The filters mimic Ansible builtins not available in plain Jinja2,
|
||||
making it possible to render Ansible templates outside of Ansible
|
||||
(e.g. in unit tests or config generation scripts).
|
||||
|
||||
Usage::
|
||||
|
||||
from devx.utils.jinja import make_env, render_template
|
||||
|
||||
env = make_env("/path/to/templates")
|
||||
output = render_template(env, "alert-rules.yml.j2", grafana_base_url="https://grafana.example.com")
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import json
|
||||
import re
|
||||
|
||||
import jinja2
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Filters (mimic Ansible builtins not available in plain Jinja2)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def to_json(value) -> str:
|
||||
return json.dumps(value)
|
||||
|
||||
|
||||
def to_bool(value) -> bool:
|
||||
"""Mimic Ansible's |bool filter for plain Jinja2 tests."""
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
return value.lower() not in ("", "false", "0", "no", "off", "null", "none")
|
||||
return bool(value)
|
||||
|
||||
|
||||
def regex_replace(value, pattern: str, replacement: str) -> str:
|
||||
"""Mimic Ansible's |regex_replace filter."""
|
||||
return re.sub(pattern, replacement, str(value))
|
||||
|
||||
|
||||
def regex_escape(value) -> str:
|
||||
"""Mimic Ansible's |regex_escape filter."""
|
||||
return re.escape(str(value))
|
||||
|
||||
|
||||
def regex_search(value, pattern: str) -> str | None:
|
||||
"""Mimic Ansible's |regex_search filter.
|
||||
|
||||
Returns the first match (group 0) or None if no match.
|
||||
Ansible returns the full match string or None.
|
||||
"""
|
||||
m = re.search(pattern, str(value))
|
||||
return m.group(0) if m else None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Environment factory
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_FILTERS = {
|
||||
"to_json": to_json,
|
||||
"bool": to_bool,
|
||||
"regex_replace": regex_replace,
|
||||
"regex_escape": regex_escape,
|
||||
"regex_search": regex_search,
|
||||
}
|
||||
|
||||
|
||||
@functools.cache
|
||||
def make_env(loader_path: str) -> jinja2.Environment:
|
||||
"""Create a cached Jinja2 Environment with standard filters.
|
||||
|
||||
``auto_reload=False`` skips stat() on every get_template call —
|
||||
templates don't change during a test run so this is safe and
|
||||
cuts ~40% off render time.
|
||||
"""
|
||||
env = jinja2.Environment( # nosec B701 — renders YAML/config templates, not HTML
|
||||
loader=jinja2.FileSystemLoader(loader_path),
|
||||
undefined=jinja2.StrictUndefined,
|
||||
auto_reload=False,
|
||||
cache_size=400,
|
||||
)
|
||||
env.filters.update(_FILTERS)
|
||||
return env
|
||||
|
||||
|
||||
@functools.cache
|
||||
def make_value_env() -> jinja2.Environment:
|
||||
"""Cached environment for rendering individual manifest string values."""
|
||||
env = jinja2.Environment( # nosec B701 — renders config values, not HTML
|
||||
undefined=jinja2.ChainableUndefined,
|
||||
auto_reload=False,
|
||||
)
|
||||
env.filters.update(_FILTERS)
|
||||
return env
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Render helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def render_template(env: jinja2.Environment, template_name: str, **kwargs) -> str:
|
||||
"""Render a named template from a FileSystemLoader-backed env."""
|
||||
return env.get_template(template_name).render(**kwargs)
|
||||
|
||||
|
||||
def render_value(value, ctx: dict):
|
||||
"""Render a single string value as a Jinja2 template if it contains expressions."""
|
||||
if not isinstance(value, str):
|
||||
return value
|
||||
if "{{" not in value and "{%" not in value:
|
||||
return value
|
||||
return make_value_env().from_string(value).render(**ctx)
|
||||
|
||||
|
||||
def render_manifest_values(obj, ctx: dict):
|
||||
"""Recursively render all Jinja2 expressions in manifest string values."""
|
||||
if isinstance(obj, dict):
|
||||
return {k: render_manifest_values(v, ctx) for k, v in obj.items()}
|
||||
if isinstance(obj, list):
|
||||
return [render_manifest_values(v, ctx) for v in obj]
|
||||
return render_value(obj, ctx)
|
||||
@@ -0,0 +1,79 @@
|
||||
"""User-facing output utilities combining console and log output.
|
||||
|
||||
Console messages are colorised via ``click.style`` for visual feedback.
|
||||
The persistent log file always receives plain text (no ANSI codes).
|
||||
|
||||
This is a generalisation of grm's ``ui.say()`` function, extracted so
|
||||
that any CLI tool can use the same pattern. The logger name and
|
||||
console-level env var are configurable.
|
||||
|
||||
Usage::
|
||||
|
||||
from devx.utils.ui import say
|
||||
|
||||
say("Starting deployment...")
|
||||
say("Error occurred", level=logging.ERROR, err=True, color="red")
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
import click
|
||||
|
||||
# Configurable env var for console verbosity — projects can override
|
||||
# via :func:`configure_ui`.
|
||||
_LOG_LEVEL_ENV_VAR = "DEVX_LOG_LEVEL"
|
||||
_LOGGER_NAME = "devx"
|
||||
|
||||
|
||||
def configure_ui(*, log_level_env_var: str = "DEVX_LOG_LEVEL", logger_name: str = "devx") -> None:
|
||||
"""Override the env var name and logger name used by :func:`say`.
|
||||
|
||||
This allows downstream projects (e.g. grm) to use their own env var
|
||||
names (e.g. ``GRM_LOG_LEVEL``) and logger names while still using
|
||||
devx's ui module.
|
||||
|
||||
Args:
|
||||
log_level_env_var: Environment variable name for console log level.
|
||||
logger_name: Logger name for persistent log file output.
|
||||
"""
|
||||
global _LOG_LEVEL_ENV_VAR, _LOGGER_NAME
|
||||
_LOG_LEVEL_ENV_VAR = log_level_env_var
|
||||
_LOGGER_NAME = logger_name
|
||||
|
||||
|
||||
def _console_level() -> int:
|
||||
"""Return the minimum level for console output from the configured env var."""
|
||||
value = os.getenv(_LOG_LEVEL_ENV_VAR, "INFO")
|
||||
try:
|
||||
return getattr(logging, value.upper())
|
||||
except AttributeError:
|
||||
return logging.INFO
|
||||
|
||||
|
||||
def say(
|
||||
msg: str,
|
||||
level: int = logging.INFO,
|
||||
err: bool = False,
|
||||
color: str | None = None,
|
||||
) -> None:
|
||||
"""Output a message to the user and also log it for auditing.
|
||||
|
||||
Console output goes via ``click.echo`` (handles encoding, CliRunner,
|
||||
Windows colorama) only when *level* is at least the configured
|
||||
console log level (default ``DEVX_LOG_LEVEL``, falls back to INFO).
|
||||
The same message is always sent to the configured logger so it
|
||||
appears in the persistent log file regardless of console verbosity.
|
||||
|
||||
Args:
|
||||
msg: Message to display.
|
||||
level: Logging level (e.g. ``logging.INFO``, ``logging.ERROR``).
|
||||
err: If True, output to stderr instead of stdout.
|
||||
color: Optional ``click.style`` fg color (e.g. ``"green"``, ``"red"``).
|
||||
"""
|
||||
if level >= _console_level():
|
||||
styled = click.style(msg, fg=color) if color else msg
|
||||
click.echo(styled, err=err)
|
||||
logging.getLogger(_LOGGER_NAME).log(level, msg)
|
||||
Reference in New Issue
Block a user