GRM-51: fix: API resilience with retry, idempotent releases, and graceful Vikunja errors

src/gitea_runner_manager/api_clients.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import logging
+import time
 from typing import Any
 
 import requests
@@ -12,6 +13,11 @@ from .exceptions import APIError
 
 logger = logging.getLogger("grm")
 
+# Retry configuration for transient errors (429, 5xx, connection errors)
+MAX_RETRIES = 3
+RETRY_BACKOFF_BASE = 2  # seconds: 2, 4, 8
+RETRY_STATUS_CODES = {429, 500, 502, 503, 504}
+
 
 def _parse_error(e: requests.HTTPError) -> tuple[int, str]:
     """Extract status code and message from an HTTPError response."""
@@ -25,6 +31,16 @@ def _parse_error(e: requests.HTTPError) -> tuple[int, str]:
     return status, message
 
 
+def _is_retryable(e: Exception) -> bool:
+    """Check if an exception is a transient error worth retrying."""
+    if isinstance(e, requests.ConnectionError):
+        return True
+    if isinstance(e, requests.HTTPError):
+        status, _ = _parse_error(e)
+        return status in RETRY_STATUS_CODES
+    return isinstance(e, requests.Timeout)
+
+
 class GiteaClient:
     """Low-level Gitea REST API client with connection pooling."""
 
@@ -45,13 +61,48 @@ class GiteaClient:
 
     def _request(self, method: str, path: str, **kwargs: Any) -> requests.Response:
         url = self._url(path)
-        try:
-            response = self._session.request(method, url, timeout=DEFAULT_TIMEOUT, **kwargs)
-            response.raise_for_status()
-        except requests.HTTPError as e:
-            status, message = _parse_error(e)
-            raise APIError(status, message) from e
-        return response
+        last_exc: Exception | None = None
+        for attempt in range(MAX_RETRIES):
+            try:
+                response = self._session.request(method, url, timeout=DEFAULT_TIMEOUT, **kwargs)
+                response.raise_for_status()
+                return response
+            except requests.HTTPError as e:
+                status, message = _parse_error(e)
+                if _is_retryable(e) and attempt < MAX_RETRIES - 1:
+                    wait = RETRY_BACKOFF_BASE ** (attempt + 1)
+                    logger.warning(
+                        "Transient HTTP %d on %s %s, retrying in %ds (attempt %d/%d)",
+                        status,
+                        method,
+                        path,
+                        wait,
+                        attempt + 1,
+                        MAX_RETRIES,
+                    )
+                    time.sleep(wait)
+                    last_exc = e
+                    continue
+                raise APIError(status, message) from e
+            except (requests.ConnectionError, requests.Timeout) as e:
+                if attempt < MAX_RETRIES - 1:
+                    wait = RETRY_BACKOFF_BASE ** (attempt + 1)
+                    logger.warning(
+                        "Connection error on %s %s, retrying in %ds (attempt %d/%d)",
+                        method,
+                        path,
+                        wait,
+                        attempt + 1,
+                        MAX_RETRIES,
+                    )
+                    time.sleep(wait)
+                    last_exc = e
+                    continue
+                raise APIError(0, str(e)) from e
+        # Should not reach here, but just in case
+        if last_exc:  # pragma: no cover
+            raise APIError(0, str(last_exc)) from last_exc
+        raise APIError(0, "Max retries exceeded")  # pragma: no cover
 
     # -- repo settings --
 
@@ -202,6 +253,32 @@ class GiteaClient:
         r = self._request("POST", "/releases", json=payload)
         return r.json()
 
+    def get_release_by_tag(self, tag: str) -> dict[str, Any] | None:
+        """Fetch a release by its tag name. Returns None if not found."""
+        try:
+            r = self._request("GET", f"/releases/tags/{tag}")
+            return r.json()
+        except APIError:
+            return None
+
+    def create_release_idempotent(
+        self,
+        tag: str,
+        name: str = "",
+        body: str = "",
+        draft: bool = False,
+        prerelease: bool = False,
+    ) -> dict[str, Any]:
+        """Create a release, or return the existing one if it already exists.
+
+        This is idempotent — safe to call multiple times for the same tag.
+        """
+        existing = self.get_release_by_tag(tag)
+        if existing:
+            logger.info("Release for tag %s already exists (ID %s), skipping creation.", tag, existing.get("id"))
+            return existing
+        return self.create_release(tag=tag, name=name, body=body, draft=draft, prerelease=prerelease)
+
 
 class VikunjaClient:
     """Low-level Vikunja REST API client with connection pooling."""
@@ -213,13 +290,47 @@ class VikunjaClient:
 
     def _request(self, method: str, path: str, **kwargs: Any) -> requests.Response:
         url = f"{self._base_url}{path}"
-        try:
-            response = self._session.request(method, url, timeout=DEFAULT_TIMEOUT, **kwargs)
-            response.raise_for_status()
-        except requests.HTTPError as e:
-            status, message = _parse_error(e)
-            raise APIError(status, message) from e
-        return response
+        last_exc: Exception | None = None
+        for attempt in range(MAX_RETRIES):
+            try:
+                response = self._session.request(method, url, timeout=DEFAULT_TIMEOUT, **kwargs)
+                response.raise_for_status()
+                return response
+            except requests.HTTPError as e:
+                status, message = _parse_error(e)
+                if _is_retryable(e) and attempt < MAX_RETRIES - 1:
+                    wait = RETRY_BACKOFF_BASE ** (attempt + 1)
+                    logger.warning(
+                        "Transient HTTP %d on %s %s, retrying in %ds (attempt %d/%d)",
+                        status,
+                        method,
+                        path,
+                        wait,
+                        attempt + 1,
+                        MAX_RETRIES,
+                    )
+                    time.sleep(wait)
+                    last_exc = e
+                    continue
+                raise APIError(status, message) from e
+            except (requests.ConnectionError, requests.Timeout) as e:
+                if attempt < MAX_RETRIES - 1:
+                    wait = RETRY_BACKOFF_BASE ** (attempt + 1)
+                    logger.warning(
+                        "Connection error on %s %s, retrying in %ds (attempt %d/%d)",
+                        method,
+                        path,
+                        wait,
+                        attempt + 1,
+                        MAX_RETRIES,
+                    )
+                    time.sleep(wait)
+                    last_exc = e
+                    continue
+                raise APIError(0, str(e)) from e
+        if last_exc:  # pragma: no cover
+            raise APIError(0, str(last_exc)) from last_exc
+        raise APIError(0, "Max retries exceeded")  # pragma: no cover
 
     def list_tasks(self, **params: Any) -> list[dict[str, Any]]:
         r = self._request("GET", "/tasks", params=params)