DEVX-162: fix: serialize registry uploads and retry on HTTP 500

src/devx/tools/build_image.py

@@ -50,6 +50,7 @@ from dataclasses import dataclass, field
 from pathlib import Path
 
 import click
+from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential
 
 from devx.i18n import _
 from devx.tokens import get_developer_token
@@ -254,6 +255,29 @@ def delete_remote_manifest(
     return True
 
 
+class PushHTTP500Error(Exception):
+    """Raised when docker push fails with an HTTP 500 from the registry."""
+
+
+def _run_push(cmd: list[str]) -> subprocess.CompletedProcess[str]:
+    """Run a docker push command, raising PushHTTP500Error on registry 500.
+
+    The Gitea container registry (v1.27.x) has a race condition in
+    BlobUploader.Append that causes intermittent HTTP 500 "offset
+    mismatch" errors during concurrent blob uploads.  Retrying the
+    push gives the registry time to recover.
+    """
+    result = subprocess.run(  # nosec B603
+        cmd,
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    if result.returncode != 0 and "500" in result.stderr:
+        raise PushHTTP500Error(result.stderr.strip())
+    return result
+
+
 def push_image(
     spec: ImageSpec,
     registry: str,
@@ -270,6 +294,9 @@ def push_image(
     with Gitea #31964 ("package version already exists") do we delete
     the old manifest and retry.  This avoids losing the existing tag
     when the push fails for unrelated reasons (e.g. HTTP 500).
+
+    HTTP 500 errors from the Gitea registry race condition are retried
+    up to 3 times with exponential backoff (5s, 10s) via tenacity.
     """
     full_tags = [build_full_tag(registry, spec.name, t) for t in spec.tags]
     all_ok = True
@@ -279,12 +306,26 @@ def push_image(
             click.echo(f"[dry-run] {' '.join(cmd)}")
             continue
         click.echo(f"Pushing {ft}...")
-        result = subprocess.run(  # nosec B603
-            cmd,
-            capture_output=True,
-            text=True,
-            check=False,
+
+        @retry(
+            stop=stop_after_attempt(3),
+            wait=wait_exponential(multiplier=5, min=5, max=20),
+            retry=retry_if_exception_type(PushHTTP500Error),
+            reraise=True,
         )
+        def _attempt(_cmd: list[str] = cmd) -> subprocess.CompletedProcess[str]:
+            return _run_push(_cmd)
+
+        try:
+            result = _attempt()
+        except PushHTTP500Error as e:
+            click.echo(
+                _("Push failed for {tag}: {error}", tag=ft, error=str(e)),
+                err=True,
+            )
+            all_ok = False
+            continue
+
         if result.returncode == 0:
             click.echo(f"Pushed {ft}")
             continue