Public Access
DEVX-164: fix: increase HTTP 500 retry count to 5 with longer backoff and visible logging
This commit was merged in pull request #311.
This commit is contained in:
@@ -0,0 +1,34 @@
|
||||
# DEVX-164: Increase HTTP 500 retry count and backoff for docker push
|
||||
|
||||
## Problem
|
||||
The HTTP 500 retry logic (DEVX-162, DEVX-163) works correctly — 3 retry
|
||||
attempts are made. But all 3 attempts fail because the Gitea registry's
|
||||
"offset mismatch" race condition needs more than ~15s to recover. The
|
||||
current backoff is 5s-20s with 3 attempts (total ~15s of waiting).
|
||||
|
||||
## Approach
|
||||
Increase retry count from 3 to 5 and backoff from 5-20s to 10-60s,
|
||||
giving the registry up to ~2 minutes to recover. Add visible logging
|
||||
between retry attempts so the CI logs show the retry happening.
|
||||
|
||||
REQ-1: Increase retry count from 3 to 5
|
||||
REQ-2: Increase backoff from 5-20s to 10-60s exponential
|
||||
REQ-3: Add visible logging between retry attempts (click.echo)
|
||||
REQ-4: All tests pass with 100% coverage
|
||||
|
||||
## Test Plan
|
||||
- Unit tests verify retry count and backoff parameters
|
||||
- Unit tests verify logging output on retry
|
||||
- Manual: trigger build-images workflow and verify retries visible in logs
|
||||
|
||||
## Deploy Plan
|
||||
- Merge to master
|
||||
|
||||
## Rollback Plan
|
||||
- Revert the merge commit
|
||||
|
||||
## Acceptance Criteria
|
||||
- [x] REQ-1: Increase retry count from 3 to 5
|
||||
- [x] REQ-2: Increase backoff from 5-20s to 10-60s exponential
|
||||
- [x] REQ-3: Add visible logging between retry attempts (click.echo)
|
||||
- [x] REQ-4: All tests pass with 100% coverage
|
||||
@@ -315,9 +315,17 @@ def push_image(
|
||||
click.echo(f"Pushing {ft}...")
|
||||
|
||||
@retry(
|
||||
stop=stop_after_attempt(3),
|
||||
wait=wait_exponential(multiplier=5, min=5, max=20),
|
||||
stop=stop_after_attempt(5),
|
||||
wait=wait_exponential(multiplier=10, min=10, max=60),
|
||||
retry=retry_if_exception_type(PushHTTP500Error),
|
||||
before_sleep=lambda retry_state: click.echo(
|
||||
_(
|
||||
" HTTP 500 from registry, retrying in {wait:.0f}s (attempt {attempt}/5)...",
|
||||
wait=retry_state.next_action.sleep if retry_state.next_action else 0,
|
||||
attempt=retry_state.attempt_number + 1,
|
||||
),
|
||||
err=True,
|
||||
),
|
||||
reraise=True,
|
||||
)
|
||||
def _attempt(_cmd: list[str] = cmd) -> subprocess.CompletedProcess[str]:
|
||||
|
||||
@@ -5152,5 +5152,13 @@
|
||||
"pl": "PR has 'refactoring' label — size check bypassed.",
|
||||
"ru": "PR has 'refactoring' label — size check bypassed.",
|
||||
"zh": "PR has 'refactoring' label — size check bypassed."
|
||||
},
|
||||
" HTTP 500 from registry, retrying in {wait:.0f}s (attempt {attempt}/5)...": {
|
||||
"en": "HTTP 500 from registry, retrying in {wait:.0f}s (attempt {attempt}/5)...",
|
||||
"bg": " HTTP 500 от регистъра, повторен опит след {wait:.0f}с (опит {attempt}/5)...",
|
||||
"de": " HTTP 500 vom Registry, Wiederholung in {wait:.0f}s (Versuch {attempt}/5)...",
|
||||
"pl": " HTTP 500 z rejestru, ponawianie za {wait:.0f}s (próba {attempt}/5)...",
|
||||
"ru": " HTTP 500 от реестра, повтор через {wait:.0f}с (попытка {attempt}/5)...",
|
||||
"zh": " 注册表返回 HTTP 500,{wait:.0f}秒后重试(第{attempt}/5次尝试)..."
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user