Public Access
DEVX-164: fix: increase HTTP 500 retry count to 5 with longer backoff and visible logging
This commit was merged in pull request #311.
This commit is contained in:
@@ -0,0 +1,34 @@
|
|||||||
|
# DEVX-164: Increase HTTP 500 retry count and backoff for docker push
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
The HTTP 500 retry logic (DEVX-162, DEVX-163) works correctly — 3 retry
|
||||||
|
attempts are made. But all 3 attempts fail because the Gitea registry's
|
||||||
|
"offset mismatch" race condition needs more than ~15s to recover. The
|
||||||
|
current backoff is 5s-20s with 3 attempts (total ~15s of waiting).
|
||||||
|
|
||||||
|
## Approach
|
||||||
|
Increase retry count from 3 to 5 and backoff from 5-20s to 10-60s,
|
||||||
|
giving the registry up to ~2 minutes to recover. Add visible logging
|
||||||
|
between retry attempts so the CI logs show the retry happening.
|
||||||
|
|
||||||
|
REQ-1: Increase retry count from 3 to 5
|
||||||
|
REQ-2: Increase backoff from 5-20s to 10-60s exponential
|
||||||
|
REQ-3: Add visible logging between retry attempts (click.echo)
|
||||||
|
REQ-4: All tests pass with 100% coverage
|
||||||
|
|
||||||
|
## Test Plan
|
||||||
|
- Unit tests verify retry count and backoff parameters
|
||||||
|
- Unit tests verify logging output on retry
|
||||||
|
- Manual: trigger build-images workflow and verify retries visible in logs
|
||||||
|
|
||||||
|
## Deploy Plan
|
||||||
|
- Merge to master
|
||||||
|
|
||||||
|
## Rollback Plan
|
||||||
|
- Revert the merge commit
|
||||||
|
|
||||||
|
## Acceptance Criteria
|
||||||
|
- [x] REQ-1: Increase retry count from 3 to 5
|
||||||
|
- [x] REQ-2: Increase backoff from 5-20s to 10-60s exponential
|
||||||
|
- [x] REQ-3: Add visible logging between retry attempts (click.echo)
|
||||||
|
- [x] REQ-4: All tests pass with 100% coverage
|
||||||
@@ -315,9 +315,17 @@ def push_image(
|
|||||||
click.echo(f"Pushing {ft}...")
|
click.echo(f"Pushing {ft}...")
|
||||||
|
|
||||||
@retry(
|
@retry(
|
||||||
stop=stop_after_attempt(3),
|
stop=stop_after_attempt(5),
|
||||||
wait=wait_exponential(multiplier=5, min=5, max=20),
|
wait=wait_exponential(multiplier=10, min=10, max=60),
|
||||||
retry=retry_if_exception_type(PushHTTP500Error),
|
retry=retry_if_exception_type(PushHTTP500Error),
|
||||||
|
before_sleep=lambda retry_state: click.echo(
|
||||||
|
_(
|
||||||
|
" HTTP 500 from registry, retrying in {wait:.0f}s (attempt {attempt}/5)...",
|
||||||
|
wait=retry_state.next_action.sleep if retry_state.next_action else 0,
|
||||||
|
attempt=retry_state.attempt_number + 1,
|
||||||
|
),
|
||||||
|
err=True,
|
||||||
|
),
|
||||||
reraise=True,
|
reraise=True,
|
||||||
)
|
)
|
||||||
def _attempt(_cmd: list[str] = cmd) -> subprocess.CompletedProcess[str]:
|
def _attempt(_cmd: list[str] = cmd) -> subprocess.CompletedProcess[str]:
|
||||||
|
|||||||
@@ -5152,5 +5152,13 @@
|
|||||||
"pl": "PR has 'refactoring' label — size check bypassed.",
|
"pl": "PR has 'refactoring' label — size check bypassed.",
|
||||||
"ru": "PR has 'refactoring' label — size check bypassed.",
|
"ru": "PR has 'refactoring' label — size check bypassed.",
|
||||||
"zh": "PR has 'refactoring' label — size check bypassed."
|
"zh": "PR has 'refactoring' label — size check bypassed."
|
||||||
|
},
|
||||||
|
" HTTP 500 from registry, retrying in {wait:.0f}s (attempt {attempt}/5)...": {
|
||||||
|
"en": "HTTP 500 from registry, retrying in {wait:.0f}s (attempt {attempt}/5)...",
|
||||||
|
"bg": " HTTP 500 от регистъра, повторен опит след {wait:.0f}с (опит {attempt}/5)...",
|
||||||
|
"de": " HTTP 500 vom Registry, Wiederholung in {wait:.0f}s (Versuch {attempt}/5)...",
|
||||||
|
"pl": " HTTP 500 z rejestru, ponawianie za {wait:.0f}s (próba {attempt}/5)...",
|
||||||
|
"ru": " HTTP 500 от реестра, повтор через {wait:.0f}с (попытка {attempt}/5)...",
|
||||||
|
"zh": " 注册表返回 HTTP 500,{wait:.0f}秒后重试(第{attempt}/5次尝试)..."
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user