From d9a5a097b4fa4f86a6244bf6d4928c53e449a072 Mon Sep 17 00:00:00 2001 From: emil Date: Tue, 30 Jun 2026 07:13:39 +0200 Subject: [PATCH] fix: retry wiki integrity check on transient API timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The wiki integrity check failed when the Gitea API was temporarily unavailable after a batch of wiki page updates. list_wiki_pages caught APIError and returned {}, causing all pages to be reported as missing and empty — even though the sync itself succeeded. Fix: list_wiki_pages now raises APIError instead of swallowing it. verify_wiki_integrity uses tenacity (same pattern as api_clients) for exponential backoff retry on the page list. If retries are exhausted, it returns no failures with a warning, since the sync already succeeded and a transient API outage should not fail the job. The --verify path in main() also uses the retry wrapper and skips gracefully on failure. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- src/devx/ci/sync_wiki.py | 75 ++++++++++++++++++++++++++++---- src/devx/translations.json | 16 +++++++ tests/unit/test_sync_wiki.py | 84 +++++++++++++++++++++++++++++++++--- 3 files changed, 162 insertions(+), 13 deletions(-) diff --git a/src/devx/ci/sync_wiki.py b/src/devx/ci/sync_wiki.py index 6d41569..8e23476 100644 --- a/src/devx/ci/sync_wiki.py +++ b/src/devx/ci/sync_wiki.py @@ -21,11 +21,19 @@ from __future__ import annotations import base64 import json +import logging import os from pathlib import Path import click from dotenv import load_dotenv # pyright: ignore[reportMissingImports,reportUnknownVariableType] +from tenacity import ( + before_sleep_log, + retry, + retry_if_exception_type, + stop_after_attempt, + wait_exponential, +) from devx.api_clients import GiteaClient from devx.config import GITEA_API_URL, REPO_NAME, REPO_OWNER @@ -86,11 +94,12 @@ def decode_content(content_b64: str) -> str: def list_wiki_pages(client: GiteaClient) -> dict[str, str]: - """List existing wiki pages, returning {title: sub_url}.""" - try: - pages = client._request("GET", "/wiki/pages").json() - except APIError: - return {} + """List existing wiki pages, returning {title: sub_url}. + + Raises :class:`APIError` if the wiki API is unavailable — the caller + is responsible for retrying or handling the failure. + """ + pages = client._request("GET", "/wiki/pages").json() return {page.get("title", ""): page.get("sub_url", page.get("title", "")) for page in pages} @@ -161,6 +170,28 @@ def verify_wiki_page( return actual.strip() == expected_content.strip() +def _list_wiki_pages_with_retry(client: GiteaClient) -> dict[str, str]: + """List wiki pages with tenacity retry on APIError. + + The Gitea API can be briefly unavailable right after a batch of wiki + page updates. Uses the same tenacity pattern as ``api_clients`` for + exponential backoff. + """ + _logger = logging.getLogger("sync_wiki") + + @retry( + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=2, min=2, max=8), + retry=retry_if_exception_type(APIError), + before_sleep=before_sleep_log(_logger, logging.WARNING), + reraise=True, + ) + def _do_list() -> dict[str, str]: + return list_wiki_pages(client) + + return _do_list() + + def verify_wiki_integrity( client: GiteaClient, mapping: dict[str, str], @@ -176,9 +207,25 @@ def verify_wiki_integrity( 5. Page count matches Returns a list of failure messages (empty if all checks pass). + If the wiki API is temporarily unavailable (all retry attempts + fail), returns an empty list with a warning — the sync itself + already succeeded, so a transient API outage should not fail the job. """ failures: list[str] = [] - existing_pages = list_wiki_pages(client) + + try: + existing_pages = _list_wiki_pages_with_retry(client) + except APIError: + click.echo( + _( + "WARNING: Could not fetch wiki page list after retries. " + "The sync itself succeeded ({count} pages updated), but the " + "integrity check could not verify them due to a transient API issue.", + count=len(synced), + ) + ) + return [] + expected_titles = set(mapping.values()) # Check 1: Page count @@ -243,7 +290,10 @@ def main(dry_run: bool, repo: str | None, verify: bool, strict: bool) -> None: click.echo(_("Syncing {count} documentation pages to wiki...", count=len(mapping))) - existing_pages = list_wiki_pages(client) + try: + existing_pages = list_wiki_pages(client) + except APIError: + existing_pages = {} if existing_pages: click.echo(_("Found {count} existing wiki pages.", count=len(existing_pages))) @@ -302,7 +352,16 @@ def main(dry_run: bool, repo: str | None, verify: bool, strict: bool) -> None: else: click.echo(_("\nVerifying wiki pages have content...")) # Re-fetch the page list to get updated sub_urls - existing_pages = list_wiki_pages(client) + try: + existing_pages = _list_wiki_pages_with_retry(client) + except APIError: + click.echo( + _( + "WARNING: Could not re-fetch wiki page list for verification. " + "Skipping content verification due to transient API issue." + ) + ) + return failures = 0 for page_title, expected_content in sorted(synced.items()): ok = verify_wiki_page(client, page_title, expected_content, existing_pages) diff --git a/src/devx/translations.json b/src/devx/translations.json index 93444e4..c97ba4e 100644 --- a/src/devx/translations.json +++ b/src/devx/translations.json @@ -2527,6 +2527,22 @@ "ru": "ВНИМАНИЕ: Файл .taskid ({file_id}) устарел и не совпадает с именем ветки ({branch_id}). Удалите .taskid из репозитория — имя ветки — единственный источник истины.", "zh": "警告：.taskid 文件 ({file_id}) 已弃用，与分支名称 ({branch_id}) 不一致。请从仓库中删除 .taskid — 分支名称是唯一的真实来源。" }, + "WARNING: Could not fetch wiki page list after retries. The sync itself succeeded ({count} pages updated), but the integrity check could not verify them due to a transient API issue.": { + "bg": "WARNING: Could not fetch wiki page list after retries. The sync itself succeeded ({count} pages updated), but the integrity check could not verify them due to a transient API issue.", + "de": "WARNING: Could not fetch wiki page list after retries. The sync itself succeeded ({count} pages updated), but the integrity check could not verify them due to a transient API issue.", + "en": "WARNING: Could not fetch wiki page list after retries. The sync itself succeeded ({count} pages updated), but the integrity check could not verify them due to a transient API issue.", + "pl": "OSTRZEŻENIE: Nie można pobrać listy stron wiki po ponownych próbach. Sama synchronizacja zakończyła się sukcesem (zaktualizowano {count} stron), ale kontrola integralności nie mogła ich zweryfikować z powodu przejściowego problemu z API.", + "ru": "WARNING: Could not fetch wiki page list after retries. The sync itself succeeded ({count} pages updated), but the integrity check could not verify them due to a transient API issue.", + "zh": "WARNING: Could not fetch wiki page list after retries. The sync itself succeeded ({count} pages updated), but the integrity check could not verify them due to a transient API issue." + }, + "WARNING: Could not re-fetch wiki page list for verification. Skipping content verification due to transient API issue.": { + "bg": "WARNING: Could not re-fetch wiki page list for verification. Skipping content verification due to transient API issue.", + "de": "WARNING: Could not re-fetch wiki page list for verification. Skipping content verification due to transient API issue.", + "en": "WARNING: Could not re-fetch wiki page list for verification. Skipping content verification due to transient API issue.", + "pl": "OSTRZEŻENIE: Nie można ponownie pobrać listy stron wiki do weryfikacji. Pomijanie weryfikacji treści z powodu przejściowego problemu z API.", + "ru": "WARNING: Could not re-fetch wiki page list for verification. Skipping content verification due to transient API issue.", + "zh": "WARNING: Could not re-fetch wiki page list for verification. Skipping content verification due to transient API issue." + }, "WARNING: VIKUNJA_TOKEN not set — skipping task existence check. Set it in .env to enable full validation.": { "bg": "ПРЕДУПРЕЖДЕНИЕ: VIKUNJA_TOKEN не е зададен — пропускане на проверката за съществуване на задача. Задайте го в .env за пълна валидация.", "de": "WARNUNG: VIKUNJA_TOKEN nicht gesetzt — Task-Existenzprüfung übersprungen. In .env setzen für volle Validierung.", diff --git a/tests/unit/test_sync_wiki.py b/tests/unit/test_sync_wiki.py index 6fbb00b..b155965 100644 --- a/tests/unit/test_sync_wiki.py +++ b/tests/unit/test_sync_wiki.py @@ -21,6 +21,7 @@ from devx.ci.sync_wiki import ( verify_wiki_integrity, verify_wiki_page, ) +from devx.exceptions import APIError class TestEncodeContent: @@ -97,13 +98,11 @@ class TestReadDocContent: class TestListWikiPages: - def test_returns_empty_on_api_error(self) -> None: - from devx.exceptions import APIError - + def test_raises_on_api_error(self) -> None: client = MagicMock() client._request.side_effect = APIError(404, "not found") - result = list_wiki_pages(client) - assert result == {} + with pytest.raises(APIError): + list_wiki_pages(client) def test_returns_page_dict(self) -> None: client = MagicMock() @@ -284,6 +283,43 @@ class TestVerifyWikiIntegrity: failures = verify_wiki_integrity(client, mapping, synced) assert len(failures) >= 3 # count mismatch, missing FAQ, stale Stale, empty Home + def test_transient_api_failure_returns_empty(self) -> None: + """When the wiki API is unavailable after retries, integrity check + should return no failures (sync already succeeded).""" + client = MagicMock() + + # _list_wiki_pages_with_retry raises APIError (retries exhausted) + with patch("devx.ci.sync_wiki._list_wiki_pages_with_retry", side_effect=APIError(0, "timeout")): + mapping = {"index.md": "Home", "faq.md": "FAQ"} + synced = {"Home": "# Home", "FAQ": "# FAQ"} + failures = verify_wiki_integrity(client, mapping, synced) + assert failures == [] + + def test_transient_api_failure_recovers_on_retry(self) -> None: + """When the wiki API recovers after a retry, integrity check proceeds normally.""" + client = MagicMock() + pages = {"Home": "Home", "FAQ": "FAQ"} + contents = {"Home": "# Home", "FAQ": "# FAQ"} + + def mock_request(method, path, **kwargs): + resp = MagicMock() + if path == "/wiki/pages": + page_list = [{"title": t, "sub_url": s} for t, s in pages.items()] + resp.json.return_value = page_list + elif path.startswith("/wiki/page/"): + sub_url = path.replace("/wiki/page/", "") + content = contents.get(sub_url, "") + encoded = base64.b64encode(content.encode()).decode("ascii") if content else "" + resp.json.return_value = {"content_base64": encoded} + return resp + + client._request.side_effect = mock_request + + mapping = {"index.md": "Home", "faq.md": "FAQ"} + synced = {"Home": "# Home", "FAQ": "# FAQ"} + failures = verify_wiki_integrity(client, mapping, synced) + assert failures == [] + class TestMain: @patch.dict("os.environ", {"CI_GITEA_TOKEN": "tok"}) @@ -498,3 +534,41 @@ class TestMain: result = runner.invoke(main, ["--dry-run", "--strict", "--repo", "owner/repo"]) assert result.exit_code == 0 assert "Integrity check" not in result.output + + @patch.dict("os.environ", {"CI_GITEA_TOKEN": "tok"}, clear=True) + @patch("devx.ci.sync_wiki.GiteaClient") + def test_initial_list_api_error_treated_as_empty(self, mock_client_cls: MagicMock) -> None: + """When the initial page list fails, sync proceeds treating wiki as empty.""" + mock_client = MagicMock() + mock_client_cls.return_value = mock_client + with patch("devx.ci.sync_wiki.MAPPING_FILE") as mock_mapping: + mock_mapping.exists.return_value = True + with patch("devx.ci.sync_wiki.load_mapping", return_value={"index.md": "Home"}): + with patch("devx.ci.sync_wiki.read_doc_content", return_value="# Home"): + with patch("devx.ci.sync_wiki.list_wiki_pages", side_effect=APIError(0, "timeout")): + with patch("devx.ci.sync_wiki.sync_page", return_value="created"): + runner = CliRunner() + result = runner.invoke(main, ["--repo", "owner/repo"]) + assert result.exit_code == 0 + assert "Created: Home" in result.output + + @patch.dict("os.environ", {"CI_GITEA_TOKEN": "tok"}, clear=True) + @patch("devx.ci.sync_wiki.GiteaClient") + def test_verify_skips_when_refetch_fails(self, mock_client_cls: MagicMock) -> None: + """When --verify re-fetch fails after retries, verification is skipped gracefully.""" + mock_client = MagicMock() + mock_client_cls.return_value = mock_client + with patch("devx.ci.sync_wiki.MAPPING_FILE") as mock_mapping: + mock_mapping.exists.return_value = True + with patch("devx.ci.sync_wiki.load_mapping", return_value={"index.md": "Home"}): + with patch("devx.ci.sync_wiki.read_doc_content", return_value="# Home"): + with patch("devx.ci.sync_wiki.list_wiki_pages", return_value={"Home": "Home"}): + with patch("devx.ci.sync_wiki.sync_page", return_value="updated"): + with patch( + "devx.ci.sync_wiki._list_wiki_pages_with_retry", + side_effect=APIError(0, "timeout"), + ): + runner = CliRunner() + result = runner.invoke(main, ["--repo", "owner/repo", "--verify"]) + assert result.exit_code == 0 + assert "Skipping content verification" in result.output -- 2.54.0