From deb93992329167c6ac1a13becb2084d3b14d0d31 Mon Sep 17 00:00:00 2001 From: emil Date: Wed, 12 Aug 2026 23:51:48 +0200 Subject: [PATCH] fix: add GitHub mirror fallback for actionlint and vale downloads CI image builds failed repeatedly because GitHub releases dropped connections (RemoteDisconnected) on all 5 retry attempts. Add ghproxy.com as a fallback URL for _download_and_extract_tarball so the build can fall through to a mirror when GitHub is flaky. - Add fallback_urls parameter to _download_and_extract_tarball - Add ghproxy.com mirror fallback for actionlint and vale - Add tests for fallback success and all-URLs-fail paths DEVX-159 Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- src/devx/tools/install_tools.py | 56 ++++++++++++++++++++------------ tests/unit/test_install_tools.py | 44 +++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 21 deletions(-) diff --git a/src/devx/tools/install_tools.py b/src/devx/tools/install_tools.py index 29e775a..1b35d33 100644 --- a/src/devx/tools/install_tools.py +++ b/src/devx/tools/install_tools.py @@ -131,26 +131,37 @@ def _download_with_fallback(urls: list[str], binary_name: str) -> Path: raise click.ClickException(f"Failed to download {binary_name} from all URLs: {'; '.join(errors)}") -def _download_and_extract_tarball(url: str, binary_name: str) -> Path: +def _download_and_extract_tarball(url: str, binary_name: str, *, fallback_urls: list[str] | None = None) -> Path: """Download a tarball, extract the binary, and install it to TARGET_DIR. - Returns the path to the installed binary. + Returns the path to the installed binary. Falls back to ``fallback_urls`` + if the primary ``url`` fails all retries. """ target_dir = _ensure_target_dir() dest = target_dir / binary_name - with tempfile.TemporaryDirectory() as tmpdir: - tarball = Path(tmpdir) / "archive.tar.gz" - _download(url, tarball) - with tarfile.open(tarball, "r:gz") as tar: - tar.extractall(tmpdir) # nosec B202 - # Find the binary in the extracted tree - extracted = Path(tmpdir).rglob(binary_name) - found = next(extracted, None) - if found is None: - raise click.ClickException(f"Binary {binary_name} not found in archive from {url}") - shutil.copy2(found, dest) - dest.chmod(0o755) - return dest + urls = [url, *(fallback_urls or [])] + errors: list[str] = [] + for try_url in urls: + with tempfile.TemporaryDirectory() as tmpdir: + tarball = Path(tmpdir) / "archive.tar.gz" + try: + _download(try_url, tarball) + except Exception as exc: # noqa: BLE001 + errors.append(f"{try_url}: {exc}") + click.echo(f" {binary_name}: fallback — {exc}") + continue + with tarfile.open(tarball, "r:gz") as tar: + tar.extractall(tmpdir) # nosec B202 + # Find the binary in the extracted tree + extracted = Path(tmpdir).rglob(binary_name) + found = next(extracted, None) + if found is None: + errors.append(f"{try_url}: binary not found in archive") + continue + shutil.copy2(found, dest) + dest.chmod(0o755) + return dest + raise click.ClickException(f"Failed to download {binary_name} from all URLs: {'; '.join(errors)}") def _download_binary(url: str, binary_name: str) -> Path: @@ -178,11 +189,12 @@ def install_actionlint() -> bool: click.echo("actionlint: already installed") return True arch = _arch() - url = ( - f"https://github.com/rhysd/actionlint/releases/download/" - f"v{ACTIONLINT_VERSION}/actionlint_{ACTIONLINT_VERSION}_linux_{arch}.tar.gz" + path = ( + f"rhysd/actionlint/releases/download/v{ACTIONLINT_VERSION}/actionlint_{ACTIONLINT_VERSION}_linux_{arch}.tar.gz" ) - dest = _download_and_extract_tarball(url, "actionlint") + url = f"https://github.com/{path}" + fallback = [f"https://ghproxy.com/{path}"] + dest = _download_and_extract_tarball(url, "actionlint", fallback_urls=fallback) click.echo(f"actionlint: installed to {dest}") return True @@ -276,8 +288,10 @@ def install_vale() -> bool: return True machine = platform.machine().lower() arch = "64-bit" if machine in {"x86_64", "amd64"} else "arm64" - url = f"https://github.com/errata-ai/vale/releases/download/v{VALE_VERSION}/vale_{VALE_VERSION}_Linux_{arch}.tar.gz" - dest = _download_and_extract_tarball(url, "vale") + path = f"errata-ai/vale/releases/download/v{VALE_VERSION}/vale_{VALE_VERSION}_Linux_{arch}.tar.gz" + url = f"https://github.com/{path}" + fallback = [f"https://ghproxy.com/{path}"] + dest = _download_and_extract_tarball(url, "vale", fallback_urls=fallback) click.echo(f"vale: installed to {dest}") return True diff --git a/tests/unit/test_install_tools.py b/tests/unit/test_install_tools.py index 96c7dd4..3b32c79 100644 --- a/tests/unit/test_install_tools.py +++ b/tests/unit/test_install_tools.py @@ -201,6 +201,50 @@ class TestDownloadAndExtractTarball: with pytest.raises(ClickException, match="not found in archive"): install_tools._download_and_extract_tarball("https://example.com/actionlint.tar.gz", "actionlint") + def test_fallback_url_succeeds(self, tmp_path: Path) -> None: + import io + import tarfile + + tarball_path = tmp_path / "archive.tar.gz" + binary_content = b"fake binary" + with tarfile.open(tarball_path, "w:gz") as tar: + info = tarfile.TarInfo(name="actionlint") + info.size = len(binary_content) + tar.addfile(info, io.BytesIO(binary_content)) + + target_dir = tmp_path / "bin" + target_dir.mkdir() + tarball_bytes = tarball_path.read_bytes() + + def fake_download(url: str, dest: Path) -> None: + if "primary" in url: + raise OSError("connection refused") + Path(dest).write_bytes(tarball_bytes) + + with patch.object(install_tools, "TARGET_DIR", target_dir): + with patch.object(install_tools, "_download", side_effect=fake_download): + result = install_tools._download_and_extract_tarball( + "https://primary.com/actionlint.tar.gz", + "actionlint", + fallback_urls=["https://fallback.com/actionlint.tar.gz"], + ) + + assert result == target_dir / "actionlint" + assert result.read_bytes() == binary_content + + def test_all_urls_fail(self, tmp_path: Path) -> None: + target_dir = tmp_path / "bin" + target_dir.mkdir() + + with patch.object(install_tools, "TARGET_DIR", target_dir): + with patch.object(install_tools, "_download", side_effect=OSError("connection refused")): + with pytest.raises(ClickException, match="Failed to download"): + install_tools._download_and_extract_tarball( + "https://primary.com/actionlint.tar.gz", + "actionlint", + fallback_urls=["https://fallback.com/actionlint.tar.gz"], + ) + class TestInstallActionlint: def test_already_installed(self) -> None: -- 2.54.0