DEVX-150: feat(setup): mirror Ansible collections from Gitea registry with auth

Co-authored-by: emil User <emil.simeonov@tutanota.com>

tests/unit/test_setup.py

@@ -14,6 +14,7 @@ from devx.tools.setup import (
     _install_pre_commit_hooks,
     _install_python_deps,
     _run,
+    _try_gitea_mirror_install,
     _verify,
     main,
 )
@@ -106,16 +107,260 @@ class TestInstallAnsibleCollections:
         with patch("devx.tools.setup.Path") as mock_path:
             mock_path.return_value.exists.return_value = True
             mock_path.return_value.__str__ = lambda _: str(req)
+            mock_path.return_value.read_text = lambda: req.read_text()
             _install_ansible_collections(".venv/bin")
         mock_run.assert_called_once()
-        args = mock_run.call_args[0][0]
-        assert "--no-cache" in args, "ansible-galaxy must use --no-cache to avoid concurrent cache corruption"
 
     @patch("devx.tools.setup._run")
     def test_skips_when_no_requirements(self, mock_run: MagicMock) -> None:
         _install_ansible_collections(".venv/bin")
         mock_run.assert_not_called()
 
+    @patch("tenacity.nap.time.sleep")
+    @patch("devx.tools.setup.shutil.which", return_value="/usr/local/bin/ansible-galaxy")
+    @patch("devx.tools.setup._run")
+    def test_retries_on_transient_failure(
+        self, mock_run: MagicMock, mock_which: MagicMock, mock_sleep: MagicMock, tmp_path: Path
+    ) -> None:
+        """ansible-galaxy install should retry on transient network errors."""
+        import subprocess as _subprocess
+
+        req = tmp_path / "ansible" / "requirements.yml"
+        req.parent.mkdir(parents=True)
+        req.write_text("collections: []")
+        # First call fails (timeout), second succeeds
+        mock_run.side_effect = [
+            _subprocess.CalledProcessError(1, ["ansible-galaxy", "collection", "install"]),
+            None,
+        ]
+        with patch("devx.tools.setup.Path") as mock_path:
+            mock_path.return_value.exists.return_value = True
+            mock_path.return_value.__str__ = lambda _: str(req)
+            mock_path.return_value.read_text = lambda: req.read_text()
+            _install_ansible_collections(".venv/bin")
+        assert mock_run.call_count == 2
+
+    @patch("tenacity.nap.time.sleep")
+    @patch("devx.tools.setup.shutil.which", return_value="/usr/local/bin/ansible-galaxy")
+    @patch("devx.tools.setup._run")
+    def test_exhausts_retries_then_raises(
+        self, mock_run: MagicMock, mock_which: MagicMock, mock_sleep: MagicMock, tmp_path: Path
+    ) -> None:
+        """After 3 attempts, the error should propagate."""
+        import subprocess as _subprocess
+
+        req = tmp_path / "ansible" / "requirements.yml"
+        req.parent.mkdir(parents=True)
+        req.write_text("collections: []")
+        mock_run.side_effect = _subprocess.CalledProcessError(1, ["ansible-galaxy"])
+        with patch("devx.tools.setup.Path") as mock_path:
+            mock_path.return_value.exists.return_value = True
+            mock_path.return_value.__str__ = lambda _: str(req)
+            mock_path.return_value.read_text = lambda: req.read_text()
+            with pytest.raises(_subprocess.CalledProcessError):
+                _install_ansible_collections(".venv/bin")
+        assert mock_run.call_count == 3
+
+    @patch("devx.tools.setup._try_gitea_mirror_install", return_value=True)
+    @patch("devx.tools.setup.shutil.which", return_value="/usr/local/bin/ansible-galaxy")
+    @patch("devx.tools.setup._run")
+    def test_mirror_install_skips_galaxy_fallback(
+        self, mock_run: MagicMock, mock_which: MagicMock, mock_mirror: MagicMock, tmp_path: Path
+    ) -> None:
+        """When mirror install succeeds, galaxy fallback is not called."""
+        req = tmp_path / "ansible" / "requirements.yml"
+        req.parent.mkdir(parents=True)
+        req.write_text("collections: []")
+        with patch("devx.tools.setup.Path") as mock_path:
+            mock_path.return_value.exists.return_value = True
+            mock_path.return_value.__str__ = lambda _: str(req)
+            mock_path.return_value.read_text = lambda: req.read_text()
+            _install_ansible_collections(".venv/bin")
+        # _run should not be called because mirror install returns True
+        mock_run.assert_not_called()
+
+
+class TestTryGiteaMirrorInstall:
+    """Tests for _try_gitea_mirror_install — Gitea mirror with auth + fallback."""
+
+    _GITEA_URL = "https://git.example.com/api/packages/org/generic/ansible-collections/1.0.0/ansible-posix-1.0.0.tar.gz"
+
+    @patch.dict(os.environ, {}, clear=True)
+    def test_no_url_entries_returns_false(self, tmp_path: Path) -> None:
+        """Requirements without type: url entries should return False."""
+        req = tmp_path / "requirements.yml"
+        req.write_text("collections:\n  - name: ansible.posix\n    version: '1.0.0'\n")
+        result = _try_gitea_mirror_install("ansible-galaxy", req)
+        assert result is False
+
+    @patch.dict(os.environ, {}, clear=True)
+    def test_no_token_returns_false(self, tmp_path: Path) -> None:
+        """No Gitea token set → return False to fall back to galaxy."""
+        req = tmp_path / "requirements.yml"
+        req.write_text(
+            "collections:\n"
+            "  - name: ansible.posix\n"
+            "    version: '1.0.0'\n"
+            "    type: url\n"
+            f"    source: '{self._GITEA_URL}'\n"
+        )
+        result = _try_gitea_mirror_install("ansible-galaxy", req)
+        assert result is False
+
+    @patch.dict(os.environ, {"CI_GITEA_TOKEN": "tok123"}, clear=True)
+    def test_yaml_parse_error_returns_false(self, tmp_path: Path) -> None:
+        """Malformed YAML → return False."""
+        req = tmp_path / "requirements.yml"
+        req.write_text("not: valid: yaml: [[")
+        result = _try_gitea_mirror_install("ansible-galaxy", req)
+        assert result is False
+
+    @patch("devx.tools.setup._run")
+    @patch("urllib.request.urlopen")
+    @patch.dict(os.environ, {"CI_GITEA_TOKEN": "tok123"}, clear=True)
+    def test_successful_mirror_install(self, mock_urlopen: MagicMock, mock_run: MagicMock, tmp_path: Path) -> None:
+        """Valid URL entries + token → downloads with auth and installs offline."""
+        req = tmp_path / "requirements.yml"
+        req.write_text(
+            f"collections:\n"
+            f"  - name: ansible.posix\n"
+            f"    version: '1.0.0'\n"
+            f"    type: url\n"
+            f"    source: '{self._GITEA_URL}'\n"
+        )
+        mock_resp = MagicMock()
+        mock_resp.read.return_value = b"fake-tarball"
+        mock_resp.__enter__ = lambda _: mock_resp
+        mock_resp.__exit__ = lambda *a: None
+        mock_urlopen.return_value = mock_resp
+
+        result = _try_gitea_mirror_install("ansible-galaxy", req)
+        assert result is True
+
+        # Verify auth header was added
+        call_args = mock_urlopen.call_args[0][0]
+        assert call_args.get_header("Authorization") == "token tok123"
+
+        # Verify offline install was called
+        install_cmd = mock_run.call_args[0][0]
+        assert "collection" in install_cmd
+        assert "install" in install_cmd
+        assert "--offline" in install_cmd
+
+    @patch("urllib.request.urlopen")
+    @patch.dict(os.environ, {"CI_GITEA_TOKEN": "tok123"}, clear=True)
+    def test_download_failure_returns_false(self, mock_urlopen: MagicMock, tmp_path: Path) -> None:
+        """Download failure → return False to fall back to galaxy."""
+        req = tmp_path / "requirements.yml"
+        req.write_text(
+            f"collections:\n"
+            f"  - name: ansible.posix\n"
+            f"    version: '1.0.0'\n"
+            f"    type: url\n"
+            f"    source: '{self._GITEA_URL}'\n"
+        )
+        mock_urlopen.side_effect = Exception("401 Unauthorized")
+
+        result = _try_gitea_mirror_install("ansible-galaxy", req)
+        assert result is False
+
+    @patch("devx.tools.setup._run")
+    @patch("urllib.request.urlopen")
+    @patch.dict(os.environ, {"CI_GITEA_API_TOKEN": "tok456"}, clear=True)
+    def test_prefers_api_token_over_legacy(self, mock_urlopen: MagicMock, mock_run: MagicMock, tmp_path: Path) -> None:
+        """CI_GITEA_API_TOKEN takes priority over CI_GITEA_TOKEN."""
+        req = tmp_path / "requirements.yml"
+        req.write_text(
+            f"collections:\n"
+            f"  - name: ansible.posix\n"
+            f"    version: '1.0.0'\n"
+            f"    type: url\n"
+            f"    source: '{self._GITEA_URL}'\n"
+        )
+        mock_resp = MagicMock()
+        mock_resp.read.return_value = b"fake-tarball"
+        mock_resp.__enter__ = lambda _: mock_resp
+        mock_resp.__exit__ = lambda *a: None
+        mock_urlopen.return_value = mock_resp
+
+        result = _try_gitea_mirror_install("ansible-galaxy", req)
+        assert result is True
+
+        call_args = mock_urlopen.call_args[0][0]
+        assert call_args.get_header("Authorization") == "token tok456"
+
+    @patch("devx.tools.setup._run")
+    @patch("urllib.request.urlopen")
+    @patch.dict(os.environ, {"DEVELOPER_GITEA_API_TOKEN": "tok789"}, clear=True)
+    def test_developer_token_fallback(self, mock_urlopen: MagicMock, mock_run: MagicMock, tmp_path: Path) -> None:
+        """DEVELOPER_GITEA_API_TOKEN is used when CI tokens are absent."""
+        req = tmp_path / "requirements.yml"
+        req.write_text(
+            f"collections:\n"
+            f"  - name: ansible.posix\n"
+            f"    version: '1.0.0'\n"
+            f"    type: url\n"
+            f"    source: '{self._GITEA_URL}'\n"
+        )
+        mock_resp = MagicMock()
+        mock_resp.read.return_value = b"fake-tarball"
+        mock_resp.__enter__ = lambda _: mock_resp
+        mock_resp.__exit__ = lambda *a: None
+        mock_urlopen.return_value = mock_resp
+
+        result = _try_gitea_mirror_install("ansible-galaxy", req)
+        assert result is True
+
+        call_args = mock_urlopen.call_args[0][0]
+        assert call_args.get_header("Authorization") == "token tok789"
+
+    @patch("devx.tools.setup._run")
+    @patch("urllib.request.urlopen")
+    @patch.dict(os.environ, {"CI_GITEA_TOKEN": "tok123"}, clear=True)
+    def test_non_gitea_url_passed_through(self, mock_urlopen: MagicMock, mock_run: MagicMock, tmp_path: Path) -> None:
+        """URL entries not pointing to /api/packages/ are kept as-is (no download)."""
+        external_url = "https://galaxy.ansible.com/download/ansible-posix-1.0.0.tar.gz"
+        req = tmp_path / "requirements.yml"
+        req.write_text(
+            f"collections:\n"
+            f"  - name: ansible.posix\n"
+            f"    version: '1.0.0'\n"
+            f"    type: url\n"
+            f"    source: '{external_url}'\n"
+        )
+        # Should not call urlopen since the URL is not a Gitea package URL
+        result = _try_gitea_mirror_install("ansible-galaxy", req)
+        assert result is True
+        mock_urlopen.assert_not_called()
+
+    @patch("devx.tools.setup._run")
+    @patch("urllib.request.urlopen")
+    @patch.dict(os.environ, {"CI_GITEA_TOKEN": "tok123"}, clear=True)
+    def test_mixed_entries_gitea_and_non_gitea(
+        self, mock_urlopen: MagicMock, mock_run: MagicMock, tmp_path: Path
+    ) -> None:
+        """Mix of Gitea URL entries and regular galaxy entries."""
+        req = tmp_path / "requirements.yml"
+        req.write_text(
+            f"collections:\n"
+            f"  - name: ansible.posix\n"
+            f"    version: '1.0.0'\n"
+            f"    type: url\n"
+            f"    source: '{self._GITEA_URL}'\n"
+            f"  - name: community.general\n"
+            f"    version: '13.0.0'\n"
+        )
+        mock_resp = MagicMock()
+        mock_resp.read.return_value = b"fake-tarball"
+        mock_resp.__enter__ = lambda _: mock_resp
+        mock_resp.__exit__ = lambda *a: None
+        mock_urlopen.return_value = mock_resp
+
+        result = _try_gitea_mirror_install("ansible-galaxy", req)
+        assert result is True
+        # Only the Gitea URL entry should trigger a download
+        mock_urlopen.assert_called_once()
+
 
 class TestConfigureTeaLogin:
     @patch("devx.tools.setup.shutil.which", return_value=None)