From 9d0e4cf42956bd307205578f57975f85a87ae8f4 Mon Sep 17 00:00:00 2001 From: kireto Date: Sat, 19 Sep 2026 20:19:22 +0000 Subject: [PATCH] DEVX-173: fix(ci): resolve dep-PR container digest via registry v2 API --- docs/specs/DEVX-173.md | 49 ++++++++++++++++ src/devx/ci/create_dependency_pr.py | 77 ++++++++++++++++++------- src/devx/translations.json | 22 ++++--- tests/unit/test_create_dependency_pr.py | 74 ++++++++++++++++++------ 4 files changed, 178 insertions(+), 44 deletions(-) create mode 100644 docs/specs/DEVX-173.md diff --git a/docs/specs/DEVX-173.md b/docs/specs/DEVX-173.md new file mode 100644 index 0000000..05c4fc8 --- /dev/null +++ b/docs/specs/DEVX-173.md @@ -0,0 +1,49 @@ +# DEVX-173: Resolve container digest via registry v2 API + +## Problem + +`resolve_container_digest` returns the sha256 of the `manifest.json` +blob listed by the Gitea packages API (`/packages//container/ +//files`). That blob digest is NOT the OCI manifest digest — +`docker pull repo@sha256:` fails with "not found." Production +deploy run 6251 died pulling `sso-bridge@sha256:5ca9...` which the dep +PR had recorded in `deploy/sso-bridge-release.json`. The registry serves +0.4.1 as `sha256:c0212ed1...` — different digest entirely. + +## Approach + +REQ-1: Keep the packages-API call as the existence check (it has the +404-retry semantics needed for the publish race) but resolve the +pullable digest via the registry v2 API: `GET /v2/token` with +`scope=repository:/:pull` (basic-auth with the Gitea +token), then `GET /v2///manifests/` with OCI/Docker +manifest Accept headers and read `Docker-Content-Digest`. Registry base +URL is derived from `GITEA_API_URL` (strip `/api/v1`). + +REQ-2: If the v2 digest lookup fails (non-2xx, missing header), fail +closed with a ClickException — never record a blob sha256 as a pullable +digest. + +## Test Plan + +- Mocked v2 token + manifest endpoints return digest; recorded value is + the `Docker-Content-Digest` header. +- 404 retry semantics on the packages-API existence check unchanged. +- Missing digest header / non-2xx manifest response → ClickException. +- Unit tests cover token request scope and Accept headers. + +## Deploy Plan + +devx release tag; the sso-bridge dep-PR pin bump follows in its own PR. +The wrong digest already recorded in infra's manifest is corrected by +re-running the dep-PR with the fixed version. + +## Rollback Plan + +Revert; dep-PR records the (unpullable) blob digest again — deploys must +then use tag pulls until a corrected manifest lands. + +## Acceptance Criteria + +- [x] REQ-1: Digest resolved from `Docker-Content-Digest` via v2 API. +- [x] REQ-2: Lookup failures fail closed; no blob-sha256 fallback. diff --git a/src/devx/ci/create_dependency_pr.py b/src/devx/ci/create_dependency_pr.py index 93bb5d8..70c8156 100644 --- a/src/devx/ci/create_dependency_pr.py +++ b/src/devx/ci/create_dependency_pr.py @@ -96,18 +96,25 @@ def update_pinned_version(file_path: str, package: str, old_version: str, new_ve return changed +_MANIFEST_ACCEPT = ( + "application/vnd.oci.image.index.v1+json, " + "application/vnd.docker.distribution.manifest.list.v2+json, " + "application/vnd.oci.image.manifest.v1+json, " + "application/vnd.docker.distribution.manifest.v2+json" +) + + +# Implements: REQ-1 — existence check via packages API (keeps the 404-retry +# semantics for the publish race); the pullable digest is then resolved via +# the registry v2 API's Docker-Content-Digest header. def resolve_container_digest(api_url: str, owner: str, name: str, tag: str, token: str, timeout_s: int = 0) -> str: - """Resolve the OCI digest for a container image tag via the packages API. + """Verify the container tag exists, then resolve its pullable OCI digest. - Implements REQ-1: dependency PRs must only be opened after the producer - artifact exists — this raises ClickException when the tag is missing or - the registry call fails, so the PR is never opened against an artifact - that has not been published. The sha256 of the stored ``manifest.json`` - blob is the manifest content digest (what ``docker pull`` reports). - - Implements REQ-2: ``timeout_s`` > 0 retries the lookup every 15s until - the deadline — the dep-PR step races the producer's image-build - workflow, which pushes the tag concurrently. + The packages API proves the tag was published (and 404s while the + producer's image-build workflow races us — ``timeout_s`` retries every + 15s). The packages-API ``manifest.json`` blob sha256 is NOT pullable + via ``repo@sha256:...``, so the digest comes from the registry v2 + ``Docker-Content-Digest`` header instead. """ url = f"{api_url}/packages/{owner}/container/{name}/{tag}/files" headers = {"Authorization": f"token {token}"} @@ -150,17 +157,47 @@ def resolve_container_digest(api_url: str, owner: str, name: str, tag: str, toke ) ) from e break - for f in resp.json(): - if f.get("name") == "manifest.json" and f.get("sha256"): - return f"sha256:{f['sha256']}" - raise click.ClickException( - _( - "Registry returned no manifest blob for {owner}/{name}:{tag}.", - owner=owner, - name=name, - tag=tag, + return _resolve_registry_digest(api_url, owner, name, tag, token) + + +# Implements: REQ-2 — v2 digest resolution fails closed: non-2xx, missing +# token, or absent Docker-Content-Digest all raise; a blob sha256 is never +# recorded as a pullable digest. +def _resolve_registry_digest(api_url: str, owner: str, name: str, tag: str, token: str) -> str: + registry = api_url.removesuffix("/api/v1").removesuffix("/") + repo = f"{owner}/{name}" + try: + tok_resp = requests.get( # nosec B310 + f"{registry}/v2/token", + params={"service": "container_registry", "scope": f"repository:{repo}:pull"}, + auth=("ci", token), + timeout=30, ) - ) + tok_resp.raise_for_status() + bearer = tok_resp.json().get("token", "") + man_resp = requests.get( # nosec B310 + f"{registry}/v2/{repo}/manifests/{tag}", + headers={"Authorization": f"Bearer {bearer}", "Accept": _MANIFEST_ACCEPT}, + timeout=30, + ) + man_resp.raise_for_status() + except requests.RequestException as e: + status = getattr(getattr(e, "response", None), "status_code", "?") + raise click.ClickException( + _( + "Registry v2 digest lookup failed for {repo}:{tag} (HTTP {status}): {error}", + repo=repo, + tag=tag, + status=status, + error=e, + ) + ) from e + digest = man_resp.headers.get("Docker-Content-Digest", "") + if not digest: + raise click.ClickException( + _("Registry returned no Docker-Content-Digest for {repo}:{tag}.", repo=repo, tag=tag) + ) + return digest def update_manifest(file_path: str, section: str, fields: dict[str, str]) -> bool: diff --git a/src/devx/translations.json b/src/devx/translations.json index b93fb02..d6f8303 100644 --- a/src/devx/translations.json +++ b/src/devx/translations.json @@ -2967,13 +2967,21 @@ "ru": "Ошибка поиска в реестре для {owner}/{name}:{tag}: {error}", "zh": "注册表查询 {owner}/{name}:{tag} 失败：{error}" }, - "Registry returned no manifest blob for {owner}/{name}:{tag}.": { - "bg": "Регистърът не върна manifest blob за {owner}/{name}:{tag}.", - "de": "Registry hat keinen Manifest-Blob für {owner}/{name}:{tag} zurückgegeben.", - "en": "Registry returned no manifest blob for {owner}/{name}:{tag}.", - "pl": "Rejestr nie zwrócił blobu manifestu dla {owner}/{name}:{tag}.", - "ru": "Реестр не вернул blob манифеста для {owner}/{name}:{tag}.", - "zh": "注册表未返回 {owner}/{name}:{tag} 的清单 blob。" + "Registry returned no Docker-Content-Digest for {repo}:{tag}.": { + "bg": "Регистърът не върна Docker-Content-Digest за {repo}:{tag}.", + "de": "Registry hat keinen Docker-Content-Digest für {repo}:{tag} zurückgegeben.", + "en": "Registry returned no Docker-Content-Digest for {repo}:{tag}.", + "pl": "Rejestr nie zwrócił Docker-Content-Digest dla {repo}:{tag}.", + "ru": "Реестр не вернул Docker-Content-Digest для {repo}:{tag}.", + "zh": "注册表未返回 {repo}:{tag} 的 Docker-Content-Digest。" + }, + "Registry v2 digest lookup failed for {repo}:{tag} (HTTP {status}): {error}": { + "bg": "Неуспешна заявка за дайджест към регистър v2 за {repo}:{tag} (HTTP {status}): {error}", + "de": "Registry-v2-Digest-Abfrage für {repo}:{tag} fehlgeschlagen (HTTP {status}): {error}", + "en": "Registry v2 digest lookup failed for {repo}:{tag} (HTTP {status}): {error}", + "pl": "Zapytanie o skrót do rejestru v2 dla {repo}:{tag} nie powiodło się (HTTP {status}): {error}", + "ru": "Ошибка запроса дайджеста к реестру v2 для {repo}:{tag} (HTTP {status}): {error}", + "zh": "注册表 v2 摘要查询 {repo}:{tag} 失败 (HTTP {status})：{error}" }, "Regular merge commit — running all post-merge jobs.": { "bg": "Обикновен merge комит — изпълняват се всички post-merge задачи.", diff --git a/tests/unit/test_create_dependency_pr.py b/tests/unit/test_create_dependency_pr.py index 181ddde..e45b982 100644 --- a/tests/unit/test_create_dependency_pr.py +++ b/tests/unit/test_create_dependency_pr.py @@ -200,23 +200,39 @@ class TestCreateVikunjaTask: assert mock_client.create_task.call_args.args[0] == 3 -class TestResolveContainerDigest: - """REQ-1: pre-PR artifact verification via the packages API.""" +def _v2_mocks(digest: str = "sha256:deadbeef") -> list[MagicMock]: + """Token + manifest responses for the registry v2 digest lookup.""" + tok = MagicMock() + tok.raise_for_status = MagicMock() + tok.json.return_value = {"token": "bearer-tok"} + man = MagicMock() + man.raise_for_status = MagicMock() + man.headers = {"Docker-Content-Digest": digest} + return [tok, man] - def test_returns_digest_from_manifest_blob(self) -> None: + +class TestResolveContainerDigest: + """REQ-1: existence check via packages API; digest via registry v2.""" + + def test_returns_digest_from_registry_v2(self) -> None: from devx.ci.create_dependency_pr import resolve_container_digest - mock_resp = MagicMock() - mock_resp.raise_for_status = MagicMock() - mock_resp.json.return_value = [ - {"name": "sha256_layer", "sha256": "abc"}, - {"name": "manifest.json", "sha256": "deadbeef"}, - ] - with patch("devx.ci.create_dependency_pr.requests.get", return_value=mock_resp): + files = MagicMock() + files.raise_for_status = MagicMock() + files.json.return_value = [{"name": "manifest.json", "sha256": "blob-not-pullable"}] + with patch( + "devx.ci.create_dependency_pr.requests.get", + side_effect=[files, *_v2_mocks()], + ) as mock_get: digest = resolve_container_digest( "https://git.example.com/api/v1", "oblachno", "sso-bridge", "0.9.1", "tok" ) assert digest == "sha256:deadbeef" + man_call = mock_get.call_args_list[2] + assert "manifests/0.9.1" in man_call.args[0] + assert "oci.image.index" in man_call.kwargs["headers"]["Accept"] + tok_call = mock_get.call_args_list[1] + assert tok_call.kwargs["params"]["scope"] == "repository:oblachno/sso-bridge:pull" def test_raises_when_version_missing(self) -> None: import requests @@ -231,14 +247,38 @@ class TestResolveContainerDigest: with pytest.raises(click.ClickException, match="unpublished artifact"): resolve_container_digest("https://git.example.com/api/v1", "oblachno", "sso-bridge", "9.9.9", "tok") - def test_raises_when_no_manifest_blob(self) -> None: + def test_raises_when_v2_digest_missing(self) -> None: from devx.ci.create_dependency_pr import resolve_container_digest - mock_resp = MagicMock() - mock_resp.raise_for_status = MagicMock() - mock_resp.json.return_value = [{"name": "sha256_layer", "sha256": "abc"}] - with patch("devx.ci.create_dependency_pr.requests.get", return_value=mock_resp): - with pytest.raises(click.ClickException, match="no manifest blob"): + files = MagicMock() + files.raise_for_status = MagicMock() + files.json.return_value = [] + tok, man = _v2_mocks(digest="") + with patch( + "devx.ci.create_dependency_pr.requests.get", + side_effect=[files, tok, man], + ): + with pytest.raises(click.ClickException, match="no Docker-Content-Digest"): + resolve_container_digest("https://git.example.com/api/v1", "oblachno", "sso-bridge", "0.9.1", "tok") + + def test_raises_when_v2_manifest_request_fails(self) -> None: + import requests + + from devx.ci.create_dependency_pr import resolve_container_digest + + files = MagicMock() + files.raise_for_status = MagicMock() + files.json.return_value = [] + tok, _ = _v2_mocks() + err = requests.HTTPError("500") + err.response = MagicMock(status_code=500) + man = MagicMock() + man.raise_for_status.side_effect = err + with patch( + "devx.ci.create_dependency_pr.requests.get", + side_effect=[files, tok, man], + ): + with pytest.raises(click.ClickException, match="v2 digest lookup failed"): resolve_container_digest("https://git.example.com/api/v1", "oblachno", "sso-bridge", "0.9.1", "tok") def test_raises_on_connection_error(self) -> None: @@ -635,7 +675,7 @@ class TestBranchCreation: ok.raise_for_status = MagicMock() ok.json.return_value = [{"name": "manifest.json", "sha256": "cafe"}] with ( - patch("devx.ci.create_dependency_pr.requests.get", side_effect=[fail, ok]), + patch("devx.ci.create_dependency_pr.requests.get", side_effect=[fail, ok, *_v2_mocks("sha256:cafe")]), patch("devx.ci.create_dependency_pr.time.sleep"), ): digest = resolve_container_digest(