From c6d75ea118c18bc42e631f2bdd7d7f532f80359a Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Fri, 20 Nov 2026 14:20:26 -0800 Subject: [PATCH] walk pages of releases up to 3, support single version fetch --- src/documents/tests/test_api_release_notes.py | 111 ++++++++++++-- src/documents/views.py | 138 ++++++++++++------ 2 files changed, 196 insertions(+), 53 deletions(-) diff --git a/src/documents/tests/test_api_release_notes.py b/src/documents/tests/test_api_release_notes.py index 39d2ff6ec9..0356e4c50c 100644 --- a/src/documents/tests/test_api_release_notes.py +++ b/src/documents/tests/test_api_release_notes.py @@ -9,7 +9,15 @@ from rest_framework.test import APIClient from paperless import version REPO_URL = "https://api.github.com/repos/paperless-ngx/paperless-ngx" -RELEASES_URL = f"{REPO_URL}/releases?per_page=10" +RELEASES_URL = f"{REPO_URL}/releases?per_page=10&page=1" + + +def _page_url(page: int) -> str: + return f"{REPO_URL}/releases?per_page=10&page={page}" + + +def _tag_url(version_str: str) -> str: + return f"{REPO_URL}/releases/tags/v{version_str}" def _release(tag: str, **kwargs) -> dict: @@ -56,6 +64,15 @@ class TestApiReleaseNotes: ) return httpx_mock + @pytest.fixture + def current_release(self, httpx_mock: HTTPXMock) -> HTTPXMock: + httpx_mock.add_response( + url=_tag_url(CURRENT), + match_headers={"Accept": "application/vnd.github.html+json"}, + json=_release(f"v{CURRENT}"), + ) + return httpx_mock + @pytest.fixture def stars(self, httpx_mock: HTTPXMock) -> HTTPXMock: httpx_mock.add_response(url=REPO_URL, json={"stargazers_count": 1234}) @@ -91,26 +108,98 @@ class TestApiReleaseNotes: ) @pytest.mark.usefixtures("stars") - def test_without_since_returns_current_only( + def test_without_since_fetches_current_only( self, rest_api_client: APIClient, - releases: HTTPXMock, + current_release: HTTPXMock, ) -> None: + """ + GIVEN: + - A GitHub release for the running version + WHEN: + - Release notes are requested without `since` + THEN: + - Only that release is fetched, by tag, and returned + """ response = rest_api_client.get(self.ENDPOINT) assert response.status_code == status.HTTP_200_OK assert [r["version"] for r in response.data["releases"]] == [CURRENT] + assert len(current_release.get_requests(url=_tag_url(CURRENT))) == 1 @pytest.mark.usefixtures("stars") + def test_since_pages_back_to_since( + self, + rest_api_client: APIClient, + httpx_mock: HTTPXMock, + ) -> None: + """ + GIVEN: + - More releases since `since` than fit on one page + WHEN: + - Release notes are requested since that version + THEN: + - GitHub is paged until `since` is reached and all releases are returned + """ + major, minor, _ = version.__version__ + page_one = [f"{major}.{minor - 1}.{patch}" for patch in range(10, 0, -1)] + httpx_mock.add_response( + url=_page_url(1), + json=[_release(f"v{v}") for v in page_one], + ) + httpx_mock.add_response( + url=_page_url(2), + json=[_release(f"v{PREVIOUS}"), _release(f"v{OLDER}")], + ) + + response = rest_api_client.get(self.ENDPOINT, {"since": OLDER}) + + assert [r["version"] for r in response.data["releases"]] == [ + *page_one, + PREVIOUS, + ] + + @pytest.mark.usefixtures("stars") + def test_since_paging_is_capped( + self, + rest_api_client: APIClient, + httpx_mock: HTTPXMock, + ) -> None: + """ + GIVEN: + - Full pages of releases that never reach `since` + WHEN: + - Release notes are requested since a very old version + THEN: + - Paging stops after the page cap + """ + major, minor, _ = version.__version__ + for page in range(1, 4): + httpx_mock.add_response( + url=_page_url(page), + json=[ + _release(f"v{major}.{minor}.{page * 100 + patch}") + for patch in range(10) + ], + ) + + rest_api_client.get(self.ENDPOINT, {"since": "0.1.0"}) + + assert len(httpx_mock.get_requests()) == 4 # 3 pages + stars + + @pytest.mark.usefixtures("stars") + @pytest.mark.usefixtures("current_release") def test_releases_are_cached( self, rest_api_client: APIClient, releases: HTTPXMock, ) -> None: - rest_api_client.get(self.ENDPOINT, {"since": OLDER}) - rest_api_client.get(self.ENDPOINT) + for _ in range(2): + rest_api_client.get(self.ENDPOINT, {"since": OLDER}) + rest_api_client.get(self.ENDPOINT) assert len(releases.get_requests(url=RELEASES_URL)) == 1 + assert len(releases.get_requests(url=_tag_url(CURRENT))) == 1 assert len(releases.get_requests(url=REPO_URL)) == 1 @pytest.mark.usefixtures("stars") @@ -130,9 +219,9 @@ class TestApiReleaseNotes: upgraded = _bump(patch=1) releases.add_response(url=RELEASES_URL, json=[_release(f"v{upgraded}")]) - rest_api_client.get(self.ENDPOINT) + rest_api_client.get(self.ENDPOINT, {"since": OLDER}) with mock.patch.object(version, "__full_version_str__", upgraded): - response = rest_api_client.get(self.ENDPOINT) + response = rest_api_client.get(self.ENDPOINT, {"since": CURRENT}) assert len(releases.get_requests(url=RELEASES_URL)) == 2 assert [r["version"] for r in response.data["releases"]] == [upgraded] @@ -190,8 +279,8 @@ class TestApiReleaseNotes: httpx_mock: HTTPXMock, mock_kwargs: dict, ) -> None: - httpx_mock.add_response(url=RELEASES_URL, **mock_kwargs) - httpx_mock.add_response(url=RELEASES_URL, json=[_release(f"v{CURRENT}")]) + httpx_mock.add_response(url=_tag_url(CURRENT), **mock_kwargs) + httpx_mock.add_response(url=_tag_url(CURRENT), json=_release(f"v{CURRENT}")) httpx_mock.add_response( url=REPO_URL, json={"stargazers_count": 1234}, @@ -205,7 +294,7 @@ class TestApiReleaseNotes: response = rest_api_client.get(self.ENDPOINT) assert [r["version"] for r in response.data["releases"]] == [CURRENT] - @pytest.mark.usefixtures("releases", "stars") + @pytest.mark.usefixtures("current_release", "stars") def test_stars_are_returned_and_cached( self, rest_api_client: APIClient, @@ -226,7 +315,7 @@ class TestApiReleaseNotes: assert response.data["stars"] == 1234 assert len(httpx_mock.get_requests(url=REPO_URL)) == 1 - @pytest.mark.usefixtures("releases") + @pytest.mark.usefixtures("current_release") def test_stars_failure_returns_none_and_is_not_cached( self, rest_api_client: APIClient, diff --git a/src/documents/views.py b/src/documents/views.py index 095ff9c312..d2ff794246 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -4317,50 +4317,102 @@ class UiSettingsView(GenericAPIView[Any]): GITHUB_CACHE_KEY = "github_data:%s" -def get_github_releases( - current_version: packaging_version.Version, -) -> list[dict[str, Any]] | None: - """ - Fetch recent pngx releases from GitHub, newest first. - """ - releases = cache.get(GITHUB_CACHE_KEY % current_version) - if releases is not None: - return releases +GITHUB_RELEASES_URL = ( + "https://api.github.com/repos/paperless-ngx/paperless-ngx/releases" +) +GITHUB_RELEASES_PER_PAGE = 10 +# Caps how far back a range can walk, e.g. for a bad `since` +GITHUB_RELEASES_MAX_PAGES = 3 + + +def _fetch_github_releases_json(url: str, params: dict | None = None) -> Any | None: try: resp = httpx.get( - "https://api.github.com/repos/paperless-ngx/paperless-ngx/releases", - params={"per_page": 10}, + url, + params=params, # GitHub renders release bodies as HTML headers={"Accept": "application/vnd.github.html+json"}, ) resp.raise_for_status() - releases = [] - for release in resp.json(): - if release.get("draft") or release.get("prerelease"): - continue - try: - # Some early tags used ngx-x.y.z - release_version = packaging_version.parse( - release["tag_name"].removeprefix("ngx-"), - ) - except packaging_version.InvalidVersion: - continue - releases.append( - { - "version": str(release_version), - "name": release.get("name") or release["tag_name"], - "body_html": release.get("body_html") or "", - "html_url": release["html_url"], - "published_at": release.get("published_at"), - }, - ) - except (ValueError, KeyError, TypeError) as e: + return resp.json() + except ValueError as e: logger.debug(f"An error occurred parsing GitHub releases json: {e}") - return None except httpx.HTTPError as e: logger.debug(f"An error occurred fetching GitHub releases: {e}") + return None + + +def _parse_github_release(release: dict[str, Any]) -> dict[str, Any] | None: + if release.get("draft") or release.get("prerelease"): return None - cache.set(GITHUB_CACHE_KEY % current_version, releases, 60 * 60) + try: + # Some early tags used ngx-x.y.z + release_version = packaging_version.parse( + release["tag_name"].removeprefix("ngx-"), + ) + except packaging_version.InvalidVersion: + return None + return { + "version": str(release_version), + "name": release.get("name") or release["tag_name"], + "body_html": release.get("body_html") or "", + "html_url": release["html_url"], + "published_at": release.get("published_at"), + } + + +def get_github_release(version_str: str) -> dict[str, Any] | None: + """ + Fetch a single pngx release from GitHub by version. + """ + cache_key = GITHUB_CACHE_KEY % f"release:{version_str}" + release = cache.get(cache_key) + if release is not None: + return release + data = _fetch_github_releases_json(f"{GITHUB_RELEASES_URL}/tags/v{version_str}") + if data is None: + return None + try: + release = _parse_github_release(data) + except (KeyError, TypeError, AttributeError) as e: + logger.debug(f"An error occurred parsing GitHub release json: {e}") + return None + if release is not None: + cache.set(cache_key, release, 60 * 60) + return release + + +def get_github_releases( + current_version: packaging_version.Version, + since: packaging_version.Version, +) -> list[dict[str, Any]] | None: + """ + Fetch pngx releases from GitHub, newest first, back to `since`. + """ + cache_key = GITHUB_CACHE_KEY % f"releases:{current_version}:{since}" + releases = cache.get(cache_key) + if releases is not None: + return releases + releases = [] + # Releases are listed newest first, so stop paging once we reach `since` + for page in range(1, GITHUB_RELEASES_MAX_PAGES + 1): + data = _fetch_github_releases_json( + GITHUB_RELEASES_URL, + params={"per_page": GITHUB_RELEASES_PER_PAGE, "page": page}, + ) + if data is None: + return None + try: + batch = [r for r in map(_parse_github_release, data) if r is not None] + except (KeyError, TypeError, AttributeError) as e: + logger.debug(f"An error occurred parsing GitHub releases json: {e}") + return None + releases.extend(batch) + if len(data) < GITHUB_RELEASES_PER_PAGE or any( + packaging_version.parse(r["version"]) <= since for r in batch + ): + break + cache.set(cache_key, releases, 60 * 60) return releases @@ -4472,18 +4524,20 @@ class ReleaseNotesView(GenericAPIView[Any]): # Newer releases can be requested e.g. for "update available" until = until or current_version - releases = get_github_releases(current_version) or [] - - def in_range(release_version: str) -> bool: - v = packaging_version.parse(release_version) - if since is None: - return v == until - return since < v <= until + if since is None: + release = get_github_release(str(until)) + releases = [release] if release else [] + else: + releases = [ + r + for r in get_github_releases(current_version, since) or [] + if since < packaging_version.parse(r["version"]) <= until + ] return Response( { "version": version.__full_version_str__, - "releases": [r for r in releases if in_range(r["version"])], + "releases": releases, "stars": get_github_stars(), }, )