From cb824a49af9b0d155b89fe66a4cfebefe52beb7a Mon Sep 17 00:00:00 2001 From: "John L. Villalovos" Date: Thu, 30 Dec 2021 12:34:50 -0800 Subject: [PATCH] fix: handle situation where GitLab does not return values If a query returns more than 10,000 records than the following values are NOT returned: x.total_pages x.total Modify the code to allow no value to be set for these values. If there is not a value returned the functions will now return None. Update unit test so no longer `xfail` https://docs.gitlab.com/ee/user/gitlab_com/index.html#pagination-response-headers Closes #1686 --- docs/api-usage.rst | 13 +++++++++++-- gitlab/base.py | 4 ++-- gitlab/client.py | 33 +++++++++++++++++---------------- pyproject.toml | 3 +++ tests/unit/test_gitlab.py | 5 ++--- 5 files changed, 35 insertions(+), 23 deletions(-) diff --git a/docs/api-usage.rst b/docs/api-usage.rst index f30ed0351..66e58873a 100644 --- a/docs/api-usage.rst +++ b/docs/api-usage.rst @@ -265,8 +265,17 @@ The generator exposes extra listing information as received from the server: * ``prev_page``: if ``None`` the current page is the first one * ``next_page``: if ``None`` the current page is the last one * ``per_page``: number of items per page -* ``total_pages``: total number of pages available -* ``total``: total number of items in the list +* ``total_pages``: total number of pages available. This may be a ``None`` value. +* ``total``: total number of items in the list. This may be a ``None`` value. + +.. note:: + + For performance reasons, if a query returns more than 10,000 records, GitLab + does not return the ``total_pages`` or ``total`` headers. In this case, + ``total_pages`` and ``total`` will have a value of ``None``. + + For more information see: + https://docs.gitlab.com/ee/user/gitlab_com/index.html#pagination-response-headers Sudo ==== diff --git a/gitlab/base.py b/gitlab/base.py index 64604b487..50f09c596 100644 --- a/gitlab/base.py +++ b/gitlab/base.py @@ -288,12 +288,12 @@ def per_page(self) -> int: return self._list.per_page @property - def total_pages(self) -> int: + def total_pages(self) -> Optional[int]: """The total number of pages.""" return self._list.total_pages @property - def total(self) -> int: + def total(self) -> Optional[int]: """The total number of items.""" return self._list.total diff --git a/gitlab/client.py b/gitlab/client.py index 84fd40fc3..c1e0825a4 100644 --- a/gitlab/client.py +++ b/gitlab/client.py @@ -917,14 +917,12 @@ def _query( self._next_url = next_url except KeyError: self._next_url = None - self._current_page: Optional[Union[str, int]] = result.headers.get("X-Page") - self._prev_page: Optional[Union[str, int]] = result.headers.get("X-Prev-Page") - self._next_page: Optional[Union[str, int]] = result.headers.get("X-Next-Page") - self._per_page: Optional[Union[str, int]] = result.headers.get("X-Per-Page") - self._total_pages: Optional[Union[str, int]] = result.headers.get( - "X-Total-Pages" - ) - self._total: Optional[Union[str, int]] = result.headers.get("X-Total") + self._current_page: Optional[str] = result.headers.get("X-Page") + self._prev_page: Optional[str] = result.headers.get("X-Prev-Page") + self._next_page: Optional[str] = result.headers.get("X-Next-Page") + self._per_page: Optional[str] = result.headers.get("X-Per-Page") + self._total_pages: Optional[str] = result.headers.get("X-Total-Pages") + self._total: Optional[str] = result.headers.get("X-Total") try: self._data: List[Dict[str, Any]] = result.json() @@ -965,19 +963,22 @@ def per_page(self) -> int: assert self._per_page is not None return int(self._per_page) + # NOTE(jlvillal): When a query returns more than 10,000 items, GitLab doesn't return + # the headers 'x-total-pages' and 'x-total'. In those cases we return None. + # https://docs.gitlab.com/ee/user/gitlab_com/index.html#pagination-response-headers @property - def total_pages(self) -> int: + def total_pages(self) -> Optional[int]: """The total number of pages.""" - if TYPE_CHECKING: - assert self._total_pages is not None - return int(self._total_pages) + if self._total_pages is not None: + return int(self._total_pages) + return None @property - def total(self) -> int: + def total(self) -> Optional[int]: """The total number of items.""" - if TYPE_CHECKING: - assert self._total is not None - return int(self._total) + if self._total is not None: + return int(self._total) + return None def __iter__(self) -> "GitlabList": return self diff --git a/pyproject.toml b/pyproject.toml index 2aa5b1d1e..bc0530aee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,3 +87,6 @@ disable = [ "useless-object-inheritance", ] + +[tool.pytest.ini_options] +xfail_strict = true diff --git a/tests/unit/test_gitlab.py b/tests/unit/test_gitlab.py index 2981ebb87..4d742d39c 100644 --- a/tests/unit/test_gitlab.py +++ b/tests/unit/test_gitlab.py @@ -16,9 +16,9 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . +import copy import pickle import warnings -from copy import deepcopy import pytest import responses @@ -109,7 +109,7 @@ def _strip_pagination_headers(response): """ https://docs.gitlab.com/ee/user/gitlab_com/index.html#pagination-response-headers """ - stripped = deepcopy(response) + stripped = copy.deepcopy(response) del stripped["headers"]["X-Total-Pages"] del stripped["headers"]["X-Total"] @@ -117,7 +117,6 @@ def _strip_pagination_headers(response): return stripped -@pytest.mark.xfail(reason="See #1686") @responses.activate def test_gitlab_build_list_missing_headers(gl, resp_page_1, resp_page_2): stripped_page_1 = _strip_pagination_headers(resp_page_1)