Merge pull request #1773 from python-gitlab/jlvillal/pagination

fix: handle situation where gitlab.com does not return values

docs/api-usage.rst

@@ -265,8 +265,17 @@ The generator exposes extra listing information as received from the server:
 * ``prev_page``: if ``None`` the current page is the first one
 * ``next_page``: if ``None`` the current page is the last one
 * ``per_page``: number of items per page
-* ``total_pages``: total number of pages available
-* ``total``: total number of items in the list
+* ``total_pages``: total number of pages available. This may be a ``None`` value.
+* ``total``: total number of items in the list. This may be a ``None`` value.
+
+.. note::
+
+   For performance reasons, if a query returns more than 10,000 records, GitLab
+   does not return the ``total_pages`` or ``total`` headers.  In this case,
+   ``total_pages`` and ``total`` will have a value of ``None``.
+
+   For more information see:
+   https://docs.gitlab.com/ee/user/gitlab_com/index.html#pagination-response-headers
 
 Sudo
 ====

gitlab/base.py

@@ -288,12 +288,12 @@ def per_page(self) -> int:
         return self._list.per_page
 
     @property
-    def total_pages(self) -> int:
+    def total_pages(self) -> Optional[int]:
         """The total number of pages."""
         return self._list.total_pages
 
     @property
-    def total(self) -> int:
+    def total(self) -> Optional[int]:
         """The total number of items."""
         return self._list.total
 

gitlab/client.py

@@ -917,14 +917,12 @@ def _query(
             self._next_url = next_url
         except KeyError:
             self._next_url = None
-        self._current_page: Optional[Union[str, int]] = result.headers.get("X-Page")
-        self._prev_page: Optional[Union[str, int]] = result.headers.get("X-Prev-Page")
-        self._next_page: Optional[Union[str, int]] = result.headers.get("X-Next-Page")
-        self._per_page: Optional[Union[str, int]] = result.headers.get("X-Per-Page")
-        self._total_pages: Optional[Union[str, int]] = result.headers.get(
-            "X-Total-Pages"
-        )
-        self._total: Optional[Union[str, int]] = result.headers.get("X-Total")
+        self._current_page: Optional[str] = result.headers.get("X-Page")
+        self._prev_page: Optional[str] = result.headers.get("X-Prev-Page")
+        self._next_page: Optional[str] = result.headers.get("X-Next-Page")
+        self._per_page: Optional[str] = result.headers.get("X-Per-Page")
+        self._total_pages: Optional[str] = result.headers.get("X-Total-Pages")
+        self._total: Optional[str] = result.headers.get("X-Total")
 
         try:
             self._data: List[Dict[str, Any]] = result.json()
@@ -965,19 +963,22 @@ def per_page(self) -> int:
             assert self._per_page is not None
         return int(self._per_page)
 
+    # NOTE(jlvillal): When a query returns more than 10,000 items, GitLab doesn't return
+    # the headers 'x-total-pages' and 'x-total'. In those cases we return None.
+    # https://docs.gitlab.com/ee/user/gitlab_com/index.html#pagination-response-headers
     @property
-    def total_pages(self) -> int:
+    def total_pages(self) -> Optional[int]:
         """The total number of pages."""
-        if TYPE_CHECKING:
-            assert self._total_pages is not None
-        return int(self._total_pages)
+        if self._total_pages is not None:
+            return int(self._total_pages)
+        return None
 
     @property
-    def total(self) -> int:
+    def total(self) -> Optional[int]:
         """The total number of items."""
-        if TYPE_CHECKING:
-            assert self._total is not None
-        return int(self._total)
+        if self._total is not None:
+            return int(self._total)
+        return None
 
     def __iter__(self) -> "GitlabList":
         return self

pyproject.toml

@@ -87,3 +87,6 @@ disable = [
     "useless-object-inheritance",
 
 ]
+
+[tool.pytest.ini_options]
+xfail_strict = true

tests/unit/test_gitlab.py

@@ -16,9 +16,9 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import copy
 import pickle
 import warnings
-from copy import deepcopy
 
 import pytest
 import responses
@@ -109,15 +109,14 @@ def _strip_pagination_headers(response):
     """
     https://docs.gitlab.com/ee/user/gitlab_com/index.html#pagination-response-headers
     """
-    stripped = deepcopy(response)
+    stripped = copy.deepcopy(response)
 
     del stripped["headers"]["X-Total-Pages"]
     del stripped["headers"]["X-Total"]
 
     return stripped
 
 
-@pytest.mark.xfail(reason="See #1686")
 @responses.activate
 def test_gitlab_build_list_missing_headers(gl, resp_page_1, resp_page_2):
     stripped_page_1 = _strip_pagination_headers(resp_page_1)