Preliminary pagination support API

docs/index.rst

@@ -169,6 +169,55 @@ You can also specify filters
 
     tasks = api.tasks.list(project=1)
 
+By default list returns all objects, eventually getting the
+paginated results behind the scenes.
+
+Pagination
+^^^^^^^^^^
+
+Pagination is controlled by three parameters as explained below:
+
++------------------+----------------------------+-------------+--------------------------------------------------------+
+|`pagination`      | `page_size` (default: 100) | `page`      | Output                                                 |
++==================+============================+=============+========================================================+
+| `True` (default) | `<integer>`                | `None`      | All results retrieved by using paginated results and   |
+|                  |                            |             | loading them behind the scenes, using given page       |
+|                  |                            |             | size (higher page size could yield better performances)|
++------------------+----------------------------+-------------+--------------------------------------------------------+
+| `True` (default) | `<integer>`                | `<integer>` | Only results for the given page of the given size      |
+|                  |                            |             | are retrieved                                          |
++------------------+----------------------------+-------------+--------------------------------------------------------+
+| `False`          | `unused`                   | `unused`    | Current behavior: all results, ignoring pagination     |
++------------------+----------------------------+-------------+--------------------------------------------------------+
+
+
+.. note:: non numerical or false `page_size` values is casted to the default value
+
+Examples
+^^^^^^^^^
+
+**No pagination**
+
+.. code:: python
+
+   tasks = api.tasks.list(paginate=False)
+
+.. warning:: be aware that if the unpaginated results may exceed
+             the data the parser can handle and may result in an error.
+
+**Retrieve a single page**
+
+.. code:: python
+
+   tasks_page_1 = api.tasks.list(page=1)  # Will only return page 1
+
+**Specify the page size**
+
+.. code:: python
+
+   tasks_page_1 = api.tasks.list(page=1, page_size=200)  # Will 200 results from page 1
+
+
 Attach a file
 ~~~~~~~~~~~~~
 

taiga/models/base.py

@@ -34,11 +34,55 @@ def __init__(self, requester):
 
 class ListResource(Resource):
 
-    def list(self, **queryparams):
+    def list(self, pagination=True, page_size=None, page=None, **queryparams):
+        """
+        Retrieves a list of objects.
+
+        By default uses local cache and remote pagination
+
+        If pagination is used and no page is requested (the default), all the
+        remote objects are retrieved and appended in a single list.
+
+        If pagination is disabled, all the objects are fetched from the
+        endpojnt and returned. This may trigger some parsing error if the
+        result set is very large.
+
+        :param pagination: Use pagination (default: `True`)
+        :param page_size: Size of the pagination page (default: `100`).
+                          Any non numeric value will be casted to the
+                          default value
+        :param page: Page number to retrieve (default: `None`). Ignored if
+                     `pagination` is `False`
+        :param queryparams: Additional filter parameters as accepted by the
+                            remote API
+        :return: <SearchableList>
+        """
+        if page_size and pagination:
+            try:
+                page_size = int(page_size)
+            except (ValueError, TypeError):
+                page_size = 100
+            queryparams['page_size'] = page_size
         result = self.requester.get(
-            self.instance.endpoint, query=queryparams
+            self.instance.endpoint, query=queryparams, paginate=pagination
         )
-        objects = self.parse_list(result.json())
+        objects = SearchableList()
+        objects.extend(self.parse_list(result.json()))
+        if result.headers.get('X-Pagination-Next', False) and not page:
+            next_page = 2
+        else:
+            next_page = None
+        while next_page:
+            pageparams = queryparams.copy()
+            pageparams['page'] = next_page
+            result = self.requester.get(
+                self.instance.endpoint, query=pageparams,
+            )
+            objects.extend(self.parse_list(result.json()))
+            if result.headers.get('X-Pagination-Next', False):
+                next_page += 1
+            else:
+                next_page = None
         return objects
 
     def get(self, resource_id):

taiga/requestmaker.py

@@ -7,7 +7,7 @@
 from requests.packages.urllib3.exceptions import InsecureRequestWarning
 
 
-def _disable_pagination():
+def _requests_compatible_true():
     if LooseVersion(requests.__version__) >= LooseVersion('2.11.0'):
         return 'True'
     else:
@@ -61,12 +61,15 @@ def __init__(self,
                  api_path, host,
                  token,
                  token_type='Bearer',
-                 tls_verify=True):
+                 tls_verify=True,
+                 enable_pagination=True
+                 ):
         self.api_path = api_path
         self.host = host
         self.token = token
         self.token_type = token_type
         self.tls_verify = tls_verify
+        self.enable_pagination = enable_pagination
         self._cache = RequestCache()
         if not self.tls_verify:
             requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
@@ -78,12 +81,15 @@ def cache(self):
     def is_bad_response(self, response):
         return 400 <= response.status_code <= 500
 
-    def headers(self):
+    def headers(self, paginate=True):
         headers = {
             'Content-type': 'application/json',
             'Authorization': '{0} {1}'.format(self.token_type, self.token),
-            'x-disable-pagination': _disable_pagination()
         }
+        if self.enable_pagination and paginate:
+            headers['x-lazy-pagination'] = _requests_compatible_true()
+        else:
+            headers['x-disable-pagination'] = _requests_compatible_true()
         return headers
 
     def urljoin(self, *parts):
@@ -96,7 +102,7 @@ def get_full_url(self, uri, query={}, **parameters):
         )
         return full_url
 
-    def get(self, uri, query={}, cache=False, **parameters):
+    def get(self, uri, query={}, cache=False, paginate=True, **parameters):
         try:
             full_url = self.urljoin(
                 self.host, self.api_path,
@@ -114,7 +120,7 @@ def get(self, uri, query={}, cache=False, **parameters):
             if not result:
                 result = requests.get(
                     full_url,
-                    headers=self.headers(),
+                    headers=self.headers(paginate),
                     params=query,
                     verify=self.tls_verify
                 )
@@ -137,7 +143,7 @@ def post(self, uri, payload=None, query={}, files={}, **parameters):
         if files:
             headers = {
                 'Authorization': '{0} {1}'.format(self.token_type, self.token),
-                'x-disable-pagination': _disable_pagination()
+                'x-disable-pagination': _requests_compatible_true()
             }
             data = payload
         else:

tests/test_requestmaker.py

@@ -1,4 +1,4 @@
-from taiga.requestmaker import RequestMaker, _disable_pagination
+from taiga.requestmaker import RequestMaker, _requests_compatible_true
 import taiga.exceptions
 import requests
 import unittest
@@ -33,7 +33,7 @@ def test_call_requests_post_with_files(self, requests_post):
             data=None, params={},
             headers={
                 'Authorization': 'Bearer f4k3',
-                'x-disable-pagination': _disable_pagination()
+                'x-disable-pagination': _requests_compatible_true()
             }
         )