Preliminary pagination support API

docs/index.rst

@@ -169,6 +169,55 @@ You can also specify filters
 
     tasks = api.tasks.list(project=1)
 
+By default list returns all objects, eventually getting the
+paginated results behind the scenes.
+
+Pagination
+^^^^^^^^^^
+
+Pagination is controlled by three parameters as explained below:
+
++------------------+----------------------------+-------------+--------------------------------------------------------+
+|`pagination`      | `page_size` (default: 100) | `page`      | Output                                                 |
++==================+============================+=============+========================================================+
+| `True` (default) | `<integer>`                | `None`      | All results retrieved by using paginated results and   |
+|                  |                            |             | loading them behind the scenes, using given page       |
+|                  |                            |             | size (higher page size could yield better performances)|
++------------------+----------------------------+-------------+--------------------------------------------------------+
+| `True` (default) | `<integer>`                | `<integer>` | Only results for the given page of the given size      |
+|                  |                            |             | are retrieved                                          |
++------------------+----------------------------+-------------+--------------------------------------------------------+
+| `False`          | `unused`                   | `unused`    | Current behavior: all results, ignoring pagination     |
++------------------+----------------------------+-------------+--------------------------------------------------------+
+
+
+.. note:: non numerical or false `page_size` values is casted to the default value
+
+Examples
+^^^^^^^^^
+
+**No pagination**
+
+.. code:: python
+
+   tasks = api.tasks.list(paginate=False)
+
+.. warning:: be aware that the unpaginated results may exceed
+             the data the parser can handle and may result in an error.
+
+**Retrieve a single page**
+
+.. code:: python
+
+   tasks_page_1 = api.tasks.list(page=1)  # Will only return page 1
+
+**Specify the page size**
+
+.. code:: python
+
+   tasks_page_1 = api.tasks.list(page=1, page_size=200)  # Will 200 results from page 1
+
+
 Attach a file
 ~~~~~~~~~~~~~
 

taiga/models/base.py

@@ -34,11 +34,55 @@ def __init__(self, requester):
 
 class ListResource(Resource):
 
-    def list(self, **queryparams):
+    def list(self, pagination=True, page_size=None, page=None, **queryparams):
+        """
+        Retrieves a list of objects.
+
+        By default uses local cache and remote pagination
+
+        If pagination is used and no page is requested (the default), all the
+        remote objects are retrieved and appended in a single list.
+
+        If pagination is disabled, all the objects are fetched from the
+        endpoint and returned. This may trigger some parsing error if the
+        result set is very large.
+
+        :param pagination: Use pagination (default: `True`)
+        :param page_size: Size of the pagination page (default: `100`).
+                          Any non numeric value will be casted to the
+                          default value
+        :param page: Page number to retrieve (default: `None`). Ignored if
+                     `pagination` is `False`
+        :param queryparams: Additional filter parameters as accepted by the
+                            remote API
+        :return: <SearchableList>
+        """
+        if page_size and pagination:
+            try:
+                page_size = int(page_size)
+            except (ValueError, TypeError):
+                page_size = 100
+            queryparams['page_size'] = page_size
         result = self.requester.get(
-            self.instance.endpoint, query=queryparams
+            self.instance.endpoint, query=queryparams, paginate=pagination
         )
-        objects = self.parse_list(result.json())
+        objects = SearchableList()
+        objects.extend(self.parse_list(result.json()))
+        if result.headers.get('X-Pagination-Next', False) and not page:
+            next_page = 2
+        else:
+            next_page = None
+        while next_page:
+            pageparams = queryparams.copy()
+            pageparams['page'] = next_page
+            result = self.requester.get(
+                self.instance.endpoint, query=pageparams,
+            )
+            objects.extend(self.parse_list(result.json()))
+            if result.headers.get('X-Pagination-Next', False):
+                next_page += 1
+            else:
+                next_page = None
         return objects
 
     def get(self, resource_id):

taiga/requestmaker.py

@@ -7,7 +7,7 @@
 from requests.packages.urllib3.exceptions import InsecureRequestWarning
 
 
-def _disable_pagination():
+def _requests_compatible_true():
     if LooseVersion(requests.__version__) >= LooseVersion('2.11.0'):
         return 'True'
     else:
@@ -61,12 +61,15 @@ def __init__(self,
                  api_path, host,
                  token,
                  token_type='Bearer',
-                 tls_verify=True):
+                 tls_verify=True,
+                 enable_pagination=True
+                 ):
         self.api_path = api_path
         self.host = host
         self.token = token
         self.token_type = token_type
         self.tls_verify = tls_verify
+        self.enable_pagination = enable_pagination
         self._cache = RequestCache()
         if not self.tls_verify:
             requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
@@ -78,12 +81,15 @@ def cache(self):
     def is_bad_response(self, response):
         return 400 <= response.status_code <= 500
 
-    def headers(self):
+    def headers(self, paginate=True):
         headers = {
             'Content-type': 'application/json',
             'Authorization': '{0} {1}'.format(self.token_type, self.token),
-            'x-disable-pagination': _disable_pagination()
         }
+        if self.enable_pagination and paginate:
+            headers['x-lazy-pagination'] = _requests_compatible_true()
+        else:
+            headers['x-disable-pagination'] = _requests_compatible_true()
         return headers
 
     def urljoin(self, *parts):
@@ -96,7 +102,7 @@ def get_full_url(self, uri, query={}, **parameters):
         )
         return full_url
 
-    def get(self, uri, query={}, cache=False, **parameters):
+    def get(self, uri, query={}, cache=False, paginate=True, **parameters):
         try:
             full_url = self.urljoin(
                 self.host, self.api_path,
@@ -114,7 +120,7 @@ def get(self, uri, query={}, cache=False, **parameters):
             if not result:
                 result = requests.get(
                     full_url,
-                    headers=self.headers(),
+                    headers=self.headers(paginate),
                     params=query,
                     verify=self.tls_verify
                 )
@@ -137,7 +143,7 @@ def post(self, uri, payload=None, query={}, files={}, **parameters):
         if files:
             headers = {
                 'Authorization': '{0} {1}'.format(self.token_type, self.token),
-                'x-disable-pagination': _disable_pagination()
+                'x-disable-pagination': _requests_compatible_true()
             }
             data = payload
         else:

tests/resources/fakes_list_success.json

@@ -0,0 +1,29 @@
+[
+    {
+        "id": 1
+    },
+    {
+        "id": 2
+    },
+    {
+        "id": 3
+    },
+    {
+        "id": 4
+    },
+    {
+        "id": 5
+    },
+    {
+        "id": 6
+    },
+    {
+        "id": 7
+    },
+    {
+        "id": 8
+    },
+    {
+        "id": 9
+    }
+]

tests/test_issues.py

@@ -16,11 +16,14 @@ class TestIssues(unittest.TestCase):
 
     @patch('taiga.requestmaker.RequestMaker.get')
     def test_list_attachments(self, mock_requestmaker_get):
+        mock_requestmaker_get.return_value = MockResponse(200,
+            create_mock_json('tests/resources/issues_list_success.json'))
         rm = RequestMaker('/api/v1', 'fakehost', 'faketoken')
         Issue(rm, id=1).list_attachments()
         mock_requestmaker_get.assert_called_with(
             'issues/attachments',
             query={"object_id": 1},
+            paginate=True
         )
 
     @patch('taiga.requestmaker.RequestMaker.post')

tests/test_model_base.py

@@ -1,11 +1,13 @@
 # -*- coding: utf-8 -*-
+import json
 
+from taiga.models import Projects
 from taiga.requestmaker import RequestMaker
 from taiga.models.base import InstanceResource, ListResource, SearchableList
 import unittest
 from mock import patch
 import datetime
-from .tools import MockResponse
+from .tools import MockResponse, create_mock_json
 
 
 class Fake(InstanceResource):
@@ -18,14 +20,28 @@ class Fake(InstanceResource):
 
     def my_method(self):
         response = self.requester.get('/users/{id}/starred', id=self.id)
-        return projects.Projects.parse(response.json(), self.requester)
+        return Projects.parse(response.json(), self.requester)
 
 
 class Fakes(ListResource):
 
     instance = Fake
 
 
+class FakeHeaders(dict):
+    sequence = []
+    counter = -1
+
+    def __init__(self, sequence=[], *args, **kwargs):
+        self.sequence = sequence
+        self.counter = -1
+        super(FakeHeaders, self).__init__(*args, **kwargs)
+
+    def get(self, k, d=None):
+        self.counter += 1
+        return self.sequence[self.counter]
+
+
 class TestModelBase(unittest.TestCase):
 
     def test_encoding(self):
@@ -136,12 +152,112 @@ def test_call_model_base_delete_element_from_list(self, mock_requestmaker_delete
 
     @patch('taiga.requestmaker.RequestMaker.get')
     def test_call_model_base_list_elements(self, mock_requestmaker_get):
+        js_list = json.loads(create_mock_json('tests/resources/fakes_list_success.json'))
+        rm = RequestMaker('/api/v1', 'fakehost', 'faketoken')
+        fakes = Fakes(rm)
+
+        data = json.dumps(js_list)
+        mock_requestmaker_get.return_value = MockResponse(200, data)
+        f_list = fakes.list()
+        mock_requestmaker_get.assert_called_with('fakes',  query={}, paginate=True)
+        self.assertEqual(len(f_list), 9)
+
+        data = json.dumps(js_list[0])
+        mock_requestmaker_get.return_value = MockResponse(200, data)
+        f_list = fakes.list(id=1)
+        mock_requestmaker_get.assert_called_with('fakes', query={'id':1}, paginate=True)
+        self.assertEqual(len(f_list), 1)
+
+    @patch('taiga.requestmaker.RequestMaker.get')
+    def test_call_model_base_list_page_size(self, mock_requestmaker_get):
+        js_list = json.loads(create_mock_json('tests/resources/fakes_list_success.json'))
+        rm = RequestMaker('/api/v1', 'fakehost', 'faketoken')
+        fakes = Fakes(rm)
+
+        data = json.dumps(js_list)
+        mock_requestmaker_get.return_value = MockResponse(200, data, FakeHeaders(
+            [True, True, False],
+            **{'X-Pagination-Next': True}
+        ))
+        f_list = fakes.list(page_size=2)
+        mock_requestmaker_get.assert_called_with('fakes',  query={'page': 3, 'page_size': 2})
+        self.assertEqual(len(f_list), 27)
+
+        data = json.dumps(js_list)
+        mock_requestmaker_get.return_value = MockResponse(200, data)
+        f_list = fakes.list(page_size='wrong')
+        mock_requestmaker_get.assert_called_with('fakes',  query={'page_size': 100}, paginate=True)
+        self.assertEqual(len(f_list), 9)
+
+    @patch('taiga.requestmaker.RequestMaker.get')
+    def test_call_model_base_list_elements_no_paginate(self, mock_requestmaker_get):
+        js_list = json.loads(create_mock_json('tests/resources/fakes_list_success.json'))
+        rm = RequestMaker('/api/v1', 'fakehost', 'faketoken')
+        fakes = Fakes(rm)
+
+        data = json.dumps(js_list)
+        mock_requestmaker_get.return_value = MockResponse(200, data)
+        f_list = fakes.list(pagination=False)
+        mock_requestmaker_get.assert_called_with('fakes',  query={}, paginate=False)
+        self.assertEqual(len(f_list), 9)
+
+        data = json.dumps(js_list[0])
+        mock_requestmaker_get.return_value = MockResponse(200, data)
+        f_list = fakes.list(id=1, pagination=False)
+        mock_requestmaker_get.assert_called_with('fakes', query={'id':1}, paginate=False)
+        self.assertEqual(len(f_list), 1)
+
+    @patch('taiga.requestmaker.requests.get')
+    def test_call_model_base_list_elements_no_paginate_check_requests(self, mock_requestmaker_get):
+        js_list = json.loads(create_mock_json('tests/resources/fakes_list_success.json'))
+        rm = RequestMaker('/api/v1', 'fakehost', 'faketoken')
+        fakes = Fakes(rm)
+
+        data = json.dumps(js_list)
+        mock_requestmaker_get.return_value = MockResponse(200, data)
+        f_list = fakes.list(pagination=False)
+        mock_requestmaker_get.assert_called_with(
+            'fakehost/api/v1/fakes', verify=True, params={},
+            headers={
+                'x-disable-pagination': 'True', 'Content-type': 'application/json', 'Authorization': 'Bearer faketoken'
+            }
+        )
+        self.assertEqual(len(f_list), 9)
+
+    @patch('taiga.requestmaker.requests.get')
+    def test_call_model_base_list_elements_paginate_check_requests(self, mock_requestmaker_get):
+        js_list = json.loads(create_mock_json('tests/resources/fakes_list_success.json'))
         rm = RequestMaker('/api/v1', 'fakehost', 'faketoken')
         fakes = Fakes(rm)
-        fakes.list()
-        mock_requestmaker_get.assert_called_with('fakes',  query={})
-        fakes.list(project_id=1)
-        mock_requestmaker_get.assert_called_with('fakes', query={'project_id':1})
+
+        data = json.dumps(js_list)
+        mock_requestmaker_get.return_value = MockResponse(200, data)
+        f_list = fakes.list()
+        mock_requestmaker_get.assert_called_with(
+            'fakehost/api/v1/fakes', verify=True, params={},
+            headers={
+                'x-lazy-pagination': 'True', 'Content-type': 'application/json', 'Authorization': 'Bearer faketoken'
+            }
+        )
+        self.assertEqual(len(f_list), 9)
+
+    @patch('taiga.requestmaker.RequestMaker.get')
+    def test_call_model_base_list_elements_single_page(self, mock_requestmaker_get):
+        js_list = json.loads(create_mock_json('tests/resources/fakes_list_success.json'))
+        rm = RequestMaker('/api/v1', 'fakehost', 'faketoken')
+        fakes = Fakes(rm)
+
+        data = json.dumps(js_list[:5])
+        mock_requestmaker_get.return_value = MockResponse(200, data)
+        f_list = fakes.list(page_size=5, page=1)
+        self.assertEqual(len(f_list), 5)
+        mock_requestmaker_get.assert_called_with('fakes',  query={'page_size': 5}, paginate=True)
+
+        data = json.dumps(js_list[5:])
+        mock_requestmaker_get.return_value = MockResponse(200, data)
+        f_list = fakes.list(page_size=5, page=2)
+        self.assertEqual(len(f_list), 4)
+        mock_requestmaker_get.assert_called_with('fakes',  query={'page_size': 5}, paginate=True)
 
     def test_to_dict_method(self):
         rm = RequestMaker('/api/v1', 'fakehost', 'faketoken')