Merge pull request #64 from jantman/issues/63

python-gitlab Issue #63 - implement pagination for list()

gitlab/__init__.py

@@ -189,6 +189,8 @@ def set_url(self, url):
         self._url = '%s/api/v3' % url
 
     def _construct_url(self, id_, obj, parameters):
+        if 'next_url' in parameters:
+            return parameters['next_url']
         args = _sanitize_dict(parameters)
         if id_ is None and obj._urlPlural is not None:
             url = obj._urlPlural % args
@@ -346,8 +348,13 @@ def list(self, obj_class, **kwargs):
                 if key in cls_kwargs:
                     del cls_kwargs[key]
 
-            return [cls(self, item, **cls_kwargs) for item in r.json()
-                    if item is not None]
+            results = [cls(self, item, **cls_kwargs) for item in r.json()
+                       if item is not None]
+            if 'next' in r.links and 'url' in r.links['next']:
+                args = kwargs.copy()
+                args['next_url'] = r.links['next']['url']
+                results.extend(self.list(obj_class, **args))
+            return results
         else:
             _raise_error_from_response(r, GitlabListError)
 

gitlab/tests/test_gitlab.py

@@ -22,6 +22,7 @@
     import unittest
 except ImportError:
     import unittest2 as unittest
+import json
 
 from httmock import HTTMock  # noqa
 from httmock import response  # noqa
@@ -178,6 +179,56 @@ def resp_cont(url, request):
             self.assertEqual(data.project_id, 1)
             self.assertEqual(data.ref, "a")
 
+    def test_list_next_link(self):
+        @urlmatch(scheme="http", netloc="localhost",
+                  path='/api/v3/projects/1/repository/branches', method="get",
+                  query=r'per_page=1')
+        def resp_one(url, request):
+            """
+            First request:
+            http://localhost/api/v3/projects/1/repository/branches?per_page=1
+            """
+            headers = {
+                'content-type': 'application/json',
+                'link': '<http://localhost/api/v3/projects/1/repository/branc' \
+                'hes?page=2&per_page=0>; rel="next", <http://localhost/api/v3' \
+                '/projects/1/repository/branches?page=2&per_page=0>; rel="las' \
+                't", <http://localhost/api/v3/projects/1/repository/branches?' \
+                'page=1&per_page=0>; rel="first"'
+            }
+            content = ('[{"branch_name": "otherbranch", '
+                       '"project_id": 1, "ref": "b"}]').encode("utf-8")
+            resp = response(200, content, headers, None, 5, request)
+            return resp
+
+        @urlmatch(scheme="http", netloc="localhost",
+                  path='/api/v3/projects/1/repository/branches', method="get",
+                  query=r'.*page=2.*')
+        def resp_two(url, request):
+            headers = {
+                'content-type': 'application/json',
+                'link': '<http://localhost/api/v3/projects/1/repository/branc' \
+                'hes?page=1&per_page=0>; rel="prev", <http://localhost/api/v3' \
+                '/projects/1/repository/branches?page=2&per_page=0>; rel="las' \
+                't", <http://localhost/api/v3/projects/1/repository/branches?' \
+                'page=1&per_page=0>; rel="first"'
+            }
+            content = ('[{"branch_name": "testbranch", '
+                       '"project_id": 1, "ref": "a"}]').encode("utf-8")
+            resp = response(200, content, headers, None, 5, request)
+            return resp
+
+        with HTTMock(resp_one, resp_two):
+            data = self.gl.list(ProjectBranch, project_id=1,
+                                per_page=1)
+            self.assertEqual(data[1].branch_name, "testbranch")
+            self.assertEqual(data[1].project_id, 1)
+            self.assertEqual(data[1].ref, "a")
+            self.assertEqual(data[0].branch_name, "otherbranch")
+            self.assertEqual(data[0].project_id, 1)
+            self.assertEqual(data[0].ref, "b")
+            self.assertEqual(len(data), 2)
+
     def test_list_401(self):
         @urlmatch(scheme="http", netloc="localhost",
                   path="/api/v3/projects/1/repository/branches", method="get")