Merge branch 'feature/files-cont-token' into develop

docs/quickstart.rst

@@ -858,6 +858,37 @@ Examples
         parent=new_folder, name='new-file-name'
     )
 
+List Files - introduction to pagination
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The results of files queries - :code:`api.files.query()` and :code:`api.files.list_files()`, can be paginated
+in two ways: using offset or continuation token parameter.
+
+Working with offset pagination is equivalent to that explained in 'List Projects - introduction to pagination and iteration' section,
+so one can get informed there about using :code:`offset` and :code:`limit` parameters.
+
+The continuation token pagination can be achieved by using :code:`cont_token` parameter (in combination with :code:`limit`)
+in query methods (:code:`query()`/:code:`list_files()`). This parameter is a base64 encoded value, telling server
+where to start next page. If one wants to use continuation token pagination, :code:`query()`/:code:`list_files()` method
+should be called with :code:`cont_token` parameter, as in example below:
+
+.. code:: python
+
+    files_list = api.files.query(cont_token='init', limit=10)
+
+The :code:`'init'` value is special initial value that should be passed at first call of :code:`query()`/:code:`list_files()`
+method if continuation token pagination is desired.
+
+There are certain restrictions on using this parameter:
+
+1. The continuation token pagination is advance access feature, so :code:`cont_token` parameter can be used only if
+   :code:`advance_access` is set to :code:`True`, otherwise :code:`SbgError` will be returned.
+2. The continuation token pagination cannot be used together with metadata filtering, so if :code:`cont_token` and
+   :code:`metadata` parameters are both provided to :code:`query()`/:code:`list_files()`, :code:`SbgError` will be returned.
+3. The continuation token pagination and offset pagination are mutually exclusive, so if there are passed both
+   :code:`cont_token` and :code:`offset` parameters to :code:`query()`/:code:`list_files()`, :code:`SbgError` will be returned.
+
+
 Managing file upload and download
 ---------------------------------
 

sevenbridges/models/file.py

@@ -38,6 +38,7 @@ class File(Resource):
 
     _URL = {
         'query': '/files',
+        'scroll': '/files/scroll',
         'get': '/files/{id}',
         'delete': '/files/{id}',
         'copy': '/files/{id}/actions/copy',
@@ -52,6 +53,7 @@ class File(Resource):
 
         'create_folder': '/files',
         'list_folder': '/files/{id}/list',
+        'scroll_folder': '/files/{id}/scroll',
         'copy_to_folder': '/files/{file_id}/actions/copy',
         'move_to_folder': '/files/{file_id}/actions/move',
     }
@@ -97,8 +99,8 @@ def secondary_files(self):
 
     @classmethod
     def query(cls, project=None, names=None, metadata=None, origin=None,
-              tags=None, offset=None, limit=None, dataset=None, api=None,
-              parent=None):
+              tags=None, offset=None, limit=None, dataset=None,
+              api=None, parent=None, cont_token=None):
         """
         Query ( List ) files, requires project or dataset
         :param project: Project id
@@ -111,8 +113,22 @@ def query(cls, project=None, names=None, metadata=None, origin=None,
         :param dataset: Dataset id
         :param api: Api instance.
         :param parent: Folder id or File object with type folder
+        :param cont_token: Pagination continuation token
         :return: Collection object.
         """
+
+        if cont_token and offset:
+            raise SbgError(
+                'Offset and continuation token parameters'
+                'are mutually exclusive.'
+            )
+
+        if cont_token and metadata:
+            raise SbgError(
+                'Metadata filtering cannot be combined '
+                'with continuation token pagination.'
+            )
+
         api = api or cls._API
 
         query_params = {}
@@ -159,7 +175,8 @@ def query(cls, project=None, names=None, metadata=None, origin=None,
         query_params.update(origin_params)
 
         return super(File, cls)._query(
-            api=api, url=cls._URL['query'], offset=offset,
+            api=api, url=cls._URL['scroll' if cont_token else 'query'],
+            token=cont_token, offset=offset,
             limit=limit, fields='_all', **query_params
         )
 
@@ -500,22 +517,33 @@ def bulk_edit(cls, files, api=None):
         response = api.post(url=cls._URL['bulk_edit'], data=data)
         return FileBulkRecord.parse_records(response=response, api=api)
 
-    def list_files(self, offset=None, limit=None, api=None):
+    def list_files(self, offset=None, limit=None, api=None, cont_token=None):
         """List files in a folder
         :param api: Api instance
         :param offset: Pagination offset
         :param limit: Pagination limit
+        :param cont_token: Pagination continuation token
         :return: List of files
         """
+
+        if cont_token and offset:
+            raise SbgError(
+                'Offset and continuation token parameters'
+                'are mutually exclusive.'
+            )
+
         api = api or self._API
 
         if not self.is_folder():
             raise SbgError('{name} is not a folder'.format(name=self.name))
 
-        url = self._URL['list_folder'].format(id=self.id)
+        url = self._URL[
+            'scroll_folder' if cont_token else 'list_folder'
+        ].format(id=self.id)
 
         return super(File, self.__class__)._query(
-            api=api, url=url, offset=offset, limit=limit, fields='_all'
+            api=api, url=url, token=cont_token, offset=offset,
+            limit=limit, fields='_all'
         )
 
     @classmethod

sevenbridges/tests/providers.py

@@ -380,9 +380,11 @@ def tags_can_be_saved(self, id):
             'PUT', '/files/{id}/tags'.format(id=id), json=file_['tags']
         )
 
-    def files_exist_for_project(self, project, num_of_files):
+    def files_exist_for_project(self, project, num_of_files, scroll=False):
         items = [FileProvider.default_file() for _ in range(num_of_files)]
-        href = self.base_url + '/files?project={project}'.format(
+        suffix = 'files/scroll' if scroll else 'files'
+        href = self.base_url + '/{suffix}?project={project}'.format(
+            suffix=suffix,
             project=project
         )
         links = []
@@ -466,9 +468,13 @@ def files_exist_for_file_tag(self, project, tags, num_of_files):
         self.request_mocker.get(href, json=response, headers={
             'x-total-matching-query': str(num_of_files)})
 
-    def files_in_folder(self, num_of_files, folder_id):
+    def files_in_folder(self, num_of_files, folder_id, scroll=False):
         items = [FileProvider.default_file() for _ in range(num_of_files)]
-        url = '/files/{folder_id}/list'.format(folder_id=folder_id)
+        suffix = 'scroll' if scroll else 'list'
+        url = '/files/{folder_id}/{suffix}'.format(
+            folder_id=folder_id,
+            suffix=suffix
+        )
         href = self.base_url + url
 
         response = {

sevenbridges/tests/test_files.py

@@ -26,6 +26,37 @@ def test_files_query(api, given, verifier):
     verifier.file.queried(project=project.id)
 
 
+def test_files_query_with_token(api, given, verifier):
+    # preconditions
+    total = 20
+    owner = generator.user_name()
+    project_short_name = generator.slug()
+    id = '{}/{}'.format(owner, project_short_name)
+    given.file.files_exist_for_project(id, 20, scroll=True)
+    given.project.exists(id=id)
+
+    # action
+    project = api.projects.get(id)
+    projects = api.files.query(project=project, limit=10, cont_token='init')
+
+    # verification
+    assert projects.total == total
+    assert len(projects) == total
+
+    verifier.file.queried_with_token(project=project.id)
+
+
+def test_files_query_with_token_and_offset(api, given, verifier):
+    # action
+    with pytest.raises(SbgError):
+        api.files.query(
+            project='some project',
+            limit=10,
+            cont_token='some token',
+            offset=10
+        )
+
+
 def test_files_query_folder(api, given, verifier):
     # preconditions
     total = 10
@@ -87,6 +118,17 @@ def test_files_query_file_metadata(api, given, verifier):
     verifier.file.queried_with_file_metadata(id, key, value)
 
 
+def test_files_query_file_metadata_with_token(api, given, verifier):
+    # action
+    with pytest.raises(SbgError):
+        api.files.query(
+            project='some project',
+            metadata={'key': 'value'},
+            limit=10,
+            cont_token='some token'
+        )
+
+
 def test_files_query_file_origin(api, given, verifier):
     # preconditions
     total = 10
@@ -301,6 +343,7 @@ def test_create_folder(api, given, verifier):
 
 
 def test_list_files(api, given, verifier):
+    # precondition
     total = 10
     folder_id = generator.uuid4()
     given.file.exists(id=folder_id, type='folder')
@@ -316,6 +359,38 @@ def test_list_files(api, given, verifier):
     verifier.file.folder_files_listed(id=folder_id)
 
 
+def test_list_files_with_token(api, given, verifier):
+    # precondition
+    total = 10
+    folder_id = generator.uuid4()
+    given.file.exists(id=folder_id, type='folder')
+
+    given.file.files_in_folder(
+        num_of_files=total,
+        folder_id=folder_id,
+        scroll=True
+    )
+
+    # action
+    folder = api.files.get(folder_id)
+    response = folder.list_files(cont_token='init')
+
+    # verification
+    assert len(response) == total
+    verifier.file.folder_files_listed_scroll(id=folder_id)
+
+
+def test_list_files_with_token_and_offset(api, given, verifier):
+    # precondition
+    folder_id = generator.uuid4()
+    given.file.exists(id=folder_id, type='folder')
+
+    # action
+    folder = api.files.get(folder_id)
+    with pytest.raises(SbgError):
+        folder.list_files(cont_token='init', offset=10)
+
+
 def test_copy_to_folder(api, given, verifier):
     name = generator.slug()
     new_id = generator.uuid4()

sevenbridges/tests/verifiers.py

@@ -122,6 +122,16 @@ def queried(self, project):
         qs = {'project': [project], 'fields': ['_all'], 'limit': ['10']}
         self.checker.check_url('/files') and self.checker.check_query(qs)
 
+    def queried_with_token(self, project):
+        qs = {'project': [project],
+              'fields': ['_all'],
+              'limit': ['10'],
+              'token': ['init']
+              }
+        self.checker.check_url(
+            '/files/scroll'
+        ) and self.checker.check_query(qs)
+
     def queried_with_parent(self, parent):
         qs = {'parent': [parent], 'fields': ['_all'], 'limit': ['10']}
         self.checker.check_url('/files') and self.checker.check_query(qs)
@@ -182,6 +192,14 @@ def folder_created(self):
     def folder_files_listed(self, id):
         self.checker.check_url('/files/{}/list'.format(id))
 
+    def folder_files_listed_scroll(self, id):
+        qs = {'token': ['init'],
+              'fields': ['_all']
+              }
+        self.checker.check_url(
+            '/files/{}/scroll'.format(id)
+        ) and self.checker.check_query(qs)
+
     def copied_to_folder(self, id):
         self.checker.check_url('/files/{}/actions/copy'.format(id))