[DEV-5635] ADD: pagination to client

examples/submission.py

@@ -107,3 +107,13 @@
 job = client.call(SubmitReview(submission.id, changes=changes, rejected=rejected))
 job = client.call(JobStatus(job.id))
 print("Review", job.id, "has result", job.result)
+
+"""
+Example 5
+Use the client paginator to retrieve all PROCESSING submissions
+Without the paginator, the hard limit is 1000
+"""
+sub_filter = SubmissionFilter(status="PROCESSING")
+for submission in client.paginate(ListSubmissions(filters=sub_filter)):
+    print(f"Submission {submission.id}")
+    # do other cool things

indico/client/client.py

@@ -5,7 +5,7 @@
 
 from indico.config import IndicoConfig
 from indico.http.client import HTTPClient
-from indico.client.request import HTTPRequest, RequestChain
+from indico.client.request import HTTPRequest, RequestChain, PagedRequest
 
 
 class IndicoClient:
@@ -64,3 +64,16 @@ def call(self, request: Union[HTTPRequest, RequestChain]):
             return self._handle_request_chain(request)
         elif request and isinstance(request, HTTPRequest):
             return self._http.execute_request(request)
+
+    def paginate(self, request: PagedRequest):
+        """
+        Provide a generator that continues paging through responses
+        Available with List<> Requests that offer pagination
+
+        Example:
+            for s in client.paginate(ListSubmissions()):
+                print("Submission", s)
+        """
+        while request.has_next_page:
+            for r in self._http.execute_request(request):
+                yield r

indico/client/request.py

@@ -47,6 +47,40 @@ def process_response(self, response):
         return response["data"]
 
 
+class PagedRequest(GraphQLRequest):
+    """
+    To enable pagination, query must include $after as an argument
+    and request pageInfo
+        query Name(
+            ...
+            $after: Int
+        ){
+            items(
+                ...
+                after: $after
+            ){
+                items {...}
+                pageInfo {
+                    endCursor
+                    hasNextPage
+                }
+            }
+        }
+    """
+
+    def __init__(self, query: str, variables: Dict[str, Any] = None):
+        variables["after"] = None
+        self.has_next_page = True
+        super().__init__(query, variables=variables)
+
+    def process_response(self, response):
+        response = super().process_response(response)
+        _pg = next(iter(response.values()))["pageInfo"]
+        self.has_next_page = _pg["hasNextPage"]
+        self.variables["after"] = _pg["endCursor"] if self.has_next_page else None
+        return response
+
+
 class RequestChain:
     previous: Any = None
     result: Any = None

indico/queries/submission.py

@@ -4,17 +4,18 @@
 from operator import eq, ne
 from typing import Dict, List, Union
 
-from indico.client.request import GraphQLRequest, RequestChain
+from indico.client.request import GraphQLRequest, RequestChain, PagedRequest
 from indico.errors import IndicoInputError, IndicoTimeoutError
 from indico.filters import SubmissionFilter
 from indico.queries import JobStatus
 from indico.types import Job, Submission
 from indico.types.submission import VALID_SUBMISSION_STATUSES
 
 
-class ListSubmissions(GraphQLRequest):
+class ListSubmissions(PagedRequest):
     """
     List all Submissions visible to the authenticated user by most recent.
+    Supports pagination, where limit is page size
 
     Options:
         submission_ids (List[int]): Submission ids to filter by
@@ -23,6 +24,7 @@ class ListSubmissions(GraphQLRequest):
         limit (int, default=1000): Maximum number of Submissions to return
         orderBy (str, default="ID"): Submission attribute to filter by
         desc: (bool, default=True): List in descending order
+
     Returns:
         List[Submission]: All the found Submission objects
     """
@@ -35,6 +37,7 @@ class ListSubmissions(GraphQLRequest):
             $limit: Int,
             $orderBy: SUBMISSION_COLUMN_ENUM,
             $desc: Boolean
+            $after: Int,
         ){
             submissions(
                 submissionIds: $submissionIds,
@@ -43,6 +46,7 @@ class ListSubmissions(GraphQLRequest):
                 limit: $limit
                 orderBy: $orderBy,
                 desc: $desc
+                after: $after
             ){
                 submissions {
                     id
@@ -55,6 +59,10 @@ class ListSubmissions(GraphQLRequest):
                     retrieved
                     errors
                 }
+                pageInfo {
+                    endCursor
+                    hasNextPage
+                }
             }
         }
     """

tests/integration/queries/test_workflow.py

@@ -138,6 +138,24 @@ def test_list_workflow_submission_retrieved(
     assert submission_id not in [s.id for s in submissions]
 
 
+def test_list_workflow_submission_paginate(
+    indico, airlines_dataset, airlines_model_group: ModelGroup
+):
+    client = IndicoClient()
+    wfs = client.call(ListWorkflows(dataset_ids=[airlines_dataset.id]))
+    wf = max(wfs, key=lambda w: w.id)
+
+    dataset_filepath = str(Path(__file__).parents[1]) + "/data/mock.pdf"
+
+    submission_ids = client.call(
+        WorkflowSubmission(workflow_id=wf.id, files=[dataset_filepath]*5)
+    )
+    for sub in client.paginate(ListSubmissions(workflow_ids=[wf.id], limit=3)):
+        if not submission_ids:
+            break
+        assert sub.id == submission_ids.pop()  # list is desc by default
+
+
 def test_workflow_submission_missing_workflow(indico):
     client = IndicoClient()
     dataset_filepath = str(Path(__file__).parents[1]) + "/data/mock.pdf"