diff --git a/examples/submission.py b/examples/submission.py index 6be73c76..9dcb0eb0 100644 --- a/examples/submission.py +++ b/examples/submission.py @@ -107,3 +107,13 @@ job = client.call(SubmitReview(submission.id, changes=changes, rejected=rejected)) job = client.call(JobStatus(job.id)) print("Review", job.id, "has result", job.result) + +""" +Example 5 +Use the client paginator to retrieve all PROCESSING submissions +Without the paginator, the hard limit is 1000 +""" +sub_filter = SubmissionFilter(status="PROCESSING") +for submission in client.paginate(ListSubmissions(filters=sub_filter)): + print(f"Submission {submission.id}") + # do other cool things diff --git a/indico/client/client.py b/indico/client/client.py index b0c34906..3975ed12 100644 --- a/indico/client/client.py +++ b/indico/client/client.py @@ -5,7 +5,7 @@ from indico.config import IndicoConfig from indico.http.client import HTTPClient -from indico.client.request import HTTPRequest, RequestChain +from indico.client.request import HTTPRequest, RequestChain, PagedRequest class IndicoClient: @@ -64,3 +64,16 @@ def call(self, request: Union[HTTPRequest, RequestChain]): return self._handle_request_chain(request) elif request and isinstance(request, HTTPRequest): return self._http.execute_request(request) + + def paginate(self, request: PagedRequest): + """ + Provide a generator that continues paging through responses + Available with List<> Requests that offer pagination + + Example: + for s in client.paginate(ListSubmissions()): + print("Submission", s) + """ + while request.has_next_page: + for r in self._http.execute_request(request): + yield r diff --git a/indico/client/request.py b/indico/client/request.py index eea60683..dcb13c8d 100644 --- a/indico/client/request.py +++ b/indico/client/request.py @@ -47,6 +47,40 @@ def process_response(self, response): return response["data"] +class PagedRequest(GraphQLRequest): + """ + To enable pagination, query must include $after as an argument + and request pageInfo + query Name( + ... + $after: Int + ){ + items( + ... + after: $after + ){ + items {...} + pageInfo { + endCursor + hasNextPage + } + } + } + """ + + def __init__(self, query: str, variables: Dict[str, Any] = None): + variables["after"] = None + self.has_next_page = True + super().__init__(query, variables=variables) + + def process_response(self, response): + response = super().process_response(response) + _pg = next(iter(response.values()))["pageInfo"] + self.has_next_page = _pg["hasNextPage"] + self.variables["after"] = _pg["endCursor"] if self.has_next_page else None + return response + + class RequestChain: previous: Any = None result: Any = None diff --git a/indico/queries/submission.py b/indico/queries/submission.py index c903ef79..b68ce02d 100644 --- a/indico/queries/submission.py +++ b/indico/queries/submission.py @@ -4,7 +4,7 @@ from operator import eq, ne from typing import Dict, List, Union -from indico.client.request import GraphQLRequest, RequestChain +from indico.client.request import GraphQLRequest, RequestChain, PagedRequest from indico.errors import IndicoInputError, IndicoTimeoutError from indico.filters import SubmissionFilter from indico.queries import JobStatus @@ -12,9 +12,10 @@ from indico.types.submission import VALID_SUBMISSION_STATUSES -class ListSubmissions(GraphQLRequest): +class ListSubmissions(PagedRequest): """ List all Submissions visible to the authenticated user by most recent. + Supports pagination, where limit is page size Options: submission_ids (List[int]): Submission ids to filter by @@ -23,6 +24,7 @@ class ListSubmissions(GraphQLRequest): limit (int, default=1000): Maximum number of Submissions to return orderBy (str, default="ID"): Submission attribute to filter by desc: (bool, default=True): List in descending order + Returns: List[Submission]: All the found Submission objects """ @@ -35,6 +37,7 @@ class ListSubmissions(GraphQLRequest): $limit: Int, $orderBy: SUBMISSION_COLUMN_ENUM, $desc: Boolean + $after: Int, ){ submissions( submissionIds: $submissionIds, @@ -43,6 +46,7 @@ class ListSubmissions(GraphQLRequest): limit: $limit orderBy: $orderBy, desc: $desc + after: $after ){ submissions { id @@ -55,6 +59,10 @@ class ListSubmissions(GraphQLRequest): retrieved errors } + pageInfo { + endCursor + hasNextPage + } } } """ diff --git a/tests/integration/queries/test_workflow.py b/tests/integration/queries/test_workflow.py index db7a34ce..d0d0c901 100644 --- a/tests/integration/queries/test_workflow.py +++ b/tests/integration/queries/test_workflow.py @@ -138,6 +138,24 @@ def test_list_workflow_submission_retrieved( assert submission_id not in [s.id for s in submissions] +def test_list_workflow_submission_paginate( + indico, airlines_dataset, airlines_model_group: ModelGroup +): + client = IndicoClient() + wfs = client.call(ListWorkflows(dataset_ids=[airlines_dataset.id])) + wf = max(wfs, key=lambda w: w.id) + + dataset_filepath = str(Path(__file__).parents[1]) + "/data/mock.pdf" + + submission_ids = client.call( + WorkflowSubmission(workflow_id=wf.id, files=[dataset_filepath]*5) + ) + for sub in client.paginate(ListSubmissions(workflow_ids=[wf.id], limit=3)): + if not submission_ids: + break + assert sub.id == submission_ids.pop() # list is desc by default + + def test_workflow_submission_missing_workflow(indico): client = IndicoClient() dataset_filepath = str(Path(__file__).parents[1]) + "/data/mock.pdf"