From 1ecb428b2150dec1aff2189a169766bf83d07724 Mon Sep 17 00:00:00 2001 From: Astha Patni Date: Mon, 25 Oct 2021 12:46:28 -0400 Subject: [PATCH] [CAT-126] ADD: pagination to ListSubmissions --- examples/submission.py | 10 +++ indico/queries/document_report.py | 79 +++++++++++----------- indico/queries/submission.py | 16 +++-- tests/integration/queries/test_workflow.py | 19 +++++- 4 files changed, 80 insertions(+), 44 deletions(-) diff --git a/examples/submission.py b/examples/submission.py index 4ad6099b..9687f5e6 100644 --- a/examples/submission.py +++ b/examples/submission.py @@ -105,3 +105,13 @@ job = client.call(SubmitReview(submission.id, changes=changes, rejected=rejected)) job = client.call(JobStatus(job.id)) print("Review", job.id, "has result", job.result) + +""" +Example 5 +Use the client paginator to retrieve all PROCESSING submissions +Without the paginator, the hard limit is 1000 +""" +sub_filter = SubmissionFilter(status="PROCESSING") +for submission in client.paginate(ListSubmissions(filters=sub_filter)): + print(f"Submission {submission.id}") + # do other cool things diff --git a/indico/queries/document_report.py b/indico/queries/document_report.py index eaac1a2e..0be070f0 100644 --- a/indico/queries/document_report.py +++ b/indico/queries/document_report.py @@ -11,46 +11,49 @@ class _DocumentReportList(BaseType): submissions: List[DocumentReport] pass + class GetDocumentReport(PagedRequest): - """ - Query to generate a Document Report. - Generates a paged request and paged response. - See examples for a sample query. - """ - query = """ - query SubmissionsLog($filters: SubmissionLogFilter, $limit: Int){ - submissionsLog(filters: $filters, limit: $limit){ - submissions{ - datasetId - workflowId - status - createdAt - updatedAt - updatedBy - completedAt - errors - retrieved - inputFiles{ - filename - submissionId - } - } - pageInfo{ - startCursor - endCursor - hasNextPage - aggregateCount - } - } -} - """ + """ + Query to generate a Document Report. + Generates a paged request and paged response. + See examples for a sample query. + """ - def __init__(self, filters: Union[dict, DocumentReportFilter] = None, limit: int = None): - variables = { - "filters": filters, - "limit": limit + query = """ + query SubmissionsLog($filters: SubmissionLogFilter, $limit: Int, $after: Int){ + submissionsLog(filters: $filters, limit: $limit, after: $after){ + submissions{ + datasetId + workflowId + status + createdAt + updatedAt + updatedBy + completedAt + errors + retrieved + inputFiles{ + filename + submissionId + } + } + pageInfo{ + startCursor + endCursor + hasNextPage + aggregateCount + } + } } + """ + + def __init__( + self, filters: Union[dict, DocumentReportFilter] = None, limit: int = None + ): + variables = {"filters": filters, "limit": limit} super().__init__(self.query, variables=variables) - def process_response(self, response): - return _DocumentReportList(**super().process_response(response)["submissionsLog"]).submissions + def process_response(self, response): + return _DocumentReportList( + **super().process_response(response)["submissionsLog"] + ).submissions diff --git a/indico/queries/submission.py b/indico/queries/submission.py index 52ff9d92..f2da54ef 100644 --- a/indico/queries/submission.py +++ b/indico/queries/submission.py @@ -4,7 +4,7 @@ from operator import eq, ne from typing import Dict, List, Union -from indico.client.request import GraphQLRequest, RequestChain +from indico.client.request import GraphQLRequest, RequestChain, PagedRequest from indico.errors import IndicoInputError, IndicoTimeoutError from indico.filters import SubmissionFilter from indico.queries import JobStatus @@ -12,9 +12,10 @@ from indico.types.submission import VALID_SUBMISSION_STATUSES -class ListSubmissions(GraphQLRequest): +class ListSubmissions(PagedRequest): """ List all Submissions visible to the authenticated user by most recent. + Supports pagination (limit becomes page_size) Options: submission_ids (List[int]): Submission ids to filter by @@ -26,6 +27,7 @@ class ListSubmissions(GraphQLRequest): Returns: List[Submission]: All the found Submission objects + If paginated, yields results one at a time """ query = """ @@ -35,7 +37,8 @@ class ListSubmissions(GraphQLRequest): $filters: SubmissionFilter, $limit: Int, $orderBy: SUBMISSION_COLUMN_ENUM, - $desc: Boolean + $desc: Boolean, + $after: Int ){ submissions( @@ -44,7 +47,8 @@ class ListSubmissions(GraphQLRequest): filters: $filters, limit: $limit orderBy: $orderBy, - desc: $desc + desc: $desc, + after: $after ){ submissions { @@ -59,6 +63,10 @@ class ListSubmissions(GraphQLRequest): retrieved errors } + pageInfo { + endCursor + hasNextPage + } } } """ diff --git a/tests/integration/queries/test_workflow.py b/tests/integration/queries/test_workflow.py index 02aabff9..f359427b 100644 --- a/tests/integration/queries/test_workflow.py +++ b/tests/integration/queries/test_workflow.py @@ -1,5 +1,3 @@ -import io - from indico.queries.workflow import GetWorkflow import pytest from pathlib import Path @@ -248,6 +246,23 @@ def test_list_workflow_submission_retrieved( assert all([not s.retrieved for s in submissions]) assert submission_id not in [s.id for s in submissions] +def test_list_workflow_submission_paginate( + indico, airlines_dataset, airlines_model_group: ModelGroup +): + client = IndicoClient() + wfs = client.call(ListWorkflows(dataset_ids=[airlines_dataset.id])) + wf = max(wfs, key=lambda w: w.id) + + dataset_filepath = str(Path(__file__).parents[1]) + "/data/mock.pdf" + + submission_ids = client.call( + WorkflowSubmission(workflow_id=wf.id, files=[dataset_filepath]*5) + ) + for sub in client.paginate(ListSubmissions(workflow_ids=[wf.id], limit=3)): + + if not submission_ids: + break + assert sub.id == submission_ids.pop() # list is desc by default def test_workflow_submission_missing_workflow(indico): client = IndicoClient()