Add 'count' field for object listing endpoints (#755)

docs/setup-and-configuration.rst

@@ -308,6 +308,7 @@ Extra features:
 * ``remotes`` (comma separated strings) - list of MWDB remotes (experimental)
 * ``enable_hooks`` (0 or 1) - enable plugin hooks
 * ``enable_oidc`` (0 or 1) - enable OIDC (experimental)
+* ``listing_endpoints_count_limit`` (integer) - Limits number of objects returned by listing endpoints. Default is ``1000``.
 
 Registration feature settings:
 

mwdb/core/config.py

@@ -89,6 +89,8 @@ class MWDBConfig(Config):
     s3_storage_bucket_name = key(cast=str, required=False)
     # S3 storage Authorize through IAM role (No credentials)
     s3_storage_iam_auth = key(cast=intbool, required=False)
+    # Limit number of objects returned by listing endpoints
+    listing_endpoints_count_limit = key(cast=int, required=False, default=1000)
 
     # Administrator account login
     admin_login = key(cast=str, required=False, default="admin")

mwdb/resources/blob.py

@@ -56,11 +56,13 @@ def get(self):
         description: |
             Returns list of text blobs matching provided query,
             ordered from the latest one.
+            If you want to fetch older blobs use `older_than` parameter.
 
-            Limited to 10 objects, use `older_than` parameter to fetch more.
+            Number of returned blobs is limited by 'count' parameter
+            (default value is 10).
 
-            Don't rely on maximum count of returned objects
-            because it can be changed/parametrized in future.
+            `Note:` Maximal number of returned blobs is limited in
+            MWDB's configuration (default value is 1 000)
         security:
             - bearerAuth: []
         tags:
@@ -71,7 +73,7 @@ def get(self):
               schema:
                 type: string
               description: |
-                Fetch objects which are older than the object
+                Fetch text blobs which are older than the object
                 specified by identifier.
 
                 Used for pagination
@@ -82,6 +84,13 @@ def get(self):
                 type: string
               description: Filter results using Lucene query
               required: false
+            - in: query
+              name: count
+              schema:
+                type: integer
+              description: Number of objects to return
+              required: false
+              default: 10
         responses:
             200:
                 description: List of text blobs

mwdb/resources/config.py

@@ -171,11 +171,13 @@ def get(self):
         description: |
             Returns list of configs matching provided query,
             ordered from the latest one.
+            If you want to fetch older configs use `older_than` parameter.
 
-            Limited to 10 objects, use `older_than` parameter to fetch more.
+            Number of returned configs is limited by 'count' parameter
+            (default value is 10).
 
-            Don't rely on maximum count of returned objects because it
-            can be changed/parametrized in future.
+            `Note:` Maximal number of returned configs is limited in
+            MWDB's configuration (default value is 1 000)
         security:
             - bearerAuth: []
         tags:
@@ -186,7 +188,7 @@ def get(self):
               schema:
                 type: string
               description: |
-                Fetch objects which are older than the object specified by identifier.
+                Fetch configs which are older than the object specified by identifier.
                 Used for pagination
               required: false
             - in: query
@@ -195,6 +197,13 @@ def get(self):
                 type: string
               description: Filter results using Lucene query
               required: false
+            - in: query
+              name: count
+              schema:
+                type: integer
+              description: Number of objects to return
+              required: false
+              default: 10
         responses:
             200:
                 description: List of configs

mwdb/resources/file.py

@@ -58,13 +58,15 @@ def get(self):
         ---
         summary: Search or list files
         description: |
-            Returns a list of files matching provided query,
+            Returns list of files matching provided query,
             ordered from the latest one.
+            If you want to fetch older files use `older_than` parameter.
 
-            Limited to 10 objects, use `older_than` parameter to fetch more.
+            Number of returned files is limited by 'count' parameter
+            (default value is 10).
 
-            Don't rely on maximum count of returned objects because
-            it can be changed/parametrized in future.
+            `Note:` Maximal number of returned files is limited in
+            MWDB's configuration (default value is 1 000)
         security:
             - bearerAuth: []
         tags:
@@ -75,7 +77,7 @@ def get(self):
               schema:
                 type: string
               description: |
-                Fetch objects which are older than the object specified by identifier.
+                Fetch files which are older than the object specified by identifier.
 
                 Used for pagination
               required: false
@@ -85,6 +87,13 @@ def get(self):
                 type: string
               description: Filter results using Lucene query
               required: false
+            - in: query
+              name: count
+              schema:
+                type: integer
+              description: Number of objects to return
+              required: false
+              default: 10
         responses:
             200:
                 description: List of files

mwdb/resources/object.py

@@ -156,11 +156,13 @@ def get(self):
         description: |
             Returns list of objects matching provided query,
             ordered from the latest one.
+            If you want to fetch older objects use `older_than` parameter.
 
-            Limited to 10 objects, use `older_than` parameter to fetch more.
+            Number of returned objects is limited by 'count' parameter
+            (default value is 10).
 
-            Don't rely on maximum count of returned objects
-            because it can be changed/parametrized in future.
+            `Note:` Maximal number of returned objects is limited in
+            MWDB's configuration (default value is 1 000)
         security:
             - bearerAuth: []
         tags:
@@ -182,6 +184,13 @@ def get(self):
                 type: string
               description: Filter results using Lucene query
               required: false
+            - in: query
+              name: count
+              schema:
+                type: integer
+              description: Number of objects to return
+              required: false
+              default: 10
         responses:
             200:
                 description: List of objects
@@ -222,6 +231,8 @@ def get(self):
         else:
             db_query = db.session.query(self.ObjectType)
 
+        limit = min(obj["count"], app_config.mwdb.listing_endpoints_count_limit)
+
         db_query = db_query.filter(
             g.auth_user.has_access_to_object(Object.id)
         ).order_by(Object.id.desc())
@@ -231,7 +242,7 @@ def get(self):
         elif obj["page"] is not None and obj["page"] > 1:
             db_query = db_query.offset((obj["page"] - 1) * 10)
 
-        objects = db_query.limit(10).all()
+        objects = db_query.limit(limit).all()
 
         schema = self.ListResponseSchema()
         return schema.dump(objects, many=True)

mwdb/schema/object.py

@@ -19,6 +19,7 @@ class ObjectListRequestSchema(Schema):
     page = fields.Int(missing=None)  # legacy, to be removed in future
     query = fields.Str(missing=None)
     older_than = fields.Str(missing=None)
+    count = fields.Int(missing=10)
 
     @validates_schema
     def validate_key(self, data, **kwargs):

tests/backend/test_search.py

@@ -8,6 +8,32 @@
 import random
 
 
+def test_listing_endpoints(admin_session):
+    test = admin_session
+
+    filename = base62uuid()
+    file_content = base62uuid()
+    for i in range(5):
+        test.add_sample(filename, file_content + str(i))
+
+    filename2 = base62uuid()
+    for i in range(7):
+        test.add_sample(filename2, file_content + str(5 + i))
+
+    # empty query, no count value -> default 10
+    listed_obj = test.list_objects()
+    assert len(listed_obj['objects']) == 10
+
+    listed_obj = test.list_objects(count=11)
+    assert len(listed_obj['objects']) == 11
+
+    listed_obj = test.list_objects(query=f'file.name:"{filename}"')
+    assert len(listed_obj['objects']) == 5
+
+    listed_obj = test.list_objects(query=f'file.name:"{filename}"', count=3)
+    assert len(listed_obj['objects']) == 3
+
+
 def test_file_name_search(admin_session):
     test = admin_session
 

tests/backend/utils.py

@@ -245,6 +245,24 @@ def remove_object(self, identifier):
         res = self.session.delete(self.mwdb_url + "/object/" + identifier)
         res.raise_for_status()
         return res.json()
+
+    def list_objects(self, query: str = None, count: int = None):
+        if query is None and count is None:
+            res = self.session.get(self.mwdb_url + "/object")
+            res.raise_for_status()
+            return res.json()
+        elif query is None and count is not None:
+            res = self.session.get(self.mwdb_url + "/object?count=" + str(count))
+            res.raise_for_status()
+            return res.json()
+        elif query is not None and count is None:
+            res = self.session.get(self.mwdb_url + "/object?query=" + query)
+            res.raise_for_status()
+            return res.json()
+        else:
+            res = self.session.get(f"{self.mwdb_url}/object?query={query}&count={count}")
+            res.raise_for_status()
+            return res.json()
 
     def add_tag(self, identifier, tag):
         res = self.session.put(