From 8b8bd936d9e9c5c9df00398829ace762f58c07ec Mon Sep 17 00:00:00 2001 From: Einar Huseby Date: Mon, 28 Jan 2019 22:44:12 +0100 Subject: [PATCH 1/8] Implemented draft count replacement for newer pymongo --- eve/default_settings.py | 8 +++++--- eve/io/mongo/mongo.py | 23 ++++++++++++++++++++++- eve/methods/get.py | 5 ++--- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/eve/default_settings.py b/eve/default_settings.py index 7d092fb3c..848eff4e4 100644 --- a/eve/default_settings.py +++ b/eve/default_settings.py @@ -11,6 +11,8 @@ :copyright: (c) 2017 by Nicola Iarocci. :license: BSD, see LICENSE for more details. + .. versionchanged:: 0.8.2 + 'PAGINATION_STRATEGY' options are None, full and estimate and set to full. .. versionchanged:: 0.8 'RENDERERS' added with XML and JSON renderers. 'JSON' removed. @@ -164,6 +166,9 @@ PAGINATION = True # pagination enabled by default. PAGINATION_LIMIT = 50 PAGINATION_DEFAULT = 25 +HEADER_TOTAL_COUNT = "X-Total-Count" +PAGINATION_STRATEGY = "full" +OPTIMIZE_PAGINATION_FOR_SPEED = False VERSIONING = False # turn document versioning on or off. VERSIONS = "_versions" # suffix for parallel collection w/old versions VERSION_PARAM = "version" # URL param for specific version of a document. @@ -238,9 +243,6 @@ QUERY_EMBEDDED = "embedded" QUERY_AGGREGATION = "aggregate" -HEADER_TOTAL_COUNT = "X-Total-Count" -OPTIMIZE_PAGINATION_FOR_SPEED = False - # user-restricted resource access is disabled by default. AUTH_FIELD = None diff --git a/eve/io/mongo/mongo.py b/eve/io/mongo/mongo.py index fa308bbea..d6efa5532 100644 --- a/eve/io/mongo/mongo.py +++ b/eve/io/mongo/mongo.py @@ -149,6 +149,10 @@ def find(self, resource, req, sub_resource_lookup): :param req: a :class:`ParsedRequest`instance. :param sub_resource_lookup: sub-resource lookup from the endpoint url. + .. versionchanged:: 0.8.2 + Datalayer breaking changes for count using PyMongo>=3.7 + Support for PAGINATION_STRATEGY + .. versionchanged:: 0.6 Support for multiple databases. Filter soft deleted documents by default @@ -249,7 +253,24 @@ def find(self, resource, req, sub_resource_lookup): if projection: args["projection"] = projection - return self.pymongo(resource).db[datasource].find(**args) + # PAGINATION STRATEGY for resource + if config.DOMAIN[resource]["pagination_strategy"] is not None: + pagination_strategy = config.DOMAIN[resource]["pagination_strategy"] + elif config.PAGINATION_STRATEGY is not None: + pagination_strategy = config.PAGINATION_STRATEGY + + if pagination_strategy == "full": + return ( + self.pymongo(resource).db[datasource].count_documents(**args), + self.pymongo(resource).db[datasource].find(**args), + ) + elif pagination_strategy == "estimated": + return ( + self.pymongo(resource).db[datasource].estimated_document_count(**args), + self.pymongo(resource).db[datasource].find(**args), + ) + else: + return (None, self.pymongo(resource).db[datasource].find(**args)) def find_one( self, diff --git a/eve/methods/get.py b/eve/methods/get.py index f068e813b..6b29f2648 100644 --- a/eve/methods/get.py +++ b/eve/methods/get.py @@ -405,11 +405,11 @@ def getitem_internal(resource, **lookup): # default sort for 'all', required sort for 'diffs' req.sort = '[("%s", 1)]' % config.VERSION req.if_modified_since = None # we always want the full history here - cursor = app.data.find(resource + config.VERSIONS, req, lookup) + count, cursor = app.data.find(resource + config.VERSIONS, req, lookup) # build all versions documents = [] - if cursor.count() == 0: + if count == 0: # this is the scenario when the document existed before # document versioning got turned on documents.append(latest_doc) @@ -461,7 +461,6 @@ def getitem_internal(resource, **lookup): if config.DOMAIN[resource]["hateoas"]: # use the id of the latest document for multi-document requests if cursor: - count = cursor.count(with_limit_and_skip=False) response[config.LINKS] = _pagination_links( resource, req, count, latest_doc[resource_def["id_field"]] ) From 4c7a5fcfffb5477328778b2ca2350700cf98f338 Mon Sep 17 00:00:00 2001 From: Einar Huseby Date: Tue, 29 Jan 2019 00:45:20 +0100 Subject: [PATCH 2/8] Working version, benchmarked --- eve/default_settings.py | 6 ++++++ eve/io/mongo/mongo.py | 30 +++++++++++++++++++++++++++--- eve/methods/get.py | 5 ++++- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/eve/default_settings.py b/eve/default_settings.py index 848eff4e4..91059aa91 100644 --- a/eve/default_settings.py +++ b/eve/default_settings.py @@ -167,7 +167,13 @@ PAGINATION_LIMIT = 50 PAGINATION_DEFAULT = 25 HEADER_TOTAL_COUNT = "X-Total-Count" + +# http://api.mongodb.com/python/current/api/pymongo/collection.html +# full=count_documents(filter, session=None, **kwargs) interesting options are hint=index name or index spec and maxTimeMS +# estimated=estimated_document_count(**kwargs) also maxTimeMS can be given! +# None=None PAGINATION_STRATEGY = "full" + OPTIMIZE_PAGINATION_FOR_SPEED = False VERSIONING = False # turn document versioning on or off. VERSIONS = "_versions" # suffix for parallel collection w/old versions diff --git a/eve/io/mongo/mongo.py b/eve/io/mongo/mongo.py index d6efa5532..0a329f564 100644 --- a/eve/io/mongo/mongo.py +++ b/eve/io/mongo/mongo.py @@ -254,19 +254,43 @@ def find(self, resource, req, sub_resource_lookup): args["projection"] = projection # PAGINATION STRATEGY for resource - if config.DOMAIN[resource]["pagination_strategy"] is not None: + if config.DOMAIN[resource].get("pagination_strategy", None) is not None: pagination_strategy = config.DOMAIN[resource]["pagination_strategy"] elif config.PAGINATION_STRATEGY is not None: pagination_strategy = config.PAGINATION_STRATEGY + else: + pagination_strategy = None + + if pagination_strategy is not None: + args_count = args.copy() + args_count.pop("sort", None) + args_count.pop("projection", None) + args_count.pop("limit", None) + if "filter" not in args_count: + args_count["filter"] = {} + + # Operator Replacement + # $where $expr + # $near $geoWithin with $center + # $nearSphere $geoWithin with $centerSphere + # args_count["filter"] = args_count["filter"].replace("$where", "$expr", 1) + # args_count["filter"].replace("$geoWithin", "$center") + # args_count["filter"].replace("$where", "$expr") + + print("COUNT", args_count) + print("ARGS ", args) + # args_count.pop("sort", None) if pagination_strategy == "full": return ( - self.pymongo(resource).db[datasource].count_documents(**args), + self.pymongo(resource).db[datasource].count_documents(**args_count), self.pymongo(resource).db[datasource].find(**args), ) elif pagination_strategy == "estimated": return ( - self.pymongo(resource).db[datasource].estimated_document_count(**args), + self.pymongo(resource) + .db[datasource] + .estimated_document_count(**args_count), self.pymongo(resource).db[datasource].find(**args), ) else: diff --git a/eve/methods/get.py b/eve/methods/get.py index 6b29f2648..a13bab975 100644 --- a/eve/methods/get.py +++ b/eve/methods/get.py @@ -213,7 +213,7 @@ def _perform_find(resource, lookup): # If-Modified-Since disabled on collections (#334) req.if_modified_since = None - cursor = app.data.find(resource, req, lookup) + count, cursor = app.data.find(resource, req, lookup) # If soft delete is enabled, data.find will not include items marked # deleted unless req.show_deleted is True for document in cursor: @@ -230,10 +230,13 @@ def _perform_find(resource, lookup): response[config.ITEMS] = documents + """ if config.OPTIMIZE_PAGINATION_FOR_SPEED: count = None else: count = cursor.count(with_limit_and_skip=False) + """ + if count is not None: headers.append((config.HEADER_TOTAL_COUNT, count)) if config.DOMAIN[resource]["hateoas"]: From ea45bcd9249e7a4a4bf725cd25a93321df4774a1 Mon Sep 17 00:00:00 2001 From: Einar Huseby Date: Tue, 29 Jan 2019 00:46:16 +0100 Subject: [PATCH 3/8] Removed redudant paranthesis --- eve/io/mongo/mongo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eve/io/mongo/mongo.py b/eve/io/mongo/mongo.py index 0a329f564..6c6aa2873 100644 --- a/eve/io/mongo/mongo.py +++ b/eve/io/mongo/mongo.py @@ -294,7 +294,7 @@ def find(self, resource, req, sub_resource_lookup): self.pymongo(resource).db[datasource].find(**args), ) else: - return (None, self.pymongo(resource).db[datasource].find(**args)) + return None, self.pymongo(resource).db[datasource].find(**args) def find_one( self, From 2ccfdaeda849b13b812dc3d4ab6736c5b28a5f25 Mon Sep 17 00:00:00 2001 From: Einar Huseby Date: Tue, 29 Jan 2019 12:13:06 +0100 Subject: [PATCH 4/8] Only using filter for count queries --- eve/io/mongo/mongo.py | 28 +++++----------------------- 1 file changed, 5 insertions(+), 23 deletions(-) diff --git a/eve/io/mongo/mongo.py b/eve/io/mongo/mongo.py index 6c6aa2873..4961927e6 100644 --- a/eve/io/mongo/mongo.py +++ b/eve/io/mongo/mongo.py @@ -253,44 +253,26 @@ def find(self, resource, req, sub_resource_lookup): if projection: args["projection"] = projection - # PAGINATION STRATEGY for resource + # pagination_strategy for resource if config.DOMAIN[resource].get("pagination_strategy", None) is not None: pagination_strategy = config.DOMAIN[resource]["pagination_strategy"] + # Global PAGINATION_STRATEGY elif config.PAGINATION_STRATEGY is not None: pagination_strategy = config.PAGINATION_STRATEGY else: pagination_strategy = None - if pagination_strategy is not None: - args_count = args.copy() - args_count.pop("sort", None) - args_count.pop("projection", None) - args_count.pop("limit", None) - if "filter" not in args_count: - args_count["filter"] = {} - - # Operator Replacement - # $where $expr - # $near $geoWithin with $center - # $nearSphere $geoWithin with $centerSphere - # args_count["filter"] = args_count["filter"].replace("$where", "$expr", 1) - # args_count["filter"].replace("$geoWithin", "$center") - # args_count["filter"].replace("$where", "$expr") - - print("COUNT", args_count) - print("ARGS ", args) - # args_count.pop("sort", None) - + # Execute strategy if pagination_strategy == "full": return ( - self.pymongo(resource).db[datasource].count_documents(**args_count), + self.pymongo(resource).db[datasource].count_documents(**args["filter"]), self.pymongo(resource).db[datasource].find(**args), ) elif pagination_strategy == "estimated": return ( self.pymongo(resource) .db[datasource] - .estimated_document_count(**args_count), + .estimated_document_count(**args["filter"]), self.pymongo(resource).db[datasource].find(**args), ) else: From a23c2f9854f394810214d0f602511554b3a7afd7 Mon Sep 17 00:00:00 2001 From: Einar Huseby Date: Tue, 29 Jan 2019 12:24:57 +0100 Subject: [PATCH 5/8] Using only filter from args --- eve/io/mongo/mongo.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/eve/io/mongo/mongo.py b/eve/io/mongo/mongo.py index 4961927e6..cae317380 100644 --- a/eve/io/mongo/mongo.py +++ b/eve/io/mongo/mongo.py @@ -265,14 +265,16 @@ def find(self, resource, req, sub_resource_lookup): # Execute strategy if pagination_strategy == "full": return ( - self.pymongo(resource).db[datasource].count_documents(**args["filter"]), + self.pymongo(resource) + .db[datasource] + .count_documents(filter=args["filter"]), self.pymongo(resource).db[datasource].find(**args), ) elif pagination_strategy == "estimated": return ( self.pymongo(resource) .db[datasource] - .estimated_document_count(**args["filter"]), + .estimated_document_count(filter=args["filter"]), self.pymongo(resource).db[datasource].find(**args), ) else: From 217e1d527f941df1da34df1f1b9cd6616e0b7b76 Mon Sep 17 00:00:00 2001 From: Einar Huseby Date: Tue, 29 Jan 2019 12:27:45 +0100 Subject: [PATCH 6/8] Pymongo to 3.7 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6e5d85341..562683606 100755 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ "cerberus>=1.1", "events>=0.3,<0.4", "flask>=1.0", - "pymongo>=3.5", + "pymongo>=3.7", "simplejson>=3.3.0,<4.0", ] From edcacfb10b6ad6e524fb8c121d64b2847085f045 Mon Sep 17 00:00:00 2001 From: Einar Huseby Date: Tue, 29 Jan 2019 16:01:03 +0100 Subject: [PATCH 7/8] Corrected usage of estimated_document_count --- eve/io/mongo/mongo.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eve/io/mongo/mongo.py b/eve/io/mongo/mongo.py index cae317380..4a559f5e2 100644 --- a/eve/io/mongo/mongo.py +++ b/eve/io/mongo/mongo.py @@ -263,11 +263,11 @@ def find(self, resource, req, sub_resource_lookup): pagination_strategy = None # Execute strategy - if pagination_strategy == "full": + if pagination_strategy == "full" or ( + pagination_strategy == "estimated" and len(spec) > 0 + ): return ( - self.pymongo(resource) - .db[datasource] - .count_documents(filter=args["filter"]), + self.pymongo(resource).db[datasource].count_documents(filter=spec), self.pymongo(resource).db[datasource].find(**args), ) elif pagination_strategy == "estimated": From 8eb61007db6adff9feb6f7aee510b434d2bec849 Mon Sep 17 00:00:00 2001 From: Einar Huseby Date: Tue, 29 Jan 2019 16:09:07 +0100 Subject: [PATCH 8/8] Removed filter for estimated_document_count --- eve/io/mongo/mongo.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/eve/io/mongo/mongo.py b/eve/io/mongo/mongo.py index 4a559f5e2..74c50b252 100644 --- a/eve/io/mongo/mongo.py +++ b/eve/io/mongo/mongo.py @@ -272,9 +272,7 @@ def find(self, resource, req, sub_resource_lookup): ) elif pagination_strategy == "estimated": return ( - self.pymongo(resource) - .db[datasource] - .estimated_document_count(filter=args["filter"]), + self.pymongo(resource).db[datasource].estimated_document_count(), self.pymongo(resource).db[datasource].find(**args), ) else: