support ANNOTATION_MAX_MATCH

biothings/web/query/pipeline.py

@@ -43,10 +43,11 @@ def __init__(self, data):
         super().__init__(200, None, data)
 
 class QueryPipeline():
-    def __init__(self, builder, backend, formatter):
+    def __init__(self, builder, backend, formatter, **settings):
         self.builder = builder
         self.backend = backend
         self.formatter = formatter
+        self.settings = settings
 
     def search(self, q, **options):
         query = self.builder.build(q, **options)
@@ -169,8 +170,8 @@ async def fetch(self, id, **options):
         # the user incomplete matches. usually, when this happends,
         # it indicates bad choices of values as the default fields.
 
-        MAX_MATCH = 1000
-        options['size'] = MAX_MATCH + 1
+        MAX_MATCH = self.settings.get("fetch_max_match", 1000)
+        options['size'] = MAX_MATCH + 1  # err when len(res) > MAX
 
         # "fetch" is a wrapper over "search".
         #----------------------------------------

biothings/web/services/namespace.py

@@ -115,6 +115,7 @@ def _configure_elasticsearch(self):
                 self.fieldnote.get_field_notes(),
                 self.config.AVAILABLE_FIELDS_EXCLUDED
             ),
+            fetch_max_match=self.config.ANNOTATION_MAX_MATCH
         )
         self.elasticsearch.health = ESHealth(
             self.elasticsearch.async_client,

biothings/web/settings/default.py

@@ -142,28 +142,36 @@
 # *****************************************************************************
 ES_QUERY_PIPELINE = 'biothings.web.query.AsyncESQueryPipeline'
 ES_QUERY_BUILDER = 'biothings.web.query.ESQueryBuilder'
+ES_QUERY_BACKEND = 'biothings.web.query.AsyncESQueryBackend'
+ES_RESULT_TRANSFORM = 'biothings.web.query.ESResultFormatter'
+
+# Pipeline
+# --------
+ANNOTATION_MAX_MATCH = 1000
+
+# Builder Stage
+# -------------
 # For the userquery folder for this app
 USERQUERY_DIR = 'userquery'
 # Allow "truly" random order for q= __any__
 ALLOW_RANDOM_QUERY = False
 # Allow facets to be nested with ( )
 ALLOW_NESTED_AGGS = False
 
+# Backend Stage
+# -------------
 # Amount of time a scroll request is kept open
 ES_SCROLL_TIME = '1m'
 # Size of each scroll request return
 ES_SCROLL_SIZE = 1000
 
-
-ES_QUERY_BACKEND = 'biothings.web.query.AsyncESQueryBackend'
-ES_RESULT_TRANSFORM = 'biothings.web.query.ESResultFormatter'
-
-
+# Transform Stage
+# ---------------
 # A list of fields to exclude from metadata/fields endpoint
 AVAILABLE_FIELDS_EXCLUDED = ['all']
 # A path to the available fields notes
 AVAILABLE_FIELDS_NOTES_PATH = ''
-
+# Add "_license" fields in results
 LICENSE_TRANSFORM = {
     # "alias" :  "datasource",
     # "dot.field" :  "datasource"