Merge branch '0.10.x'

biothings/web/applications.py

@@ -107,7 +107,7 @@ def _get_handlers(biothings, addons=None):
 
         handlers = list(handlers.values())
         logger.info('API Handlers:\n%s', pformat(handlers, width=200))
-        return handlers  # TODO
+        return handlers
 
     @classmethod
     def get_app(cls, config, settings=None, handlers=None):

biothings/web/query/builder.py

@@ -137,9 +137,6 @@ def build(self, q=None, **options):
         Build a query according to q and options.
         This is the public method called by API handlers.
 
-        Regarding multisearch:
-            TODO
-
         Regarding scopes:
             scopes: [str] nonempty, match query.
             scopes: NoneType, or [], no scope, so query string query.
@@ -151,6 +148,9 @@ def build(self, q=None, **options):
         * additional keywords are passed through as es keywords
             for example: 'explain', 'version' ...
 
+        * multi-search is supported when q is a list. all queries
+            are built individually and then sent in one request.
+
         """
         options = dotdict(options)
 
@@ -346,7 +346,7 @@ def apply_extras(self, search, options):
 
         # the valid values for from and size depend on the
         # index.max_result_window elasticsearch setting.
-        
+
         # more about this constraint on:
         # https://www.elastic.co/guide/en/elasticsearch/
         # reference/current/index-modules.html

biothings/web/query/engine.py

@@ -1,6 +1,26 @@
 """
-    TODO
+    Search Execution Engine
+
+    Take the output of the query builder and feed 
+    to the corresponding database engine. This stage
+    typically resolves the db destination from a
+    biothing_type and applies presentation and/or
+    networking parameters.
+
+    Example:
+
+    >>> from biothings.web.query import ESQueryBackend
+    >>> from elasticsearch import Elasticsearch
+    >>> from elasticsearch_dsl import Search
+
+    >>> backend = ESQueryBackend(Elasticsearch())
+    >>> backend.execute(Search().query("match", _id="1017"))
+    
+    >>> _["hits"]["hits"][0]["_source"].keys()
+    dict_keys(['taxid', 'symbol', 'name', ... ])
+
 """
+
 import asyncio
 
 from biothings.web.query.builder import ESScrollID

biothings/web/query/formatter.py

@@ -1,6 +1,13 @@
 """
-    TODO
+    Search Result Formatter
+
+    Transform the raw query result into consumption-friendly
+    structures by possibly removing from, adding to, and/or
+    flattening the raw response from the database engine for
+    one or more individual queries. 
+
 """
+
 from collections import defaultdict, UserDict
 
 from biothings.utils.common import dotdict, traverse
@@ -284,13 +291,13 @@ def _sorted(_, obj):
                 obj.clear()
                 obj.update(sorted_items)
         except Exception:
-            pass  # TODO logging
+            pass
 
     @classmethod
     def _dotfield(cls, dic, options):
         """
         Flatten a dictionary.
-        #TODO examples
+        See biothings.utils.common.traverse for examples.
         """
         hit_ = defaultdict(list)
         for path, value in cls.traverse(dic, leaf_node=True):
@@ -405,18 +412,18 @@ def transform_aggs(self, res):
     def transform_mapping(self, mapping, prefix=None, search=None):
         """
         Transform Elasticsearch mapping definition to
-        user-friendly field definitions metadata result
+        user-friendly field definitions metadata results.
         """
         assert isinstance(mapping, dict)
         assert isinstance(prefix, str) or prefix is None
         assert isinstance(search, str) or search is None
 
         result = {}
-        todo = list(mapping.items())
-        todo.reverse()
+        items = list(mapping.items())
+        items.reverse()
 
-        while todo:
-            key, dic = todo.pop()
+        while items:
+            key, dic = items.pop()
             dic = dict(dic)
             dic.pop('dynamic', None)
             dic.pop('normalizer', None)
@@ -438,7 +445,7 @@ def transform_mapping(self, mapping, prefix=None, search=None):
             if 'properties' in dic:
                 dic['type'] = 'object'
                 subs = (('.'.join((key, k)), v) for k, v in dic['properties'].items())
-                todo.extend(reversed(list(subs)))
+                items.extend(reversed(list(subs)))
                 del dic['properties']
 
             if all((not self.excluded_keys or key not in self.excluded_keys,

biothings/web/query/pipeline.py

@@ -70,7 +70,11 @@ def _simplify_ES_exception(exc, debug=False):
     except IndexError:
         pass  # no root cause
     except Exception:
-        logger.exception('Error parsing es exception.')  # TODO
+        logger.exception(' '.join((
+            "Unexpected error in _simplify_ES_exception.",
+            "Caused by incompatible version, build, etc.",
+            "Update ES exception parsing logic here."
+        )))
 
     if debug:  # raw ES error response
         result["debug"] = exc.info

biothings/web/services/health.py

@@ -50,11 +50,11 @@ class ESHealth(DBHealth):
     #     "active_shards_percent_as_number": 83.33333333333334
     # }
 
-    # TODO
+    # Future Work A
     # Add transport level cluster information,
     # like connected nodes, etc
 
-    # TODO
+    # Future Work B
     # It is useful to provide two endpoints,
     # one indicating the service status, factoring in more
     # measurement dimensions like cluster health and data integrity,

biothings/web/services/metadata.py

@@ -373,6 +373,11 @@ class BiothingMetaProp:
     def __add__(self, other):
         raise NotImplementedError
 
+        # NOTE
+        # The current implementations below
+        # may not be able to properly handle
+        # field (key/source) collisions.
+
     def to_dict(self):
         raise NotImplementedError
 
@@ -382,7 +387,6 @@ def __init__(self, licenses):
         self.licenses = licenses
 
     def __add__(self, other):
-        # TODO log conflicts
         licenses = dict(self.licenses)
         licenses.update(other.licenses)
         return BiothingLicenses(licenses)
@@ -396,7 +400,6 @@ def __init__(self, properties):
         self.properties = properties
 
     def __add__(self, other):
-        # TODO conflicting fields
         mappings = dict(self.properties)
         mappings.update(other.properties)
         return BiothingMappings(mappings)

biothings/web/services/namespace.py

@@ -112,7 +112,7 @@ def _configure_elasticsearch(self):
             load_class(self.config.ES_RESULT_TRANSFORM)(
                 self.elasticsearch.metadata.biothing_licenses,
                 self.config.LICENSE_TRANSFORM,
-                self.fieldnote.get_field_notes(),  # TODO
+                self.fieldnote.get_field_notes(),
                 self.config.AVAILABLE_FIELDS_EXCLUDED
             ),
         )

biothings/web/settings/default.py

@@ -126,10 +126,8 @@
     }
 }
 
-# REMOVE THESE COMPATIBILITY SETTINGS
+# LONG TERM GOAL: REMOVE THESE COMPATIBILITY SETTINGS
 # ONCE BIOTHINGS.CLIENT OLDER VERSIONS ARE NO LONGER USED
-# TODO FINALIZE WHAT TO PUT HERE & REMOVE FROM ABOVE
-# TODO VALIDATE COMMON PARAM STRICTNESS IS PROPAGATED
 COMMON_KWARGS['_source']['strict'] = False
 COMMON_KWARGS['always_list']['strict'] = False
 COMMON_KWARGS['allow_null']['strict'] = False
@@ -160,7 +158,7 @@
 ES_QUERY_BACKEND = 'biothings.web.query.AsyncESQueryBackend'
 ES_RESULT_TRANSFORM = 'biothings.web.query.ESResultFormatter'
 
-# TODO
+
 # A list of fields to exclude from metadata/fields endpoint
 AVAILABLE_FIELDS_EXCLUDED = ['all']
 # A path to the available fields notes
@@ -185,18 +183,18 @@
 # Endpoints Specifics & Others
 # *****************************************************************************
 #
-# Base Query Handler
+# Search
 HTML_OUT_TITLE = ""  # HTML
 HTML_OUT_HEADER_IMG = ""  # URL
 HTML_OUT_ANNOTATION_DOCS = ""  # URL
 HTML_OUT_METADATA_DOCS = ""  # URL
 HTML_OUT_QUERY_DOCS = ""  # URL
 #
-# Annotation # TODO THESES SETTINGS WILL BECOME QUERY PIPELINE SETTINGS
+# Annotation
 ANNOTATION_DEFAULT_SCOPES = ['_id']
 ANNOTATION_ID_REGEX_LIST = []  # [(re.compile(r'rs[0-9]+', re.I), 'dbsnp.rsid')]
 #
-# Status #
+# Status
 # https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-get.html
 STATUS_CHECK = {
     # 'index': ''

biothings/web/settings/validators.py

@@ -33,8 +33,6 @@ def validate(self, config):
                 config.ES_INDICES = dict(config.ES_INDICES)
             config.ES_INDICES[config.ES_DOC_TYPE] = config.ES_INDEX
 
-        # assert '*' not in config.ES_DOC_TYPE TODO
-
         ERROR = "UNSUPPORTED SETTINGS."
         # encountering the following attributes indicate
         # the application is built for a previous version.

tests/web/handlers/test_annotation.py

@@ -359,5 +359,4 @@ def test_13_json_invalid(self):
         assert not res['success']
         assert res['error'] == "Invalid JSON body."
 
-    # TODO
-    # Add multiple hit test case
+    # TODO Add multiple hit test case

tests/web/handlers/test_query.py

@@ -248,41 +248,6 @@ def test_16_sort_desc(self):
         assert res['hits'][1]["taxid"] == 1868482
         assert res['hits'][2]["taxid"] == 1841481
 
-    def test_16_sorted_dotfield(self):
-        """ GET /v1/query?q=1017&dotfield
-        {
-            "took": 17,
-            "total": 1,
-            "max_score": 3.0910425,
-            "hits": [
-                {
-                    "_id": "1017",
-                    "_score": 3.0910425,
-                    "accession.translation.protein": [
-                        "AAA35667.1",
-                        "AAH03065.1",
-                        "AAP35467.1",
-                        "BAA32794.1",
-                        "BAF84630.1",
-                        "BAG56780.1",
-                        "CAA43807.1",
-                        "CAA43985.1",
-                        ...
-                    ],  // this field is flattened
-                    ...
-                },
-            ]
-        }
-        """
-        # res = self.request('/v1/query?q=1017&dotfield').json()
-        # protein = res['hits'][0]['accession.translation.protein']
-        # for index, value in enumerate(protein):
-        #     if index > 0:
-        #         assert value > protein[index-1]
-        #
-        # TODO list sorting feature is under revision
-        #
-
     def test_17_sorted_false_dotfield(self):
         """ GET /v1/query?q=1017&dotfield&_sorted=false
         {
@@ -632,7 +597,6 @@ def test_31_scroll_stale(self):
         assert res['success'] is False
 
 
-
 class TestQueryString(BiothingsWebAppTest):
 
     def test_00_all(self):