Merge pull request #175 from pagreene/service-bug-fixes

Minor bug-fixes in the service

doc/index.rst

@@ -9,8 +9,8 @@ Further INDRA Database documentation
    modules/index.rst
 
 
-INDRA Database REST API
-=======================
+INDRA Database REST Service
+===========================
 
 .. toctree::
    :maxdepth: 3

doc/requirements.txt

@@ -5,7 +5,7 @@ ipython
 matplotlib
 future
 requests
-m2r
+m2r2
 cachetools
 git+https://github.com/sorgerlab/indra.git
 

doc/rest_api_doc/readme_link.rst

@@ -1,2 +1,2 @@
-.. mdinclude:: ../../rest_api/README.md
+.. mdinclude:: ../../indra_db_service/README.md
 

indra_db/cli/content.py

@@ -1,3 +1,4 @@
+import ftplib
 import re
 import csv
 import tarfile
@@ -1142,7 +1143,13 @@ def iter_xmls(self, archives=None, continuing=False, pmcid_set=None):
 
         # Yield the contents from each archive.
         for archive in sorted(archives):
-            archive_path = self.download_archive(archive, continuing)
+            try:
+                archive_path = self.download_archive(archive, continuing)
+            except ftplib.Error as e:
+                logger.warning(f"Failed to download {archive}: {e}")
+                logger.info(f"Skipping {archive}...")
+                continue
+
             num_yielded = 0
             with tarfile.open(archive_path, mode='r:gz') as tar:
 

indra_db/client/readonly/query.py

@@ -503,14 +503,17 @@ class must be provided.
                     json_content_al.c.raw_json, json_content_al.c.pa_json]
 
         # Join up with other tables to pull metadata.
-        stmts_q = (stmts_q
-                   .outerjoin(ro.ReadingRefLink,
-                              ro.ReadingRefLink.rid == json_content_al.c.rid))
+        if ev_limit != 0:
+            stmts_q = (stmts_q
+                       .outerjoin(ro.ReadingRefLink,
+                                  ro.ReadingRefLink.rid == json_content_al.c.rid))
 
-        ref_link_keys = [k for k in ro.ReadingRefLink.__dict__.keys()
-                         if not k.startswith('_')]
+            ref_link_keys = [k for k in ro.ReadingRefLink.__dict__.keys()
+                             if not k.startswith('_')]
 
-        cols += [getattr(ro.ReadingRefLink, k) for k in ref_link_keys]
+            cols += [getattr(ro.ReadingRefLink, k) for k in ref_link_keys]
+        else:
+            ref_link_keys = None
 
         # Put it all together.
         selection = select(cols).select_from(stmts_q)
@@ -546,7 +549,8 @@ class must be provided.
             belief = next(row_gen)
             raw_json_bts = next(row_gen)
             pa_json_bts = next(row_gen)
-            ref_dict = dict(zip(ref_link_keys, row_gen))
+            if ref_link_keys is not None:
+                ref_dict = dict(zip(ref_link_keys, row_gen))
 
             if pa_json_bts is None:
                 logger.warning("Row returned without pa_json. This likely "

rest_api/api.py → indra_db_service/api.py

@@ -3,7 +3,6 @@
 import logging
 from os import path
 from datetime import datetime
-from argparse import ArgumentParser
 from collections import defaultdict
 
 from flask_cors import CORS
@@ -24,13 +23,12 @@
 from indralab_auth_tools.auth import auth, resolve_auth, config_auth
 from indralab_auth_tools.log import note_in_log, set_log_service_name, \
     user_log_endpoint
-from rest_api.call_handlers import pop_request_bool
-from rest_api.errors import HttpUserError, ResultTypeError, InvalidCredentials, \
-    InsufficientPermission
+from indra_db_service.errors import HttpUserError, ResultTypeError,\
+    InvalidCredentials, InsufficientPermission
 
-from rest_api.config import *
-from rest_api.call_handlers import *
-from rest_api.util import sec_since, get_s3_client, gilda_ground, \
+from indra_db_service.config import *
+from indra_db_service.call_handlers import *
+from indra_db_service.util import sec_since, get_s3_client, gilda_ground, \
     _make_english_from_meta, get_html_source_info
 
 
@@ -43,7 +41,45 @@
 logger.setLevel(logging.INFO)
 
 # Set the name of this service for the usage logs.
-set_log_service_name(f"db-rest-api-{DEPLOYMENT if DEPLOYMENT else 'stable'}")
+if not TESTING['status']:
+    set_log_service_name(f"db-rest-api-{DEPLOYMENT if DEPLOYMENT else 'stable'}")
+else:
+    from functools import wraps
+    from werkzeug.exceptions import HTTPException
+    logger.warning("TESTING: No logging will be performed.")
+
+    def note_in_log(*args, **kwargs):
+        logger.info(f"Faux noting in the log: {args}, {kwargs}")
+
+    def end_log(status):
+        logger.info(f"Faux ending log with status {status}")
+
+    def user_log_endpoint(func):
+
+        @wraps(func)
+        def run_logged(*args, **kwargs):
+            logger.info(f"Faux running logged: {args}, {kwargs}")
+            try:
+                resp = func(*args, **kwargs)
+                if isinstance(resp, str):
+                    status = 200
+                elif isinstance(resp, tuple) and isinstance(resp[1], int):
+                    status = resp[1]
+                else:
+                    status = resp.status_code
+            except HTTPException as e:
+                end_log(e.code)
+                raise e
+            except Exception as e:
+                logger.warning("Request experienced internal error. "
+                               "Returning 500.")
+                logger.exception(e)
+                end_log(500)
+                raise
+            end_log(status)
+            return resp
+
+        return run_logged
 
 
 # Define a custom flask class to handle the deployment name prefix.
@@ -342,8 +378,7 @@ def expand_meta_row():
                 continue
 
         # Add english...
-        eng = _make_english_from_meta(entry['agents'],
-                                      entry.get('type'))
+        eng = _make_english_from_meta(entry)
         if not eng:
             logger.warning(f"English not formed for {key}:\n"
                            f"{entry}")
@@ -508,14 +543,3 @@ def list_curations(stmt_hash, src_hash):
 def handle_user_error(error):
     logger.error(f"Got user error ({error.err_code}): {error.msg}")
     return error.response()
-
-
-def main():
-    parser = ArgumentParser()
-    parser.add_argument('-p', '--port', default=5000)
-    args = parser.parse_args()
-    app.run(port=args.port)
-
-
-if __name__ == '__main__':
-    main()

rest_api/call_handlers.py → indra_db_service/call_handlers.py

@@ -1,6 +1,6 @@
 __all__ = ['ApiCall', 'FromAgentsApiCall', 'FromHashApiCall',
            'FromHashesApiCall', 'FromPapersApiCall', 'FromSimpleJsonApiCall',
-           'FromAgentJsonApiCall', 'DirectQueryApiCall']
+           'FromAgentJsonApiCall', 'DirectQueryApiCall', 'pop_request_bool']
 
 import sys
 import json
@@ -20,14 +20,13 @@
 from indra_db.client.readonly import *
 from indra_db.client.principal.curation import *
 from indralab_auth_tools.log import note_in_log, is_log_running
-from indralab_auth_tools.src.models import UserDatabaseError
 
-from rest_api.config import MAX_STMTS, REDACT_MESSAGE, TITLE, TESTING, \
+from indra_db_service.config import MAX_STMTS, REDACT_MESSAGE, TITLE, TESTING, \
     jwt_nontest_optional, MAX_LIST_LEN
-from rest_api.errors import HttpUserError, ResultTypeError
-from rest_api.util import LogTracker, sec_since, get_source, process_agent, \
-    process_mesh_term, DbAPIError, iter_free_agents, _make_english_from_meta, \
-    get_html_source_info
+from indra_db_service.errors import HttpUserError, ResultTypeError
+from indra_db_service.util import LogTracker, sec_since, get_source,\
+    process_agent,  process_mesh_term, DbAPIError, iter_free_agents, \
+    _make_english_from_meta, get_html_source_info
 
 logger = logging.getLogger('call_handlers')
 
@@ -261,8 +260,7 @@ def process_entries(self, result):
                         eng = _format_stmt_text(stmt)
                         entry['evidence'] = _format_evidence_text(stmt)
                     else:
-                        eng = _make_english_from_meta(entry['agents'],
-                                                      entry.get('type'))
+                        eng = _make_english_from_meta(entry)
                     if not eng:
                         logger.warning(f"English not formed for {key}:\n"
                                        f"{entry}")
@@ -637,17 +635,20 @@ def _build_db_query(self):
 class FromPapersApiCall(StatementApiCall):
     def _build_db_query(self):
         # Get the paper id.
-        if 'paper_ids' not in self.web_query:
+        if 'paper_ids' not in self.web_query or not self.web_query['paper_ids']:
+            if not request.json:
+                abort(Response('No paper IDs given!', 400))
             ids = request.json.get('ids')
             if not ids:
                 logger.error("No ids provided!")
                 return abort(Response("No ids in request!", 400))
             self.web_query['paper_ids'] = ids
 
         # Extract mesh IDs.
-        if 'mesh_ids' not in self.web_query:
-            mesh_ids = request.json.get('mesh_ids', [])
-            self.web_query['mesh_ids'] = mesh_ids
+        if 'mesh_ids' not in self.web_query or not self.web_query['mesh_ids']:
+            if request.json:
+                mesh_ids = request.json.get('mesh_ids', [])
+                self.web_query['mesh_ids'] = mesh_ids
         return self._db_query_from_web_query()
 
 

rest_api/cli/zappa_tools.py → indra_db_service/cli/zappa_tools.py

@@ -61,13 +61,3 @@ def fix_permissions(deployment) -> None:
                                      SourceArn=source_arn,
                                      StatementId=statement_id)
     return
-
-
-def main():
-    import sys
-    deployment = sys.argv[1]
-    fix_permissions(deployment)
-
-
-if __name__ == '__main__':
-    main()

rest_api/config.py → indra_db_service/config.py

@@ -22,4 +22,4 @@ def jwt_nontest_optional(func):
     if TESTING['status']:
         return func
     else:
-        return jwt_optional(func)
+        return jwt_optional(func)

rest_api/data-vis/src/components/AmountView/AmountView.vue → indra_db_service/data-vis/src/components/AmountView/AmountView.vue

@@ -25,7 +25,7 @@
 
 <script>
   import LineChart from './LineChart'
-  import Multiselect from 'vue-multiselect'
+  import Multiselect from '@vueform/multiselect'
 
   export default {
     name: "AmountView",
@@ -148,7 +148,7 @@
 <!-- <style src="vue-multiselect/dist/vue-multiselect.min.css"></style> -->
 
 <style scoped>
-  @import "~vue-multiselect/dist/vue-multiselect.min.css";
+  @import "~@vueform/multiselect/themes/default.css";
 
   .form {
     width: 50em;

rest_api/data-vis/src/components/AmountView/LineChart.vue → indra_db_service/data-vis/src/components/AmountView/LineChart.vue

@@ -16,7 +16,7 @@
 </template>
 
 <script>
-  import VueApexCharts from 'vue-apexcharts'
+  import VueApexCharts from 'vue3-apexcharts';
 
   export default {
     name: "LineChart",

indra_db_service/data-vis/src/index.js

@@ -0,0 +1,2 @@
+export {default as TimeView} from './components/TimeView'
+export {default as AmountView} from './components/AmountView'

indra_db_service/data-vis/src/main.js

@@ -0,0 +1,8 @@
+import { createApp } from 'vue'
+import App from './App.vue'
+import {TimeView, AmountView} from './index'
+
+const app = createApp(App)
+app.use(TimeView)
+app.use(AmountView)
+app.mount('#app')

rest_api/test_api.py → indra_db_service/test_api.py

@@ -20,12 +20,12 @@
     StatementQueryResult
 from indra_db.client import HasAgent, HasType, FromMeshIds
 
-from rest_api.util import get_source
-from rest_api.config import MAX_STMTS, REDACT_MESSAGE, TESTING
+from indra_db_service.util import get_source
+from indra_db_service.config import MAX_STMTS, REDACT_MESSAGE, TESTING
 
 logger = logging.getLogger('db_api_unit_tests')
 try:
-    from rest_api.api import app
+    from indra_db_service.api import app
 except Exception as e:
     logger.warning(f"Could not load app: {e}")
     app = None
@@ -57,14 +57,6 @@ def _check_stmt_agents(resp, agents):
 TEST_DEPLOYMENT = False
 
 
-def test_secret():
-    import os
-    print("SECRET:", os.environ)
-    print("The Secret: it is secret")
-    assert False, "Oh no!"
-    return
-
-
 class TestDbApi(unittest.TestCase):
     def setUp(self):
         if TEST_DEPLOYMENT:
@@ -847,31 +839,7 @@ def test_mesh_concept_ev_limit(self):
                          for ev in ev_list}
             assert len(ev_tuples) == len(ev_list), "Evidence is not unique."
             for ev in ev_list:
-                found_pmid = False
-                if 'pmid' in ev:
-                    pmids.add(ev['pmid'])
-                    found_pmid = True
-
-                if 'text_refs' in ev:
-                    tr_dict = ev['text_refs']
-                    if 'TRID' in tr_dict:
-                        tr = db.select_one(db.TextRef,
-                                           db.TextRef.id == tr_dict['TRID'])
-                        pmids.add(tr.pmid)
-                        found_pmid = True
-                    if 'PMID' in tr_dict:
-                        pmids.add(tr_dict['PMID'])
-                        found_pmid = True
-                    if 'DOI' in tr_dict:
-                        tr_list = db.select_all(
-                            db.TextRef,
-                            db.TextRef.doi_in([tr_dict['DOI']])
-                        )
-                        pmids |= {tr.pmid for tr in tr_list if tr.pmid}
-                        found_pmid = True
-
-                assert found_pmid,\
-                    "How could this have been mapped to mesh?"
+                pmids.add(ev['pmid'])
         pmids = {int(pmid) for pmid in pmids if pmid is not None}
 
         mesh_pmids = {n for n, in db.select_all(
@@ -887,7 +855,14 @@ def test_mesh_concept_ev_limit(self):
             db.MtiRefAnnotationsTest.is_concept.is_(True)
         )}
 
-        assert pmids == mesh_pmids, "Not all pmids mapped ot mesh term."
+        assert pmids == mesh_pmids, "Not all pmids mapped to mesh term."
+
+    def test_EPHA6_expand_failure(self):
+        self.__simple_time_test('post', 'expand', 'with_cur_counts=true',
+                                agent_json={'0': 'EPHA6'},
+                                hashes=['-22724684590296184',
+                                        '-5536416870993077',
+                                        '-31952004721698747'])
 
 
 class WebApp:

rest_api/util.py → indra_db_service/util.py

@@ -144,7 +144,9 @@ def iter_free_agents(query_dict):
             yield entry
 
 
-def _make_english_from_meta(agent_json, stmt_type=None):
+def _make_english_from_meta(interaction):
+    stmt_type = interaction.get('type')
+    agent_json = interaction['agents']
     if stmt_type is None:
         if len(agent_json) == 0:
             eng = ''
@@ -162,6 +164,5 @@ def _make_english_from_meta(agent_json, stmt_type=None):
             else:
                 eng += ' is modified'
     else:
-        eng = _format_stmt_text(stmt_from_interaction({'agents': agent_json,
-                                                       'type': stmt_type}))
+        eng = _format_stmt_text(stmt_from_interaction(interaction))
     return eng