Merge pull request #116 from indralab/mti

Add MTI reader to DB workflow
gyorilab · Jun 28, 2020 · bd6bbe1 · bd6bbe1
2 parents 8ced690 + bd9f932
commit bd6bbe1
Show file tree

Hide file tree

Showing 5 changed files with 15 additions and 7 deletions.
diff --git a/README.md b/README.md
@@ -26,6 +26,8 @@ We have read all available content, and every day we run the following readers:
 we read all new content with the following readers:
 - [Eidos](https://github.com/clulab/eidos)
 - [ISI](https://github.com/sgarg87/big_mech_isi_gg)
+- [MTI](https://ii.nlm.nih.gov/MTI/index.shtml) - used specifically to tag
+content with topic terms.
 
 we read a limited subset of new content with the following readers:
 - [TRIPS](http://trips.ihmc.us/parser/cgi/drum)

diff --git a/indra_db/databases.py b/indra_db/databases.py
@@ -134,7 +134,7 @@ class formats(_map_class):
     EKB = 'ekb'
 
 
-readers = {'REACH': 1, 'SPARSER': 2, 'TRIPS': 3, 'ISI': 4, 'EIDOS': 5}
+readers = {'REACH': 1, 'SPARSER': 2, 'TRIPS': 3, 'ISI': 4, 'EIDOS': 5, 'MTI': 6}
 
 
 # Specify versions of readers, and preference. Later in the list is better.
@@ -145,6 +145,7 @@ class formats(_map_class):
     'trips': ['STATIC', '2019Nov14'],
     'isi': ['20180503'],
     'eidos': ['0.2.3-SNAPSHOT'],
+    'mti': ['1.0'],
 }
 
 

diff --git a/indra_db/managers/reading_manager.py b/indra_db/managers/reading_manager.py
@@ -83,8 +83,9 @@ def _get_latest_updatetime(self, db, reader_name):
             db.ReadingUpdates.reader == reader_name
             )
         if not len(update_list):
-            logger.warning("The database has not had an initial upload, or "
-                           "else the updates table has not been populated.")
+            logger.warning("The database has not had an initial upload "
+                           "for %s, or else the updates table has not "
+                           "been populated." % reader_name)
             return None
 
         return max([u.latest_datetime for u in update_list])
@@ -144,8 +145,8 @@ def read_new(self, db, reader_name):
         if latest_updatetime is not None:
             self.begin_datetime = latest_updatetime - self.buffer
         else:
-            raise ReadingUpdateError("There are no previous updates. "
-                                     "Please run_all.")
+            raise ReadingUpdateError("There are no previous updates for %s. "
+                                     "Please run_all." % reader_name)
 
         constraints = self._get_constraints(db, reader_name)
 
@@ -182,6 +183,7 @@ class BulkAwsReadingManager(BulkReadingManager):
         'isi': 2400,
         'trips': 300,
         'eidos': 1200,
+        'mti': 5400,
     }
 
     ids_per_job = {
@@ -190,6 +192,7 @@ class BulkAwsReadingManager(BulkReadingManager):
         'isi': 5000,
         'trips': 500,
         'eidos': 5000,
+        'mti': 1000,
     }
 
     def __init__(self, *args, **kwargs):
@@ -387,7 +390,7 @@ def main():
     else:
         db = get_db(args.database)
 
-    readers = ['SPARSER', 'REACH', 'TRIPS', 'ISI', 'EIDOS']
+    readers = ['SPARSER', 'REACH', 'TRIPS', 'ISI', 'EIDOS', 'MTI']
     if args.method == 'local':
         bulk_manager = BulkLocalReadingManager(readers,
                                                buffer_days=args.buffer,

diff --git a/indra_db/reading/submit_reading_pipeline.py b/indra_db/reading/submit_reading_pipeline.py
@@ -54,8 +54,10 @@ class DbReadingSubmitter(Submitter):
     _purpose = 'db_reading'
     _job_queue_dict = {'run_db_reading_queue': ['reach', 'sparser', 'isi',
                                                 'eidos'],
+                       'run_db_lite_queue': ['mti'],
                        'run_db_trips_queue': ['trips']}
     _job_def_dict = {'run_db_reading_jobdef': ['reach', 'sparser'],
+                     'run_db_lite_jobdef': ['mti'],
                      'run_db_reading_isi_jobdef': ['isi'],
                      'run_db_reading_trips_jobdef': ['trips'],
                      'run_db_reading_eidos_jobdef': ['eidos']}

diff --git a/indra_db/schemas/readonly_schema.py b/indra_db/schemas/readonly_schema.py
@@ -550,4 +550,4 @@ class MeshMeta(Base, ReadonlyTable):
                                'bel_lc', 'signor', 'biogrid', 'tas',
                                'lincs_drug', 'hprd', 'trrust'],
                  'reading': ['geneways', 'tees', 'isi', 'trips', 'rlimsp',
-                             'medscan', 'sparser', 'reach', 'eidos']}
+                             'medscan', 'sparser', 'reach', 'eidos', 'mti']}