Merge pull request #208 from dianakolusheva/disease_map

Disease map

.github/workflows/tests.yml

@@ -36,7 +36,7 @@ jobs:
       run: |
         sudo apt-get install libpq-dev
         psql --no-password -h localhost -c 'create database emmaadb_test;' -U postgres
-        wget https://files.pythonhosted.org/packages/91/a1/55b8224cbc9986bbad4b8f0f2dd11892845156b759b3495202d457ca0b73/kappy-4.0.94-cp37-cp37m-manylinux2010_x86_64.whl
+        wget -nv https://files.pythonhosted.org/packages/91/a1/55b8224cbc9986bbad4b8f0f2dd11892845156b759b3495202d457ca0b73/kappy-4.0.94-cp37-cp37m-manylinux2010_x86_64.whl
         mv kappy-4.0.94-cp37-cp37m-manylinux2010_x86_64.whl kappy-4.0.94-cp37-cp37m-manylinux2010_x86_64.zip
         unzip kappy-4.0.94-cp37-cp37m-manylinux2010_x86_64.zip
         sudo apt-get install graphviz libgraphviz-dev pkg-config
@@ -54,8 +54,9 @@ jobs:
         cd ../indralab_web_templates
         pip install .
         cd ../..
-        pip install boto3 jsonpickle pygraphviz fnvhash sqlalchemy inflection pybel==0.15 flask_jwt_extended==3.25.0 gilda tweepy nose coverage moto[iam] sqlalchemy_utils termcolor flask-cors
+        pip install boto3 jsonpickle pygraphviz fnvhash inflection pybel==0.15 flask_jwt_extended==3.25.0 gilda tweepy nose coverage moto[iam] sqlalchemy_utils termcolor flask-cors
         pip install --no-dependencies .
+        pip install -U sqlalchemy==1.3.23
         wget "https://github.com/RuleWorld/bionetgen/releases/download/BioNetGen-2.4.0/BioNetGen-2.4.0-Linux.tgz" -O bionetgen.tar.gz -nv
         tar xzf bionetgen.tar.gz
       env:

emmaa/model.py

@@ -12,6 +12,7 @@
 from indra.statements import stmts_from_json
 from indra.pipeline import AssemblyPipeline, register_pipeline
 from indra.tools.assemble_corpus import filter_grounded_only
+from indra.sources.minerva import process_from_web
 from indra_db.client.principal.curation import get_curations
 from indra_db.util import get_db, _get_trids
 from emmaa.priors import SearchTerm
@@ -245,33 +246,50 @@ def get_new_readings(self, date_limit=10):
             lit_sources = [lit_sources]
         if isinstance(readers, str):
             readers = [readers]
-        estmts = []
-        for lit_source, reader in zip(lit_sources, readers):
-            ids_to_terms = self.search_literature(lit_source, date_limit)
-            if reader == 'aws':
-                new_estmts = read_pmid_search_terms(
-                    ids_to_terms)
-                self.add_paper_ids(ids_to_terms.keys(), 'pmid')
-            elif reader == 'indra_db_pmid':
-                new_estmts = read_db_pmid_search_terms(
-                    ids_to_terms)
-                self.add_paper_ids(ids_to_terms.keys(), 'pmid')
-            elif reader == 'indra_db_doi':
-                new_estmts = read_db_doi_search_terms(
-                    ids_to_terms)
-                self.add_paper_ids(ids_to_terms.keys(), 'doi')
-            elif reader == 'elsevier_eidos':
-                new_estmts = read_elsevier_eidos_search_terms(
-                    ids_to_terms)
-                self.add_paper_ids(ids_to_terms.keys(), 'pii')
-            else:
-                raise ValueError('Unknown reader: %s' % reader)
-            estmts += new_estmts
-        logger.info('Got a total of %d new EMMAA Statements from reading' %
-                    len(estmts))
-        self.extend_unique(estmts)
+        # First get statements from literature if needed
+        # Some models are not updated from literature
+        if lit_sources is not None and readers is not None:
+            estmts = []
+            for lit_source, reader in zip(lit_sources, readers):
+                ids_to_terms = self.search_literature(lit_source, date_limit)
+                if reader == 'aws':
+                    new_estmts = read_pmid_search_terms(
+                        ids_to_terms)
+                    self.add_paper_ids(ids_to_terms.keys(), 'pmid')
+                elif reader == 'indra_db_pmid':
+                    new_estmts = read_db_pmid_search_terms(
+                        ids_to_terms)
+                    self.add_paper_ids(ids_to_terms.keys(), 'pmid')
+                elif reader == 'indra_db_doi':
+                    new_estmts = read_db_doi_search_terms(
+                        ids_to_terms)
+                    self.add_paper_ids(ids_to_terms.keys(), 'doi')
+                elif reader == 'elsevier_eidos':
+                    new_estmts = read_elsevier_eidos_search_terms(
+                        ids_to_terms)
+                    self.add_paper_ids(ids_to_terms.keys(), 'pii')
+                else:
+                    raise ValueError('Unknown reader: %s' % reader)
+                estmts += new_estmts
+            logger.info('Got a total of %d new EMMAA Statements from reading' %
+                        len(estmts))
+            self.extend_unique(estmts)
+        # Then update from other sources
         if self.reading_config.get('cord19_update'):
+            # This overwrites existing statements
             self.update_with_cord19()
+        # The following methods can extend existing statements
+        estmts = []
+        if self.reading_config.get('disease_map'):
+            new_estmts = self.update_from_disease_map(
+                self.reading_config['disease_map'])
+            estmts += new_estmts
+        if self.reading_config.get('other_files'):
+            new_estmts = self.update_from_files(
+                self.reading_config['other_files'])
+            estmts += new_estmts
+        if estmts:
+            self.extend_unique(estmts)
         self.eliminate_copies()
 
     def extend_unique(self, estmts):
@@ -309,6 +327,29 @@ def update_with_cord19(self):
         self.stmts = to_emmaa_stmts(new_stmts, datetime.datetime.now(), [])
         self.add_paper_ids(paper_ids, 'TRID')
 
+    def update_from_disease_map(self, disease_map_config):
+        """Update model by processing MINERVA Disease Map."""
+        filenames = disease_map_config['filenames']
+        map_name = disease_map_config['map_name']
+        logger.info('Loading Statements from %s Disease Map' % map_name)
+        sp = process_from_web(filenames=filenames, map_name=map_name)
+        new_estmts = to_emmaa_stmts(sp.statements, datetime.datetime.now(), [])
+        logger.info('Got %d EMMAA Statements from %s Disease Map' %
+                    (len(new_estmts), map_name))
+        return new_estmts
+
+    def update_from_files(self, files_config):
+        """Add custom statements from files."""
+        bucket = files_config['bucket']
+        filenames = files_config['filenames']
+        stmts = []
+        for fname in filenames:
+            file_stmts = load_pickle_from_s3(bucket, fname)
+            logger.info(f'Loaded {len(file_stmts)} statements from {fname}.')
+            stmts += file_stmts
+        new_estmts = to_emmaa_stmts(stmts, datetime.datetime.now(), [])
+        return new_estmts
+
     def add_paper_ids(self, initial_ids, id_type='pmid'):
         """Convert if needed and save paper IDs.
 

setup.py

@@ -35,9 +35,9 @@
         ],
       packages=find_packages(),
       install_requires=['indra', 'boto3', 'jsonpickle', 'kappy==4.0.94',
-                        'pygraphviz', 'fnvhash', 'sqlalchemy', 'inflection',
-                        'pybel==0.15', 'flask_jwt_extended==3.25.0', 'gilda',
-                        'tweepy'],
+                        'pygraphviz', 'fnvhash', 'sqlalchemy<1.4',
+                        'inflection', 'pybel==0.15',
+                        'flask_jwt_extended==3.25.0', 'gilda', 'tweepy'],
       extras_require={'test': ['nose', 'coverage', 'moto[iam]',
                                'sqlalchemy_utils']}
       )