Merge pull request #584 from monarch-initiative/issue-657-add-klat

Add knowledge level & agent type, plus constraint improvements

pyproject.toml

@@ -18,15 +18,14 @@ packages = [
 python = ">=3.10,<3.12"
 # biolink-model = "^4.2.0"
 # When 4.2.0 (or any release after 4.1.6) is released, we can remove the git dependency
-biolink-model = { git = "https://github.com/biolink/biolink-model", tag = "v4.2.0-rc.2" }
+biolink-model = { git = "https://github.com/biolink/biolink-model", branch = "master" }
 bmt = "^1.0.15"
 cat-merge = "0.2.1"
 closurizer = "0.5.1"
 kghub-downloader = "^0.3.2"
 kgx = { git = "https://github.com/biolink/kgx", branch = "master" } # ">=2.1"
 koza = ">=0.5.2"
-linkml = "^1.6.3"
-linkml-runtime = "1.6.2"
+linkml = "1.6.3"
 linkml-solr = "0.1.5" # "^0.1.3"
 multi-indexer = "0.0.5"
 # Other Dependencies
@@ -38,6 +37,7 @@ sh = "^1.14.3"
 typer = "^0.7"
 typer-cli = "^0.0.13"
 yamllint = "^1.35.1"
+linkml-runtime = "1.6.3"
 
 [tool.poetry.group.dev]
 optional = true

scripts/load_solr.sh

@@ -16,8 +16,8 @@ echo "Download the schema from monarch-py"
 
 # retrieve the schema from the main branch on monarch-app
 
-curl -O https://raw.githubusercontent.com/monarch-initiative/monarch-app/main/backend/src/monarch_py/datamodels/model.yaml
-curl -O https://raw.githubusercontent.com/monarch-initiative/monarch-app/main/backend/src/monarch_py/datamodels/similarity.yaml
+curl -O https://raw.githubusercontent.com/monarch-initiative/monarch-app/issue-675-add-kl-at/backend/src/monarch_py/datamodels/model.yaml
+curl -O https://raw.githubusercontent.com/monarch-initiative/monarch-app/issue-675-add-kl-at/backend/src/monarch_py/datamodels/similarity.yaml
 
 echo "Starting the server"
 poetry run lsolr start-server

src/monarch_ingest/cli_utils.py

@@ -1,6 +1,7 @@
 import csv
 import gc
 import os
+import pkgutil
 import sys
 import tarfile
 import yaml
@@ -188,15 +189,28 @@ def transform_phenio(
                 "predicate",
                 "object",
                 "category",
-                "relation",
                 "primary_knowledge_source",
                 "aggregator_knowledge_source",
+                "knowledge_level",
+                "agent_type"
             ]
         ),
         axis=1,
         inplace=True,
     )
 
+    # if knowledge level doesn't exist, add it and assign to knowledge_assertion
+    if "knowledge_level" not in edges_df.columns:
+        edges_df["knowledge_level"] = "knowledge_assertion"
+    # same for agent_type, setting it to manual_agent
+    if "agent_type" not in edges_df.columns:
+        edges_df["agent_type"] = "manual_agent"
+
+    # prepend infores:monarchinitiative to the aggregator_knowledge_source column for edges that don't have it
+    edges_df["aggregator_knowledge_source"] = edges_df["aggregator_knowledge_source"].apply(
+        lambda x: f"infores:monarchinitiative|{x}" if not x.startswith("infores:monarchinitiative") else x
+    )
+
     edges_df = edges_df[edges_df["predicate"].str.contains(":")]
 
     # assign level association category if edge category is empty

src/monarch_ingest/ingests/alliance/gene.py

@@ -48,7 +48,7 @@
         id=gene_id,
         symbol=row["symbol"],
         name=row["symbol"],
-        full_name=row["name"],
+        full_name=row["name"].replace("\r",""), # Replacement to remove stray carriage returns in XenBase files
         # No place in the schema for gene type (SO term) right now
         # type=row["soTermId"],
         in_taxon=[in_taxon],
@@ -62,6 +62,7 @@
             for xref in row["basicGeneticEntity"]["crossReferences"]
         ]
     if "synonyms" in row["basicGeneticEntity"].keys():
-        gene.synonym = row["basicGeneticEntity"]["synonyms"]
+        # more handling for errant carriage returns
+        gene.synonym = [synonym.replace("\r","") for synonym in row["basicGeneticEntity"]["synonyms"] ]
 
     koza_app.write(gene)

src/monarch_ingest/ingests/alliance/gene_to_expression.py

@@ -4,7 +4,7 @@
 from koza.cli_runner import get_koza_app
 from source_translation import source_map
 
-from biolink_model.datamodel.pydanticmodel_v2 import GeneToExpressionSiteAssociation
+from biolink_model.datamodel.pydanticmodel_v2 import GeneToExpressionSiteAssociation, KnowledgeLevelEnum, AgentTypeEnum
 
 from monarch_ingest.ingests.alliance.utils import get_data
 
@@ -54,7 +54,9 @@
                     qualifiers=([get_data(row, "assay")] if get_data(row, "assay") else None),
                     publications=publication_ids,
                     aggregator_knowledge_source=["infores:monarchinitiative", "infores:alliancegenome"],
-                    primary_knowledge_source=source
+                    primary_knowledge_source=source,
+                    knowledge_level=KnowledgeLevelEnum.knowledge_assertion,
+                    agent_type=AgentTypeEnum.manual_agent
                 )
             )
 
@@ -71,7 +73,9 @@
                     qualifiers=([get_data(row, "assay")] if get_data(row, "assay") else None),
                     publications=publication_ids,
                     aggregator_knowledge_source=["infores:monarchinitiative", "infores:alliancegenome"],
-                    primary_knowledge_source=source
+                    primary_knowledge_source=source,
+                    knowledge_level=KnowledgeLevelEnum.knowledge_assertion,
+                    agent_type=AgentTypeEnum.manual_agent
                 )
             )
         else:

src/monarch_ingest/ingests/alliance/gene_to_expression.yaml

@@ -50,5 +50,7 @@ edge_properties:
   - 'publications'
   - 'aggregator_knowledge_source'
   - 'primary_knowledge_source'
+  - 'knowledge_level'
+  - 'agent_type'
 
 transform_mode: 'flat'

src/monarch_ingest/ingests/alliance/gene_to_phenotype.py

@@ -5,7 +5,7 @@
 from koza.cli_runner import get_koza_app
 from source_translation import source_map
 
-from biolink_model.datamodel.pydanticmodel_v2 import GeneToPhenotypicFeatureAssociation
+from biolink_model.datamodel.pydanticmodel_v2 import GeneToPhenotypicFeatureAssociation, KnowledgeLevelEnum, AgentTypeEnum
 
 
 from loguru import logger
@@ -41,7 +41,10 @@
             object=phenotypic_feature_id,
             publications=[row["evidence"]["publicationId"]],
             aggregator_knowledge_source=["infores:monarchinitiative", "infores:alliancegenome"],
-            primary_knowledge_source=source
+            primary_knowledge_source=source,
+            knowledge_level = KnowledgeLevelEnum.knowledge_assertion,
+            agent_type = AgentTypeEnum.manual_agent
+
         )
 
         if "conditionRelations" in row.keys() and row["conditionRelations"] is not None:

src/monarch_ingest/ingests/alliance/gene_to_phenotype.yaml

@@ -35,5 +35,7 @@ edge_properties:
   - 'publications'
   - 'aggregator_knowledge_source'
   - 'primary_knowledge_source'
+  - 'knowledge_level'
+  - 'agent_type'
 
 transform_mode: 'flat'

src/monarch_ingest/ingests/bgee/gene_to_expression.yaml

@@ -75,5 +75,7 @@ edge_properties:
   - 'object'
   - 'primary_knowledge_source'
   - 'aggregator_knowledge_source'
+  - 'knowledge_level'
+  - 'agent_type'
 
 transform_mode: 'flat'

src/monarch_ingest/ingests/bgee/gene_to_expression_utils.py

@@ -2,7 +2,7 @@
 import pandas as pd
 from typing import Dict, List, Union
 from koza.app import KozaApp
-from biolink_model.datamodel.pydanticmodel_v2 import GeneToExpressionSiteAssociation
+from biolink_model.datamodel.pydanticmodel_v2 import GeneToExpressionSiteAssociation, KnowledgeLevelEnum, AgentTypeEnum
 
 
 def filter_group_by_rank(rows: List, col: str, largest_n: int = 0, smallest_n: int = 0) -> List[Dict]:
@@ -42,7 +42,9 @@ def write_group(rows: List, koza_app: KozaApp):
             predicate='biolink:expressed_in',
             object=row['Anatomical entity ID'],
             primary_knowledge_source="infores:bgee",
-            aggregator_knowledge_source=["infores:monarchinitiative"])
+            aggregator_knowledge_source=["infores:monarchinitiative"],
+            knowledge_level=KnowledgeLevelEnum.knowledge_assertion,
+            agent_type=AgentTypeEnum.not_provided)
 
         koza_app.write(association)
 

src/monarch_ingest/ingests/biogrid/biogrid.py

@@ -1,6 +1,6 @@
 import uuid
 from koza.cli_runner import get_koza_app
-from biolink_model.datamodel.pydanticmodel_v2 import PairwiseGeneToGeneInteraction
+from biolink_model.datamodel.pydanticmodel_v2 import PairwiseGeneToGeneInteraction, KnowledgeLevelEnum, AgentTypeEnum
 from biogrid_util import get_gene_id, get_evidence, get_publication_ids
 
 koza_app = get_koza_app("biogrid")
@@ -25,7 +25,9 @@
             has_evidence=evidence,
             publications=publications,
             primary_knowledge_source="infores:biogrid",
-            aggregator_knowledge_source=["infores:monarchinitiative"]
+            aggregator_knowledge_source=["infores:monarchinitiative"],
+            knowledge_level=KnowledgeLevelEnum.knowledge_assertion,
+            agent_type=AgentTypeEnum.not_provided
         )
 
         koza_app.write(association)

src/monarch_ingest/ingests/biogrid/biogrid.yaml

@@ -40,5 +40,7 @@ edge_properties:
   - 'publications'
   - 'primary_knowledge_source'
   - 'aggregator_knowledge_source'
+  - 'knowledge_level'
+  - 'agent_type'
 
 transform_mode: 'flat'

src/monarch_ingest/ingests/biogrid/biogrid_util.py

@@ -55,7 +55,7 @@ def get_evidence(methods: str) -> Optional[List[str]]:
             method = method.rstrip(")").split('(')[-1]
             if method not in EVIDENCE_CODE_MAPPINGS.keys():
                 err_msg = f"Unknown interaction detection method '{method}'. " +\
-                          "Assigning default code ECO:0000006 == 'experimental evidence'."
+                          "Assigning default code ECO:0000006 == 'experimental evidence', the ECO root."
                 logger.warning(err_msg)
                 EVIDENCE_CODE_MAPPINGS[method] = "ECO:0000006"
 

src/monarch_ingest/ingests/ctd/chemical_to_disease.py

@@ -2,7 +2,8 @@
 
 from koza.cli_runner import get_koza_app
 
-from biolink_model.datamodel.pydanticmodel_v2 import ChemicalToDiseaseOrPhenotypicFeatureAssociation
+from biolink_model.datamodel.pydanticmodel_v2 import ChemicalToDiseaseOrPhenotypicFeatureAssociation, \
+    KnowledgeLevelEnum, AgentTypeEnum
 from monarch_ingest.constants import BIOLINK_TREATS_OR_APPLIED_OR_STUDIED_TO_TREAT
 
 koza_app = get_koza_app("ctd_chemical_to_disease")
@@ -25,7 +26,9 @@
             object=disease_id,
             publications=["PMID:" + p for p in row['PubMedIDs'].split("|")],
             aggregator_knowledge_source=["infores:monarchinitiative"],
-            primary_knowledge_source="infores:ctd"
+            primary_knowledge_source="infores:ctd",
+            knowledge_level=KnowledgeLevelEnum.knowledge_assertion,
+            agent_type=AgentTypeEnum.manual_agent
         )
 
         koza_app.write(association)

src/monarch_ingest/ingests/ctd/chemical_to_disease.yaml

@@ -35,3 +35,5 @@ edge_properties:
   - 'publications'
   - 'aggregator_knowledge_source'
   - 'primary_knowledge_source'
+  - 'knowledge_level'
+  - 'agent_type'

src/monarch_ingest/ingests/dictybase/gene_to_phenotype.py

@@ -4,7 +4,8 @@
 from koza.cli_runner import get_koza_app
 from monarch_ingest.ingests.dictybase.utils import parse_gene_id, parse_phenotypes
 
-from biolink_model.datamodel.pydanticmodel_v2 import GeneToPhenotypicFeatureAssociation
+from biolink_model.datamodel.pydanticmodel_v2 import GeneToPhenotypicFeatureAssociation, KnowledgeLevelEnum, \
+    AgentTypeEnum
 
 koza_app = get_koza_app("dictybase_gene_to_phenotype")
 
@@ -29,7 +30,9 @@
                 predicate='biolink:has_phenotype',
                 object=phenotype_id,
                 aggregator_knowledge_source=["infores:monarchinitiative"],
-                primary_knowledge_source="infores:dictybase"
+                primary_knowledge_source="infores:dictybase",
+                knowledge_level=KnowledgeLevelEnum.knowledge_assertion,
+                agent_type=AgentTypeEnum.manual_agent
             )
 
             koza_app.write(association)

src/monarch_ingest/ingests/dictybase/gene_to_phenotype.yaml

@@ -31,5 +31,7 @@ edge_properties:
   - 'object'
   - 'aggregator_knowledge_source'
   - 'primary_knowledge_source'
+  - 'knowledge_level'
+  - 'agent_type'
 
 transform_mode: 'flat'

src/monarch_ingest/ingests/flybase/publication_to_gene.py

@@ -2,7 +2,8 @@
 
 from koza.cli_runner import get_koza_app
 
-from biolink_model.datamodel.pydanticmodel_v2 import InformationContentEntityToNamedThingAssociation
+from biolink_model.datamodel.pydanticmodel_v2 import InformationContentEntityToNamedThingAssociation, AgentTypeEnum, \
+    KnowledgeLevelEnum
 
 koza_app = get_koza_app("flybase_publication_to_gene")
 
@@ -24,6 +25,8 @@
         object=publication_id,
         aggregator_knowledge_source=["infores:monarchinitiative"],
         primary_knowledge_source="infores:flybase",
+        knowledge_level=KnowledgeLevelEnum.knowledge_assertion,
+        agent_type=AgentTypeEnum.manual_agent
     )
 
     koza_app.write(association)

src/monarch_ingest/ingests/flybase/publication_to_gene.yaml

@@ -27,5 +27,7 @@ edge_properties:
   - 'object'
   - 'aggregator_knowledge_source'
   - 'primary_knowledge_source'
+  - 'knowledge_level'
+  - 'agent_type'
 
 transform_mode: 'flat'

src/monarch_ingest/ingests/go/annotation.py

@@ -6,6 +6,7 @@
 """
 import uuid
 
+from biolink_model.datamodel.pydanticmodel_v2 import KnowledgeLevelEnum, AgentTypeEnum
 from koza.cli_runner import get_koza_app
 
 from monarch_ingest.ingests.go.annotation_utils import (
@@ -119,7 +120,9 @@
                 has_evidence=[eco_term],
                 # subject_context_qualifier=ncbitaxa,  # Biolink Pydantic model support missing for this slot
                 aggregator_knowledge_source=["infores:monarchinitiative"],
-                primary_knowledge_source=assigned_by
+                primary_knowledge_source=assigned_by,
+                knowledge_level=KnowledgeLevelEnum.knowledge_assertion,
+                agent_type=AgentTypeEnum.manual_agent
             )
 
             # Write the captured Association out

src/monarch_ingest/ingests/go/annotation.yaml

@@ -85,5 +85,7 @@ edge_properties:
   - 'object'
   - 'aggregator_knowledge_source'
   - 'primary_knowledge_source'
+  - 'knowledge_level'
+  - 'agent_type'
 
 transform_mode: 'flat'

src/monarch_ingest/ingests/hpoa/disease_mode_of_inheritance.py

@@ -25,8 +25,8 @@
 
 from koza.cli_runner import get_koza_app
 
-from biolink_model.datamodel.pydanticmodel_v2 import DiseaseOrPhenotypicFeatureToGeneticInheritanceAssociation
-
+from biolink_model.datamodel.pydanticmodel_v2 import DiseaseOrPhenotypicFeatureToGeneticInheritanceAssociation, \
+    KnowledgeLevelEnum, AgentTypeEnum
 
 from loguru import logger
 
@@ -71,7 +71,9 @@
           publications=publications,
           has_evidence=[evidence_curie],
           aggregator_knowledge_source=["infores:monarchinitiative"],
-          primary_knowledge_source="infores:hpo-annotations"
+          primary_knowledge_source="infores:hpo-annotations",
+          knowledge_level=KnowledgeLevelEnum.knowledge_assertion,
+          agent_type=AgentTypeEnum.manual_agent
       )
       koza_app.write(association)
 

src/monarch_ingest/ingests/hpoa/disease_mode_of_inheritance.yaml

@@ -56,3 +56,5 @@ edge_properties:
   - 'has_evidence'
   - 'aggregator_knowledge_source'
   - 'primary_knowledge_source'
+  - 'knowledge_level'
+  - 'agent_type'

src/monarch_ingest/ingests/hpoa/disease_to_phenotype.py

@@ -29,7 +29,8 @@
 
 from koza.cli_runner import get_koza_app
 
-from biolink_model.datamodel.pydanticmodel_v2 import DiseaseToPhenotypicFeatureAssociation
+from biolink_model.datamodel.pydanticmodel_v2 import DiseaseToPhenotypicFeatureAssociation, KnowledgeLevelEnum, \
+    AgentTypeEnum
 from monarch_ingest.ingests.hpoa.hpoa_utils import phenotype_frequency_to_hpo_term, FrequencyHpoTerm, Frequency
 
 from loguru import logger
@@ -96,6 +97,9 @@
         has_count=frequency.has_count,
         has_total=frequency.has_total,
         aggregator_knowledge_source=["infores:monarchinitiative"],
-        primary_knowledge_source="infores:hpo-annotations"
+        primary_knowledge_source="infores:hpo-annotations",
+        knowledge_level=KnowledgeLevelEnum.knowledge_assertion,
+        agent_type=AgentTypeEnum.manual_agent
+
     )
     koza_app.write(association)

src/monarch_ingest/ingests/hpoa/disease_to_phenotype.yaml

@@ -63,3 +63,5 @@ edge_properties:
   - 'has_evidence'
   - 'aggregator_knowledge_source'
   - 'primary_knowledge_source'
+  - 'knowledge_level'
+  - 'agent_type'