Merge 9c3cae1 into c114108

CHANGELOG.md

@@ -25,6 +25,7 @@ This changelog was started for release 4.2.0.
 - Added 'custom distance' option for faldo relation (instead of just 'included_in' and 'overlap_with')
 - Store 'version' value when storing results. Not used for now, but might be used in deprecation warnings later
 - Removed some lines from coverage computation
+- Added 'indirect relations': Two entities will be linked on the graph, without a direct relation between them (same as faldo relations). This is intended to be used with 'linked attributes' (ex: get all entities B where B.color is the same as A.color, without a direct relation between A and B)
 
 ### Fixed
 
@@ -51,6 +52,7 @@ This changelog was started for release 4.2.0.
 - 'Same strand' queries will now match 'BothStrand' with a forward or reverse strand
 - Use '+', '-' and '.' for strand values in CSV instead of raw value (for homogenization)
 - Now allows 'infinite' recursive blocks (ie, a Minus block inside a Union block, or the opposite)
+- Overhaul faldo relation: add 'direct shortcut' triples between entities and values, to avoid using slow property paths when using faldo queries
 
 ### Security
 

askomics/api/file.py

@@ -410,7 +410,7 @@ def get_column_types():
         types: list of available column types
     """
 
-    data = ["numeric", "text", "category", "boolean", "date", "reference", "strand", "start", "end", "general_relation", "symetric_relation", "label"]
+    data = ["numeric", "text", "category", "boolean", "date", "reference", "strand", "start", "end", "general_relation", "symetric_relation", "indirect_relation", "label"]
 
     return jsonify({
         "types": data

askomics/libaskomics/BedFile.py

@@ -172,7 +172,7 @@ def generate_rdf_content(self):
             attribute = self.namespace_data[self.format_uri(feature.chrom)]
             faldo_reference = attribute
             self.faldo_abstraction["reference"] = relation
-            self.graph_chunk.add((entity, relation, attribute))
+            # self.graph_chunk.add((entity, relation, attribute))
 
             if "reference" not in attribute_list:
                 attribute_list.append("reference")
@@ -195,7 +195,7 @@ def generate_rdf_content(self):
             attribute = rdflib.Literal(self.convert_type(feature.start + 1))  # +1 because bed is 0 based
             faldo_start = attribute
             self.faldo_abstraction["start"] = relation
-            self.graph_chunk.add((entity, relation, attribute))
+            # self.graph_chunk.add((entity, relation, attribute))
 
             if "start" not in attribute_list:
                 attribute_list.append("start")
@@ -212,7 +212,7 @@ def generate_rdf_content(self):
             attribute = rdflib.Literal(self.convert_type(feature.end))
             faldo_end = attribute
             self.faldo_abstraction["end"] = relation
-            self.graph_chunk.add((entity, relation, attribute))
+            # self.graph_chunk.add((entity, relation, attribute))
 
             if "end" not in attribute_list:
                 attribute_list.append("end")
@@ -233,7 +233,7 @@ def generate_rdf_content(self):
                 attribute = self.namespace_data[self.format_uri("+")]
                 faldo_strand = self.get_faldo_strand("+")
                 self.faldo_abstraction["strand"] = relation
-                self.graph_chunk.add((entity, relation, attribute))
+                # self.graph_chunk.add((entity, relation, attribute))
                 strand = True
                 strand_type = "+"
             elif feature.strand == "-":
@@ -242,7 +242,7 @@ def generate_rdf_content(self):
                 attribute = self.namespace_data[self.format_uri("-")]
                 faldo_strand = self.get_faldo_strand("-")
                 self.faldo_abstraction["strand"] = relation
-                self.graph_chunk.add((entity, relation, attribute))
+                # self.graph_chunk.add((entity, relation, attribute))
                 strand = True
                 strand_type = "-"
             else:
@@ -251,7 +251,7 @@ def generate_rdf_content(self):
                 attribute = self.namespace_data[self.format_uri(".")]
                 faldo_strand = self.get_faldo_strand(".")
                 self.faldo_abstraction["strand"] = relation
-                self.graph_chunk.add((entity, relation, attribute))
+                # self.graph_chunk.add((entity, relation, attribute))
                 strand = True
                 strand_type = "."
 
@@ -271,7 +271,7 @@ def generate_rdf_content(self):
             if feature.score != '.':
                 relation = self.namespace_data[self.format_uri("score")]
                 attribute = rdflib.Literal(self.convert_type(feature.score))
-                self.graph_chunk.add((entity, relation, attribute))
+                # self.graph_chunk.add((entity, relation, attribute))
 
                 if "score" not in attribute_list:
                     attribute_list.append("score")
@@ -283,6 +283,8 @@ def generate_rdf_content(self):
                         "range": rdflib.XSD.decimal
                     })
 
+            # Triples respecting faldo ontology
+
             location = BNode()
             begin = BNode()
             end = BNode()
@@ -297,14 +299,23 @@ def generate_rdf_content(self):
             self.graph_chunk.add((begin, self.faldo.position, faldo_start))
 
             self.graph_chunk.add((end, rdflib.RDF.type, self.faldo.ExactPosition))
-            self.graph_chunk.add((end, self.faldo.position, faldo_end))
+            self.graph_chunk.add((end, self.faldo.end, faldo_end))
 
             self.graph_chunk.add((begin, self.faldo.reference, faldo_reference))
-            self.graph_chunk.add((end, self.faldo.reference, faldo_reference))
 
             if faldo_strand:
-                self.graph_chunk.add((begin, rdflib.RDF.type, faldo_strand))
-                self.graph_chunk.add((end, rdflib.RDF.type, faldo_strand))
+                self.graph_chunk.add((begin, self.faldo.strand, faldo_strand))
+
+            # Shortcut triple for faldo queries
+            self.graph_chunk.add((entity, self.namespace_internal["faldoBegin"], faldo_start))
+            self.graph_chunk.add((entity, self.namespace_internal["faldoEnd"], faldo_end))
+            self.graph_chunk.add((entity, self.namespace_internal["faldoReference"], faldo_reference))
+
+            if faldo_strand:
+                self.graph_chunk.add((entity, self.namespace_internal["faldoStrand"], faldo_strand))
+                strand_ref = self.get_reference_strand_uri(feature.chrom, faldo_strand, None)
+                for sref in strand_ref:
+                    self.graph_chunk.add((entity, self.namespace_internal["referenceStrand"], sref))
 
             # blocks
             block_base = self.settings.getint("triplestore", "block_size")

askomics/libaskomics/CsvFile.py

@@ -416,8 +416,9 @@ def set_rdf_abstraction(self):
 
             blank = BNode()
             # Relation
-            if self.columns_type[index] in ('general_relation', 'symetric_relation'):
+            if self.columns_type[index] in ('general_relation', 'symetric_relation', 'indirect_relation'):
                 symetric_relation = True if self.columns_type[index] == 'symetric_relation' else False
+                indirect_relation = True if self.columns_type[index] == 'indirect_relation' else False
                 splitted = attribute_name.split('@')
 
                 attribute = self.rdfize(splitted[0])
@@ -439,6 +440,8 @@ def set_rdf_abstraction(self):
                 if symetric_relation:
                     self.graph_abstraction_dk.add((blank, rdflib.RDFS.domain, rdf_range))
                     self.graph_abstraction_dk.add((blank, rdflib.RDFS.range, entity))
+                if indirect_relation:
+                    self.graph_abstraction_dk.add((blank, self.namespace_internal["isIndirectRelation"], rdflib.Literal("true", datatype=rdflib.XSD.boolean)))
 
                 continue
 
@@ -597,6 +600,10 @@ def generate_rdf_content(self):
                     if current_type == "label" and column_number == 1:
                         continue
 
+                    # We ignore all data for indirect relations
+                    if current_type == "indirect_relation":
+                        continue
+
                     # Skip entity and blank cells
                     if column_number == 0 or (not cell and not current_type == "strand"):
                         continue
@@ -675,6 +682,9 @@ def generate_rdf_content(self):
                             self.graph_chunk.add((attribute, relation, entity))
 
                 if self.faldo_entity and faldo_start and faldo_end:
+
+                    # Triples respecting faldo ontology
+
                     location = BNode()
                     begin_node = BNode()
                     end_node = BNode()
@@ -699,6 +709,19 @@ def generate_rdf_content(self):
                         self.graph_chunk.add((begin_node, rdflib.RDF.type, faldo_strand))
                         self.graph_chunk.add((end_node, rdflib.RDF.type, faldo_strand))
 
+                    # Shortcut triple for faldo queries
+                    self.graph_chunk.add((entity, self.namespace_internal["faldoBegin"], faldo_start))
+                    self.graph_chunk.add((entity, self.namespace_internal["faldoEnd"], faldo_end))
+                    if faldo_reference:
+                        self.graph_chunk.add((entity, self.namespace_internal["faldoReference"], faldo_reference))
+                        if faldo_strand:
+                            strand_ref = self.get_reference_strand_uri(reference, faldo_strand, None)
+                            for sref in strand_ref:
+                                self.graph_chunk.add((entity, self.namespace_internal["referenceStrand"], sref))
+
+                    if faldo_strand:
+                        self.graph_chunk.add((entity, self.namespace_internal["faldoStrand"], faldo_strand))
+
                     # blocks
                     block_base = self.settings.getint("triplestore", "block_size")
                     block_start = int(start) // block_base

askomics/libaskomics/File.py

@@ -458,12 +458,17 @@ def get_faldo_strand_label(self, raw_strand):
 
         return "."
 
-    def get_reference_strand_uri(self, reference, strand, block):
+    def get_reference_strand_uri(self, reference, strand, block=None):
         faldo_dict = {
             self.faldo.ForwardStrandPosition: "ForwardStrand",
             self.faldo.ReverseStrandPosition: "ReverseStrand",
             self.faldo.BothStrandPosition: "BothStrand"
         }
+        if block is not None:
+            if strand == self.faldo.BothStrandPosition:
+                return [self.rdfize(self.format_uri("{}_s{}s".format(reference, dstrand))) for dstrand in faldo_dict.values()]
+            return [self.rdfize(self.format_uri("{}_s{}".format(reference, faldo_dict[strand])))]
+
         if strand == self.faldo.BothStrandPosition:
             return [self.rdfize(self.format_uri("{}_s{}_{}".format(reference, dstrand, block))) for dstrand in faldo_dict.values()]
 

askomics/libaskomics/GffFile.py

@@ -388,6 +388,8 @@ def generate_rdf_content(self):
 
                         self.graph_chunk.add((entity, relation, attribute))
 
+                # Triples respecting faldo ontology
+
                 location = BNode()
                 begin = BNode()
                 end = BNode()
@@ -411,6 +413,17 @@ def generate_rdf_content(self):
                     self.graph_chunk.add((begin, rdflib.RDF.type, faldo_strand))
                     self.graph_chunk.add((end, rdflib.RDF.type, faldo_strand))
 
+                # Shortcut triple for faldo queries
+                self.graph_chunk.add((entity, self.namespace_internal["faldoBegin"], faldo_start))
+                self.graph_chunk.add((entity, self.namespace_internal["faldoEnd"], faldo_end))
+                self.graph_chunk.add((entity, self.namespace_internal["faldoReference"], faldo_reference))
+
+                if faldo_strand:
+                    self.graph_chunk.add((entity, self.namespace_internal["faldoStrand"], faldo_strand))
+                    strand_ref = self.get_reference_strand_uri(rec.id, faldo_strand, None)
+                    for sref in strand_ref:
+                        self.graph_chunk.add((entity, self.namespace_internal["referenceStrand"], sref))
+
                 # blocks
                 block_base = self.settings.getint("triplestore", "block_size")
                 block_start = int(self.convert_type(feature.location.start)) // block_base

askomics/libaskomics/SparqlQuery.py

@@ -1261,21 +1261,22 @@ def build_query_from_json(self, preview=False, for_editor=False):
                     elif link["sameStrand"]:
                         block_uri = "includeInStrand"
 
-                    self.store_triple({
-                        "subject": source,
-                        "predicate": "askomics:{}".format(block_uri),
-                        "object": common_block,
-                        "optional": False
+                    if link["uri"] in ('included_in', 'overlap_with'):
+                        self.store_triple({
+                            "subject": source,
+                            "predicate": "askomics:{}".format(block_uri),
+                            "object": common_block,
+                            "optional": False
 
-                    }, block_id, sblock_id, pblock_ids, depth)
+                        }, block_id, sblock_id, pblock_ids, depth)
 
-                    self.store_triple({
-                        "subject": target,
-                        "predicate": "askomics:{}".format(block_uri),
-                        "object": common_block,
-                        "optional": False
+                        self.store_triple({
+                            "subject": target,
+                            "predicate": "askomics:{}".format(block_uri),
+                            "object": common_block,
+                            "optional": False
 
-                    }, block_id, sblock_id, pblock_ids, depth)
+                        }, block_id, sblock_id, pblock_ids, depth)
 
                     equal_sign = "" if link["strict"] else "="
 
@@ -1297,6 +1298,57 @@ def build_query_from_json(self, preview=False, for_editor=False):
                             equalsign=equal_sign
                         ), block_id, sblock_id, pblock_ids, depth)
                     else:
+                        if link["sameRef"]:
+                            if link['sameStrand']:
+                                self.store_triple({
+                                    "subject": source,
+                                    "predicate": "askomics:referenceStrand",
+                                    "object": common_block,
+                                    "optional": False
+
+                                }, block_id, sblock_id, pblock_ids, depth)
+
+                                self.store_triple({
+                                    "subject": target,
+                                    "predicate": "askomics:referenceStrand",
+                                    "object": common_block,
+                                    "optional": False
+
+                                }, block_id, sblock_id, pblock_ids, depth)
+                            else:
+                                self.store_triple({
+                                    "subject": source,
+                                    "predicate": "askomics:faldoReference",
+                                    "object": common_block,
+                                    "optional": False
+
+                                }, block_id, sblock_id, pblock_ids, depth)
+
+                                self.store_triple({
+                                    "subject": target,
+                                    "predicate": "askomics:faldoReference",
+                                    "object": common_block,
+                                    "optional": False
+
+                                }, block_id, sblock_id, pblock_ids, depth)
+
+                        elif link["sameStrand"]:
+                            self.store_triple({
+                                "subject": source,
+                                "predicate": "askomics:faldoStrand",
+                                "object": common_block,
+                                "optional": False
+
+                            }, block_id, sblock_id, pblock_ids, depth)
+
+                            self.store_triple({
+                                "subject": target,
+                                "predicate": "askomics:faldoStrand",
+                                "object": common_block,
+                                "optional": False
+
+                            }, block_id, sblock_id, pblock_ids, depth)
+
                         for filter in link.get('faldoFilters', []):
                             modifier_string = ""
                             if filter['filterValue']:
@@ -1330,7 +1382,8 @@ def build_query_from_json(self, preview=False, for_editor=False):
                         "optional": False
                     }
 
-                    self.store_triple(triple, block_id, sblock_id, pblock_ids, depth)
+                    if not link.get('indirect', False):
+                        self.store_triple(triple, block_id, sblock_id, pblock_ids, depth)
 
         # Store linked attributes
         for attribute in self.json["attr"]:

askomics/libaskomics/TriplestoreExplorer.py

@@ -519,7 +519,7 @@ def get_abstraction_relations(self, single_tenant=False):
         query_builder = SparqlQuery(self.app, self.session)
 
         query = '''
-        SELECT DISTINCT ?graph ?entity_uri ?entity_faldo ?entity_label ?attribute_uri ?attribute_faldo ?attribute_label ?attribute_range ?property_uri ?property_faldo ?property_label ?range_uri ?category_value_uri ?category_value_label
+        SELECT DISTINCT ?graph ?entity_uri ?entity_faldo ?entity_label ?attribute_uri ?attribute_faldo ?attribute_label ?attribute_range ?property_uri ?property_faldo ?property_label ?range_uri ?category_value_uri ?category_value_label ?indirect_relation
         WHERE {{
             # Graphs
             ?graph askomics:public ?public .
@@ -530,6 +530,7 @@ def get_abstraction_relations(self, single_tenant=False):
                 ?node a askomics:AskomicsRelation .
                 ?node rdfs:label ?property_label .
                 ?node rdfs:range ?range_uri .
+                OPTIONAL {{ ?node askomics:isIndirectRelation ?indirect_relation . }}
                 # Retrocompatibility
                 OPTIONAL {{?node askomics:uri ?new_property_uri}}
                 BIND( IF(isBlank(?node), ?new_property_uri, ?node) as ?property_uri)
@@ -563,7 +564,8 @@ def get_abstraction_relations(self, single_tenant=False):
                         "label": result["property_label"],
                         "graphs": [result["graph"], ],
                         "source": result["entity_uri"],
-                        "target": result["range_uri"]
+                        "target": result["range_uri"],
+                        "indirect": result.get("indirect_relation", False)
                     }
                     relations.append(relation)
                 else:

askomics/react/src/routes/integration/csvtable.jsx

@@ -129,6 +129,7 @@ export default class CsvTable extends Component {
               <optgroup label="Relation">
                 <option value="general_relation" >Directed</option>
                 <option value="symetric_relation" >Symetric</option>
+                <option value="indirect_relation" >Indirect</option>
               </optgroup>
               {ontoInput}
             </CustomInput>
@@ -158,6 +159,7 @@ export default class CsvTable extends Component {
             <optgroup label="Relation">
               <option value="general_relation" >Directed</option>
               <option value="symetric_relation" >Symetric</option>
+              <option value="indirect_relation" >Indirect</option>
             </optgroup>
             {ontoInput}
           </CustomInput>