Merge branch 'master' into matchmaker_search_nodes

CHANGELOG.md

@@ -9,6 +9,22 @@ About changelog [here](https://keepachangelog.com/en/1.0.0/)
 Add stuff here
 
 
+## [4.3.2]
+
+### Added
+- Dashboard data can be filtered using filters available in cases page
+- Causatives for each institute are displayed on a dedicated page
+- SNVs and and SVs are searchable across cases by gene and rank score
+- A more complete report with validated variants is downloadable from dashboard
+
+### Fixed
+- Clinsig filter is fixed so clinsig numerical values are returned
+- Split multi clinsig string values in different elements of clinsig array
+- Regex to search in multi clinsig string values or multi revstat string values
+- It works to upload vcf files with no variants now
+- Combined Pileup and IGV alignments for SVs having variant start and stop on the same chromosome
+
+
 ## [4.3.1]
 
 ### Added

scout/__version__.py

@@ -1 +1 @@
-__version__ = '4.3.1'
+__version__ = '4.3.2'

scout/adapter/mongo/query.py

@@ -1,4 +1,5 @@
 import logging
+import re
 
 logger = logging.getLogger(__name__)
 
@@ -308,33 +309,60 @@ def build_query(self, case_id, query=None, variant_ids=None, category='snv'):
 
         if query.get('clinsig'):
             rank = []
+            str_rank = []
+
             for item in query['clinsig']:
-                rank.append(item)
+                rank.append(int(item))
                 # search for human readable clinsig values in newer cases
                 rank.append(CLINSIG_MAP[int(item)])
+                str_rank.append(CLINSIG_MAP[int(item)])
 
             if query.get('clinsig_confident_always_returned') == True:
 
                 trusted_revision_level = ['mult', 'single', 'exp', 'guideline']
 
                 mongo_query_major = { "clnsig":
                         {
-                            '$elemMatch': { 'value':
-                                            { '$in': rank },
-                                            'revstat':
-                                            { '$in': trusted_revision_level }
-                                          }
+                            '$elemMatch': {
+                                '$or' : [
+                                    {
+                                        '$and' : [
+                                             {'value' : { '$in': rank }},
+                                             {'revstat': { '$in': trusted_revision_level }}
+                                        ]
+                                    },
+                                    {
+                                        '$and': [
+                                            {'value' : re.compile('|'.join(str_rank))},
+                                            {'revstat' : re.compile('|'.join(trusted_revision_level))}
+                                        ]
+                                    }
+                                ]
+                            }
                          }
                     }
 
             else:
                 logger.debug("add CLINSIG filter for rank: %s" %
                              ', '.join(str(query['clinsig'])))
+                clnsig_query = {
+                    "clnsig":
+                        {
+                            '$elemMatch': {
+                                '$or' : [
+                                    { 'value' : { '$in': rank }},
+                                    { 'value' : re.compile('|'.join(str_rank)) }
+                                ]
+                            }
+                        }
+                }
                 if mongo_query_minor:
-                    mongo_query_minor.append({'clnsig.value': {'$in': rank}})
+                    #mongo_query_minor.append({'clnsig.value': {'$in': rank}})
+                    mongo_query_minor.append(clnsig_query)
+
                 else:
                     # if this is the only minor critera, use implicit and.
-                    mongo_query['clnsig.value'] = {'$in': rank}
+                    mongo_query['clnsig'] = clnsig_query['clnsig']
 
         if mongo_query_minor and mongo_query_major:
             if gene_query:

scout/parse/variant/clnsig.py

@@ -18,7 +18,7 @@ def parse_clnsig(acc, sig, revstat, transcripts):
         clnsig_accsessions(list): A list with clnsig accessions
     """
     clnsig_accsessions = []
-    
+
     if acc:
     # New format of clinvar allways have integers as accession numbers
         try:
@@ -31,18 +31,19 @@ def parse_clnsig(acc, sig, revstat, transcripts):
             revstat_groups = []
             if revstat:
                 revstat_groups = [rev.lstrip('_') for rev in revstat.split(',')]
-            
+
             sig_groups = []
             if sig:
                 for significance in sig.split('/'):
                     splitted_word = significance.split('_')
                     sig_groups.append(' '.join(splitted_word[:2]))
-
-            clnsig_accsessions.append({
-                'value': ', '.join(sig_groups),
-                'accession': int(acc),
-                'revstat': ', '.join(revstat_groups),
-            })
+
+            for sign_term in sig_groups:
+                clnsig_accsessions.append({
+                    'value': sign_term,
+                    'accession': int(acc),
+                    'revstat': ', '.join(revstat_groups),
+                })
         else:
             # There are sometimes different separators so we need to check which
             # one to use

scout/server/blueprints/alignviewers/views.py

@@ -132,7 +132,9 @@ def igv():
     for sample in samples:
         # some samples might not have an associated bam file, take care if this
         if bam_files[counter]:
-            sample_tracks.append({ 'name' : sample, 'url' : bam_files[counter], 'indexURL' : bai_files[counter] })
+            sample_tracks.append({ 'name' : sample, 'url' : bam_files[counter],
+                                   'indexURL' : bai_files[counter],
+                                   'height' : 700, 'maxHeight' : 2000})
         counter += 1
 
     display_obj['sample_tracks'] = sample_tracks

scout/server/blueprints/cases/controllers.py

@@ -452,14 +452,38 @@ def gene_variants(store, variants_query, page=1, per_page=50):
     variants = []
     for variant_obj in variant_res:
         # hide other institutes for now
-        if (variant_obj['institute'] not in my_institutes):
-            LOG.debug("Institute {} not allowed.".format(variant_obj['institute']))
+        if variant_obj['institute'] not in my_institutes:
+            log.debug("Institute {} not allowed.".format(variant_obj['institute']))
             continue
 
+        # Populate variant case_display_name
         variant_case_obj = store.case(case_id=variant_obj['case_id'])
-        case_display_name = variant_case_obj['display_name']
+        if not variant_case_obj:
+            # A variant with missing case was encountered
+            continue
+        case_display_name = variant_case_obj.get('display_name')
         variant_obj['case_display_name'] = case_display_name
 
+        genome_build = variant_case_obj.get('genome_build', '37')
+        if genome_build not in ['37','38']:
+            genome_build = '37'
+
+        # Update the HGNC symbols if they are not set
+        variant_genes = variant_obj.get('genes')
+        if variant_genes is not None:
+            for gene_obj in variant_genes:
+                # If there is no hgnc id there is nothin we can do
+                if not gene_obj['hgnc_id']:
+                    continue
+                # Else we collect the gene object and check the id
+                if gene_obj.get('hgnc_symbol') is None or gene_obj.get('description') is None:
+                    hgnc_gene = store.hgnc_gene(gene_obj['hgnc_id'], build=genome_build)
+                    if not hgnc_gene:
+                        continue
+                    gene_obj['hgnc_symbol'] = hgnc_gene['hgnc_symbol']
+                    gene_obj['description'] = hgnc_gene['description']
+
+        # Populate variant HGVS and predictions
         gene_ids = []
         gene_symbols = []
         hgvs_c = []
@@ -468,6 +492,7 @@ def gene_variants(store, variants_query, page=1, per_page=50):
 
         if variant_genes is not None:
             functional_annotation = ''
+
             for gene_obj in variant_genes:
                 hgnc_id = gene_obj['hgnc_id']
                 gene_symbol = gene(store, hgnc_id)['symbol']
@@ -494,7 +519,10 @@ def gene_variants(store, variants_query, page=1, per_page=50):
                 else:
                     hgvs = "-"
                 variant_obj['hgvs'] = hgvs
+
+            # populate variant predictions for display
             variant_obj.update(get_predictions(variant_genes))
+
         variants.append(variant_obj)
 
     return {

scout/server/blueprints/cases/forms.py

@@ -27,3 +27,4 @@ class GeneVariantFiltersForm(FlaskForm):
     variant_type = SelectMultipleField(choices=[('clinical','clinical'),('research','research')])
     hgnc_symbols = TagListField('HGNC Symbols/Ids (case sensitive)')
     filter_variants = SubmitField(label='Filter variants')
+    rank_score = IntegerField()

scout/server/blueprints/cases/templates/cases/gene_variants.html

@@ -102,14 +102,22 @@ <h4 class="panel-title">
 
 {% macro cell_rank(variant) %}
 <a class="variants-row-item flex-small layout"
-   href="{{ url_for('variants.variant', institute_id=institute._id,
+   href="{{ url_for('variants.variant', institute_id=variant.institute,
         case_name=variant.case_display_name, variant_id=variant._id) }}">
   {{ variant.case_display_name}} : {{ variant.rank_score|int }}
 </a>
 {% endmacro %}
 
 {% macro cell_cadd(variant) %}
   <div>
+    <div data-toggle="tooltip" data-placement="left" data-html="true" title="
+    <div class='text-left'>
+      <strong>Sift</strong>
+      {{ variant.sift_predictions|join(',') }} <br>
+      <strong>PolyPhen</strong>
+      {{ (variant.polyphen_predictions or ['-'])|join(',') }}		
+    </div>
+    ">
     {{ variant.cadd_score }}
   </div>
 {% endmacro %}
@@ -145,7 +153,7 @@ <h4 class="panel-title">
         </div>
         <div class="col-xs-2">
           <label class="control-label">Rank Score</label>
-          <input type="number" class="form-control" id="rank_score" name="rank_score" min="5">
+          <input type="number" class="form-control" id="rank_score" name="rank_score" min="5" value={{form.rank_score.data}}>
         </div>
         <div class="col-xs-2">
           {{ form.variant_type.label(class="control-label") }}
@@ -174,6 +182,11 @@ <h4 class="panel-title">
   <script src="https://cdnjs.cloudflare.com/ajax/libs/bootstrap-multiselect/0.9.13/js/bootstrap-multiselect.min.js"></script>
   <script>
     $(function () {
+      $('[data-toggle="tooltip"]').tooltip();
+      $('[data-toggle="popover"]').popover({
+        container: 'body',
+      });
+
       $('table').stickyTableHeaders({
         fixedOffset: $(".navbar-fixed-top")
       });

scout/server/blueprints/cases/views.py

@@ -314,6 +314,7 @@ def gene_variants(institute_id):
     non_clinical_symbols = []
     not_found_symbols = []
     not_found_ids = []
+    data = {}
     if (form.hgnc_symbols.data) and len(form.hgnc_symbols.data) > 0:
         is_clinical = form.data.get('variant_type', 'clinical') == 'clinical'
         clinical_symbols = store.clinical_symbols(case_obj) if is_clinical else None
@@ -339,13 +340,12 @@ def gene_variants(institute_id):
             flash("Gene not included in clinical list: {}".format(", ".join(non_clinical_symbols)), 'warning')
         form.hgnc_symbols.data = hgnc_symbols
 
-    log.debug("query {}".format(form.data))
+        log.debug("query {}".format(form.data))
 
-    variants_query = store.gene_variants(query=form.data, category='snv',
+        variants_query = store.gene_variants(query=form.data, category='snv',
                             variant_type=variant_type)
-    data = {}
 
-    data = controllers.gene_variants(store, variants_query, page)
+        data = controllers.gene_variants(store, variants_query, page)
 
     return dict(institute=institute_obj, form=form, page=page, **data)
 

scout/server/blueprints/dashboard/controllers.py

@@ -1,4 +1,5 @@
 import logging
+from flask_login import current_user
 
 LOG = logging.getLogger(__name__)
 
@@ -15,6 +16,13 @@ def get_dashboard_info(adapter, institute_id=None, slice_query=None):
     Returns:
         data(dict): Dictionary with relevant information
     """
+
+    LOG.debug("General query with institute_id {}.".format(institute_id))
+
+    # if institute_id == 'None' or None, all cases and general stats will be returned
+    if institute_id == 'None':
+        institute_id = None
+
     general_info = get_general_case_info(adapter, institute_id=institute_id,
                                                     slice_query=slice_query)
     total_cases = general_info['total_cases']
@@ -154,17 +162,11 @@ def get_general_case_info(adapter, institute_id=None, slice_query=None):
         general(dict)
     """
     general = {}
-    # Fetch information about cases with certain activities
-    cases = {}
 
-    if institute_id and slice_query:
-        cases = adapter.cases(owner=institute_id, name_query=slice_query)
-    elif institute_id:
-        cases = adapter.cases(owner=institute_id)
-    elif slice_query:
-        cases = adapter.cases(name_query=slice_query)
-    else:
-        cases = adapter.cases()
+    # Potentially sensitive slice queries are assumed allowed if we have got this far
+    name_query = slice_query
+
+    cases = adapter.cases(owner=institute_id, name_query=name_query)
 
     phenotype_cases = 0
     causative_cases = 0
@@ -242,7 +244,6 @@ def get_case_groups(adapter, total_cases, institute_id=None, slice_query=None):
     pipeline = []
     group = {'$group' : {'_id': '$status', 'count': {'$sum': 1}}}
 
-
     subquery = {}
     if institute_id and slice_query:
         subquery = adapter.cases(owner=institute_id, name_query=slice_query,

scout/server/blueprints/dashboard/templates/dashboard/index.html

@@ -1,6 +1,5 @@
 {% extends "layout.html" %}
 
-
 {% block content_main %}
 {{ dashboard_search_form() }}
 
@@ -107,13 +106,17 @@ <h1 class="text-center">{{ topic.count }}</h1>
           <span class="input-group-addon">
             <span class="glyphicon glyphicon-search"></span>
           </span>
-          <input type="search" class="form-control" value="{{ query if query }}" name="query" placeholder="Enter query to filter cases summarized below"></input>
+          <input type="search" class="form-control" value="{{ query if query }}" name="query" id="query" placeholder="Enter query to filter cases summarized below" onchange="update_select({% if current_user.is_admin %}true{% else %}false{% endif %})"></input>
           </div>
       </div>
       <div class="col-md-2 col-xs-2">
-      <select name="institute" onchange="this.form.submit()">
+      <select name="institute" id="institute" onchange="this.form.submit()">
         {% for inst in institutes %}
-          <option value="{{ inst._id }}" {{ "selected" if inst._id == choice }} >{{ inst.display_name }}</option>
+          {% if inst.display_name == 'All institutes' and query and not current_user.is_admin %}
+            <option value="{{ inst._id }}" disabled>{{ inst.display_name }}</option>
+          {% else %}
+            <option value="{{ inst._id }}" {{ "selected" if inst._id == choice }} >{{ inst.display_name }}</option>
+          {% endif %}
         {% endfor %}
       </select>
       </div>
@@ -126,3 +129,23 @@ <h1 class="text-center">{{ topic.count }}</h1>
     </div>
   </form>
 {% endmacro %}
+
+{% block scripts %}
+  {{ super() }}
+  <script type="text/javascript">
+    function update_select(admin_user){
+      var query_text = document.getElementById("query").value;
+      var sel = document.getElementById("institute");
+      if(query_text && !admin_user) {
+        sel.children[0].disabled = "disabled";
+        if(sel.options[0].selected) {
+          sel.options[0].selected = false;
+          sel.options[1].selected = true;
+        }
+      }
+      else{
+        sel.children[0].disabled = "";
+      }
+    }
+  </script>
+{% endblock %}