Add extra annotations scores to the filtered variants (varfish-org#242).

HISTORY.rst

@@ -45,6 +45,7 @@ End-User Summary
 - Adding mitochondrial inheritance to case phenotype annotation (#325)
 - Fix issue with variant annotation export (#328)
 - Allowing direct update of variant annotations and ACMG ratings on case annotations details (#344)
+- Add extra annotations, i.e. additional variant scores to the filtered variants (#242)
 
 Full Change List
 ================
@@ -98,6 +99,7 @@ Full Change List
 - DgvSvs and DgvGoldStandardSvs are two different data sources now
 - Adding deep linking into case details tab (#344)
 - Allowing direct update of variant annotations and ACMG ratings on case annotations details (#344)
+- Add extra annotations to the filtered variants (#242)
 
 -------
 v0.23.9

variants/queries.py

@@ -16,6 +16,7 @@
 from cohorts.models import Cohort
 from conservation.models import KnowngeneAA
 from dbsnp.models import Dbsnp
+from extra_annos.models import ExtraAnno
 from frequencies.models import MtDb, HelixMtDb, Mitomap
 from geneinfo.models import (
     Hgnc,
@@ -1101,6 +1102,40 @@ def extend_selectable(self, query_parts):
         return query_parts.selectable.outerjoin(self.subquery, true())
 
 
+class ExtendQueryPartsCommentsExtraAnnoJoin(ExtendQueryPartsBase):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.subquery = (
+            select([func.array_agg(ExtraAnno.sa.anno_data).label("extra_annos")])
+            .select_from(ExtraAnno.sa)
+            .where(
+                and_(
+                    ExtraAnno.sa.release == SmallVariant.sa.release,
+                    ExtraAnno.sa.chromosome == SmallVariant.sa.chromosome,
+                    ExtraAnno.sa.start == SmallVariant.sa.start,
+                    ExtraAnno.sa.end == SmallVariant.sa.end,
+                    ExtraAnno.sa.reference == SmallVariant.sa.reference,
+                    ExtraAnno.sa.alternative == SmallVariant.sa.alternative,
+                )
+            )
+            .group_by(
+                ExtraAnno.sa.release,
+                ExtraAnno.sa.chromosome,
+                ExtraAnno.sa.start,
+                ExtraAnno.sa.end,
+                ExtraAnno.sa.reference,
+                ExtraAnno.sa.alternative,
+            )
+            .lateral("extra_annos_subquery")
+        )
+
+    def extend_fields(self, _query_parts):
+        return [self.subquery.c.extra_annos]
+
+    def extend_selectable(self, query_parts):
+        return query_parts.selectable.outerjoin(self.subquery, true())
+
+
 class ExtendQueryPartsFlagsJoin(ExtendQueryPartsBase):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -1353,6 +1388,7 @@ def extend_selectable(self, query_parts):
     ExtendQueryPartsGenotypeGtQualityDefaultFilter,
     ExtendQueryPartsFlagsJoinAndFilter,
     ExtendQueryPartsCommentsJoin,
+    ExtendQueryPartsCommentsExtraAnnoJoin,
     ExtendQueryPartsAcmgCriteriaJoin,
 ]
 
@@ -1404,6 +1440,7 @@ class CaseLoadPrefetchedQueryPartsBuilder(QueryPartsBuilder):
         ExtendQueryPartsMgiJoin,
         ExtendQueryPartsFlagsJoinAndFilter,
         ExtendQueryPartsCommentsJoin,
+        ExtendQueryPartsCommentsExtraAnnoJoin,
         ExtendQueryPartsAcmgCriteriaJoin,
         ExtendQueryPartsModesOfInheritanceJoin,
         ExtendQueryPartsDiseaseGeneJoin,

variants/templates/variants/filter_result/header.html

@@ -31,6 +31,9 @@
   <th class="effect-change-protein"></th>{# effect change protein #}
   <th class="effect-change-cdna"></th>{# effect change cdna #}
   <th class="distance-to-splicesite"></th>{# distance to splicesite #}
+  {% for value in result_extra_annos_header %}
+    <th class="effect-change-extra_annos-{{ forloop.counter }}"></th>
+  {% endfor %}
   {% if query_type == "project" %}
     <th></th> {# cases per gene #}
   {% endif %}
@@ -97,6 +100,9 @@
   <th class="effect-change-protein">effect protein</th>
   <th class="effect-change-cdna">effect cdna</th>
   <th class="distance-to-splicesite">dist. to splicesite</th>
+  {% for value in result_extra_annos_header %}
+    <th class="effect-change-extra_annos-{{ forloop.counter }}"> {{ value }}</th>
+  {% endfor %}
   {% if query_type == "project" %}
     <th data-toggle="tooltip" data-placement="top" data-html="true" title="Number affected cases per gene.">
       <sup>#c</sup>&frasl;<sub>g</sub>

variants/templates/variants/filter_result/row.html

@@ -387,6 +387,14 @@
   <td class="text-nowrap effect-change-protein">{{ entry.hgvs_p|default:'-' }}</td>
   <td class="text-nowrap effect-change-cdna">{{ entry.hgvs_c|default:'-' }}</td>
   <td class="text-nowrap distance-to-splicesite">{{ entry.exon_dist|default:'-' }}</td>
+  {% if entry.extra_annos is not None %}
+    {% for value in entry.extra_annos.0 %}
+      <td class="text-nowrap effect-change-extra_annos-{{ forloop.counter }}">{{ value|default:'-' }}</td>
+    {% endfor %}
+  {% else %}
+    {% for value in result_extra_annos_header %}
+      <td class="text-nowrap effect-change-extra_annos-{{ forloop.counter }}">{{ '-' }}</td>  {% endfor %}
+  {% endif %}
   {% if query_type == "project" %}
     <td>{{ entry.affected_cases_per_gene|default:0 }}</td>
   {% endif %}

variants/templates/variants/filter_result/table.html

@@ -210,6 +210,9 @@ <h2>
             <option value="effect-change-protein">Effect Protein</option>
             <option value="effect-change-cdna">Effect cDNA</option>
             <option value="distance-to-splicesite">Distance to SpliceSite</option>
+            {% for value in result_extra_annos_header %}
+              <option value="effect-change-extra_annos-{{ forloop.counter }}"}}>{{ value }}</option>
+            {% endfor %}
           </select>
         </div>
         <div class="pl-3">
@@ -307,4 +310,3 @@ <h4 class="modal-title">Multi-Variant Comments &amp; Flags</h4>
     }
     </script>
   {% endif %}
-

variants/tests/test_queries.py

@@ -10,6 +10,7 @@
 from clinvar.tests.factories import ClinvarFactory
 from cohorts.tests.factories import TestCohortBase
 from conservation.tests.factories import KnownGeneAAFactory
+from extra_annos.tests.factories import ExtraAnnoFactory
 from frequencies.tests.factories import MitomapFactory, HelixMtDbFactory, MtDbFactory
 from hgmd.tests.factories import HgmdPublicLocusFactory
 from variants.models import Case, SmallVariantSet, SmallVariantFlags
@@ -5134,3 +5135,57 @@ def test_run_with_project(self):
         self.assertEqual(len(res.small_variant_flags), 0)
         self.assertEqual(len(res.small_variant_comments), 0)
         self.assertEqual(len(res.acmg_criteria_rating), 1)
+
+
+class TestSmallVariantExtraAnno(SupportQueryTestBase):
+    """Test extra annotations."""
+
+    def setUp(self):
+        """Create 3 variants and two contain extra anno entries."""
+        super().setUp()
+        _, variant_set, _ = CaseWithVariantSetFactory.get("small")
+        small_vars = SmallVariantFactory.create_batch(3, variant_set=variant_set)
+        self.extra_anno = [
+            ExtraAnnoFactory(
+                release=small_vars[0].release,
+                chromosome=small_vars[0].chromosome,
+                start=small_vars[0].start,
+                end=small_vars[0].end,
+                bin=small_vars[0].bin,
+                reference=small_vars[0].reference,
+                alternative=small_vars[0].alternative,
+                anno_data=[9.71],
+            ),
+            ExtraAnnoFactory(
+                release=small_vars[2].release,
+                chromosome=small_vars[2].chromosome,
+                start=small_vars[2].start,
+                end=small_vars[2].end,
+                bin=small_vars[2].bin,
+                reference=small_vars[2].reference,
+                alternative=small_vars[2].alternative,
+                anno_data=[10],
+            ),
+        ]
+
+    def test_base_query_filter(self):
+        res = self.run_query(CasePrefetchQuery, {}, 3)
+        self.assertEqual(res[0]["extra_annos"][0], self.extra_anno[0].anno_data)
+        self.assertEqual(res[1]["extra_annos"], None)
+        self.assertEqual(res[2]["extra_annos"][0], self.extra_anno[1].anno_data)
+
+
+class TestSmallVariantNoExtraAnno(SupportQueryTestBase):
+    """Test extra annotations completely missing."""
+
+    def setUp(self):
+        """Create 3 variants and no extra anno entries."""
+        super().setUp()
+        _, variant_set, _ = CaseWithVariantSetFactory.get("small")
+        small_vars = SmallVariantFactory.create_batch(3, variant_set=variant_set)
+
+    def test_base_query_filter(self):
+        res = self.run_query(CasePrefetchQuery, {}, 3)
+        self.assertEqual(res[0]["extra_annos"], None)
+        self.assertEqual(res[1]["extra_annos"], None)
+        self.assertEqual(res[2]["extra_annos"], None)

variants/tests/test_ui.py

@@ -21,6 +21,7 @@
 from projectroles.tests.test_models import ProjectMixin, RoleAssignmentMixin
 import projectroles.tests.test_ui
 
+from extra_annos.tests.factories import ExtraAnnoFactory, ExtraAnnoFieldFactory
 from variants.tests.factories import (
     SampleVariantStatisticsFactory,
     SmallVariantFactory,
@@ -1106,3 +1107,125 @@ def test_variant_joint_filter_download(self):
                 )
             )
         )
+
+
+class TestVariantsCaseFilterViewExtraAnno(TestVariantsCaseFilterView):
+    """Tests for the variants case filter view."""
+
+    view = "variants:case-filter"
+    window_size = (4000, 1300)
+
+    def setUp(self):
+        super().setUp()
+        self.case, self.variant_set, _ = CaseWithVariantSetFactory.get("small")
+        small_vars = SmallVariantFactory.create_batch(3, variant_set=self.variant_set)
+        self.extra_anno = [
+            ExtraAnnoFactory(
+                release=small_vars[0].release,
+                chromosome=small_vars[0].chromosome,
+                start=small_vars[0].start,
+                end=small_vars[0].end,
+                bin=small_vars[0].bin,
+                reference=small_vars[0].reference,
+                alternative=small_vars[0].alternative,
+                anno_data=[9],
+            ),
+            ExtraAnnoFactory(
+                release=small_vars[2].release,
+                chromosome=small_vars[2].chromosome,
+                start=small_vars[2].start,
+                end=small_vars[2].end,
+                bin=small_vars[2].bin,
+                reference=small_vars[2].reference,
+                alternative=small_vars[2].alternative,
+                anno_data=[10],
+            ),
+        ]
+        self.extra_anno_field = ExtraAnnoFieldFactory()
+
+    def _find_table_column_names(self):
+        table = self.selenium.find_element_by_id("table-config")
+        row_header = table.find_element_by_xpath('//*[@id="main"]/thead/tr[2]')
+        row_header_items = row_header.find_elements_by_tag_name("th")
+        row_header_values = [i.text for i in row_header_items]
+        return table, row_header_items, row_header_values
+
+    @skipIf(SKIP_SELENIUM, SKIP_SELENIUM_MESSAGE)
+    def test_variant_filter_case_display_results_extra_anno(self):
+        """Test if submitting the filter yields the expected results."""
+        # login
+        self.compile_url_and_login(
+            {"project": self.case.project.sodar_uuid, "case": self.case.sodar_uuid}
+        )
+        self._disable_filters("case")
+        # hit submit button
+        self.selenium.find_element_by_id("submitFilter").click()
+        # wait for redirect
+        WebDriverWait(self.selenium, self.wait_time).until(
+            ec.presence_of_element_located((By.CLASS_NAME, "variant-row"))
+        )
+
+        # only Effect is on, everythign else (extra anno) off
+        bt = self.selenium.find_element_by_xpath(
+            '//*[@id="resultsTable"]/div[2]/div/div[4]/div/button'
+        )
+        self.assertTrue(bt.get_attribute("title"), "Effect")
+
+        bt.click()
+        columns_elements = self.selenium.find_element_by_xpath(
+            '//*[@id="resultsTable"]/div[2]/div/div[4]/div/div'
+        )
+
+        # any extra_anno_field missing in Columns
+        columns_items = columns_elements.find_elements_by_tag_name("li")
+        columns_values = [i.text for i in columns_items]
+        self.assertTrue(self.extra_anno_field.label in columns_values)
+
+        # click only extra anno
+        for extra_anno_columns_item in columns_items:
+            if extra_anno_columns_item.text == self.extra_anno_field.label:
+                extra_anno_columns_item.click()
+
+        # find extra anno column header in the table
+        table, row_header_items, row_header_values = self._find_table_column_names()
+        self.assertTrue(self.extra_anno_field.label in row_header_values)
+
+        # get extra anno header element
+        extra_anno_table_header = [
+            i for i in row_header_items if i.text == self.extra_anno_field.label
+        ][0]
+
+        # check extra anno values
+        row_extra_anno_items = table.find_elements_by_class_name("effect-change-extra_annos-1")
+        # remove header name, missing values and empty row value from extra anno values
+        remove_from_extra_anno_values = [self.extra_anno_field.label, "-", ""]
+        row_extra_anno_values = [
+            float(i.text)
+            for i in row_extra_anno_items
+            if i.text not in remove_from_extra_anno_values
+        ]
+        # click sort extra anno and see if sorted
+        extra_anno_table_header.click()
+        row_extra_anno_items_sorted = table.find_elements_by_class_name(
+            "effect-change-extra_annos-1"
+        )
+        row_extra_anno_values_sorted = [
+            float(i.text)
+            for i in row_extra_anno_items_sorted
+            if i.text not in remove_from_extra_anno_values
+        ]
+
+        row_extra_anno_values.sort()
+        self.assertTrue(row_extra_anno_values, row_extra_anno_values_sorted)
+
+        # against the database
+        row_extra_anno_values_db = [float(i.anno_data[0]) for i in self.extra_anno]
+        row_extra_anno_values_db.sort()
+        self.assertTrue(row_extra_anno_values_db, row_extra_anno_values_sorted)
+
+        # click to hide extra anno
+        bt.click()
+        extra_anno_columns_item.click()
+        # extra anno column header hidden in the table
+        table, row_header_items, row_header_values = self._find_table_column_names()
+        self.assertFalse(self.extra_anno_field.label in row_header_values)

variants/views.py

@@ -47,7 +47,7 @@
 from bgjobs.views import DEFAULT_PAGINATION as BGJOBS_DEFAULT_PAGINATION
 from clinvar.models import Clinvar
 from cohorts.models import Cohort
-from extra_annos.views import ExtraAnnosMixin
+from extra_annos.views import ExtraAnnosMixin, ExtraAnnoField
 from frequencies.models import MT_DB_INFO
 from geneinfo.views import get_gene_infos
 from geneinfo.models import (
@@ -2449,6 +2449,8 @@ def get_context_data(self, *args, **kwargs):
                 else:
                     hpoterms[hpo] = "unknown term"
 
+        extra_annos_header = [field.label for field in list(ExtraAnnoField.objects.all())]
+
         genomebuild = "GRCh37"
         if rows:
             genomebuild = rows[0]["release"]
@@ -2459,6 +2461,7 @@ def get_context_data(self, *args, **kwargs):
                 "user": self.request.user,
                 "case": filter_job.smallvariantquery.case,
                 "result_rows": rows,
+                "result_extra_annos_header": extra_annos_header,
                 "result_count": num_results,
                 "elapsed_seconds": elapsed.total_seconds(),
                 "database": filter_job.smallvariantquery.query_settings.get(