Merge 3c95b29 into c7e99a0

HISTORY.rst

@@ -9,13 +9,18 @@ HEAD (unreleased)
 End-User Summary
 ================
 
+- Allowing to download all users annotation for whole project in one Excel/TSV file.
+- Improving variant annotation overview per case/project and allowing download.
 - Adding "not hom. alt." filter setting.
 - Allowing users to easily copy case UUID by icon in case heading.
 - Fixing bug that made the user icon top right disappear.
 
 Full Change List
 ================
 
+- Allowing to download all users annotation for whole project in one Excel/TSV file.
+- Using SQL Alchemy query instrastructure for per-case/project annotation feature.
+- Removing vendored JS/CSS, using CDN for development and download on Docker build instead.
 - Adding "not hom. alt." filter setting.
 - Improving admin configuration documentation.
 - Extending admin tuning documentation.

beaconsite/tests/test_permissions_ajax.py

@@ -57,7 +57,6 @@ def test_get(self, r_mock):
 
 
 class TestBeaconQueryAjaxView(TestProjectAPIPermissionBase):
-    @skip(reason="missing urlescape in sodar_core, fixed in 0.9.1")
     @requests_mock.Mocker()
     def test_get(self, r_mock):
         _local_site = SiteFactory(role=Site.LOCAL)

clinvar_export/tests/test_permissions_ajax.py

@@ -635,7 +635,6 @@ def test_delete(self):
 class TestQueryOmimAjaxViews(TestProjectAPIPermissionBase):
     """Permission tests for OMIM term AJAX views"""
 
-    @skip(reason="missing urlescape in sodar_core, fixed in 0.9.1")
     def test(self):
         hpo_record = HpoFactory()
         url = (
@@ -649,16 +648,16 @@ def test(self):
             self.delegate_as.user,
             self.contributor_as.user,
             self.guest_as.user,
+            self.user_no_roles,
         ]
-        bad_users = [self.anonymous, self.user_no_roles]
+        bad_users = [self.anonymous]
         self.assert_response(url, good_users, 200, method="GET")
         self.assert_response(url, bad_users, 302, method="GET")  # redirect to login
 
 
 class TestQueryHpoAjaxViews(TestProjectAPIPermissionBase):
     """Permission tests for the AJAX views for querying for HPO terms."""
 
-    @skip(reason="missing urlescape in sodar_core, fixed in 0.9.1")
     def test(self):
         hpo_record = HpoNameFactory()
         url = (
@@ -672,8 +671,9 @@ def test(self):
             self.delegate_as.user,
             self.contributor_as.user,
             self.guest_as.user,
+            self.user_no_roles,
         ]
-        bad_users = [self.anonymous, self.user_no_roles]
+        bad_users = [self.anonymous]
         self.assert_response(url, good_users, 200, method="GET")
         self.assert_response(url, bad_users, 302, method="GET")  # redirect to login
 

clinvar_export/tests/test_views_ajax.py

@@ -833,13 +833,14 @@ def test_query(self):
             response = self.client.get(url)
         self.assertEqual(response.status_code, 200)
         res_json = response.json()
+        lst = [x.strip() for x in hpo_record.name.split(";") if x.strip()]
         expected = jsonmatch.compile(
             {
                 "query": hpo_record.database_id,
                 "result": [
-                    {"term_id": "OMIM:0", "term_name": "Alternative Description"},
-                    {"term_id": "OMIM:0", "term_name": "Disease 0"},
-                    {"term_id": "OMIM:0", "term_name": "Gene Symbol"},
+                    {"term_id": hpo_record.database_id, "term_name": lst[2]},
+                    {"term_id": hpo_record.database_id, "term_name": lst[0]},
+                    {"term_id": hpo_record.database_id, "term_name": lst[1]},
                 ],
             }
         )
@@ -863,7 +864,7 @@ def test_query(self):
         expected = jsonmatch.compile(
             {
                 "query": hpo_name_record.hpo_id,
-                "result": [{"term_id": "HP:0000000", "term_name": "Phenotype 0"}],
+                "result": [{"term_id": hpo_name_record.hpo_id, "term_name": hpo_name_record.name}],
             }
         )
         expected.assert_matches(res_json)

clinvar_export/views_ajax.py

@@ -6,7 +6,6 @@
 import pathlib
 import re
 
-import requests
 from django.contrib.postgres.aggregates import ArrayAgg
 from django.db.models import Q
 from django.http import JsonResponse, HttpResponse

geneinfo/models.py

@@ -122,6 +122,15 @@ class Meta:
         ]
 
 
+def build_entrez_id_to_symbol(entrez_ids):
+    """Helper function that builds a map from Entrez ID to gene symbol for the given Entrez gene IDs."""
+    entrez_ids = list(sorted(set(filter(bool, entrez_ids))))
+    result = {x: x for x in entrez_ids}
+    for hgnc in Hgnc.objects.filter(entrez_id__in=entrez_ids).order_by("entrez_id"):
+        result[hgnc.entrez_id] = hgnc.symbol
+    return result
+
+
 class Mim2geneMedgen(models.Model):
     """Information to translate Entrez ID int OMIM ID."""
 

variants/models.py

@@ -334,6 +334,11 @@ def get_description(self):
             map(str, (self.release, self.chromosome, self.start, self.reference, self.alternative))
         )
 
+    def human_readable(self):
+        return "{}:{}-{:,}-{}-{}".format(
+            self.release, self.chromosome, self.start, self.reference, self.alternative
+        )
+
     def __repr__(self):
         return "-".join(
             map(

variants/templates/variants/case/detail_annotation.html

@@ -11,13 +11,32 @@
       <h4>
         <i class="fa fa-flag"></i>
         Annotated Variants
+
+        <div class="float-right">
+          <div class="dropdown">
+            <button class="btn btn-secondary dropdown-toggle" type="button" data-toggle="dropdown">
+              <i class="fa fa-cloud-download"></i>
+              Download
+            </button>
+            <div class="dropdown-menu">
+              <a class="dropdown-item" href="{% url 'variants:case-download-annotations' project=project.sodar_uuid case=case.sodar_uuid %}?format=xlsx">
+                <i class="fa fa-file-excel-o"></i>
+                Excel Format
+              </a>
+              <a class="dropdown-item" href="{% url 'variants:case-download-annotations' project=project.sodar_uuid case=case.sodar_uuid %}?format=txt">
+                <i class="fa fa-file-text"></i>
+                TSV Format
+              </a>
+            </div>
+          </div>
+        </div>
       </h4>
     </div>
     <table class="table table-striped table-hover">
       <thead>
         <tr>
           <th style="width: 200px;" rowspan="2" class="text-center align-text-top"><div class="sodar-overflow-container sodar-overflow-hover">Variant</div></th>
-          <th style="width: 80px;" rowspan="2" class="text-center align-text-top"><div class="sodar-overflow-container sodar-overflow-hover">Gene(s)</div></th>
+          <th style="width: 80px;" rowspan="2" class="text-center align-text-top"><div class="sodar-overflow-container sodar-overflow-hover">Gene(s) / Effect(s)</div></th>
           <th rowspan="2" class="text-center align-text-top">ACMG Rating</th>
           <th colspan="6" class="text-center border-bottom-0">Flags</th>
           <th rowspan="2" class="text-center align-text-top">Comments</th>
@@ -33,74 +52,77 @@ <h4>
         </tr>
       </thead>
       <tbody>
-        {% for variant, data in commentsflags.items %}
-          <tr id="flags-{{ variant.0 }}-{{ variant.1 }}-{{ variant.2 }}-{{ variant.3 }}">
+        {% for data in commentsflags.values %}
+          <tr id="flags-{{ var_id }}">
             <td class="text-nowrap">
               <div class="sodar-overflow-container sodar-overflow-hover" style="max-width: 200px;">
-                chr{{ variant.0 }}:{{ variant.1|intcomma }}-{{ variant.2 }}-{{ variant.3 }}
+                {{ data.variants.0.human_readable }}
               </div>
             </td>
             <td>
-              {{ data|keyvalue:"genes"|join:", "|default:"-" }}
+              {% for variant in data.variants %}
+                {{ gene_id_to_symbol|keyvalue:variant.refseq_gene_id }}:{{ variant.refseq_hgvs_p|default:variant.refseq_hgvs_c|default:"-" }}{% if not forloop.last %},{% endif %}
+              {% endfor %}
             </td>
             <td class="text-center">
-              {% if data|keyvalue:"acmg_rating" %}
-                <span class="{{ data|keyvalue:"acmg_rating"|keyvalue:"data"|acmg_badge_class2 }}">
-                  {{ data|keyvalue:"acmg_rating"|keyvalue:"class" }}
+              {% if data.acmg_rating %}
+                <span class="{{ data.acmg_rating|acmg_badge_class2 }}">
+                  {{ data.acmg_rating|acmg_classification2 }}
                 </span>
               {% else %}
                 <p class="text-muted font-italic">No ACMG rating.</p>
               {% endif %}
             </td>
-            {% if data|keyvalue:"flags" %}
+            {% if data.flags %}
               <td class="text-center">
-                <i class="fa fa-fw fa-star {% if not data|keyvalue:"flags"|keyvalue:"flag_bookmarked" %}text-muted" style="opacity: 0.3{% endif %}"></i>
-                <i class="fa fa-fw fa-flask {% if not data|keyvalue:"flags"|keyvalue:"flag_for_validation" %}text-muted" style="opacity: 0.3{% endif %}"></i>
-                <i class="fa fa-fw fa-heart {% if not data|keyvalue:"flags"|keyvalue:"flag_candidate" %}text-muted" style="opacity: 0.3{% endif %}"></i>
-                <i class="fa fa-fw fa-flag-checkered {% if not data|keyvalue:"flags"|keyvalue:"flag_final_causative" %}text-muted" style="opacity: 0.3{% endif %}"></i>
-                <i class="fa fa-fw fa-chain-broken {% if not data|keyvalue:"flags"|keyvalue:"flag_no_disease_association" %}text-muted" style="opacity: 0.3{% endif %}"></i>
-                <i class="fa fa-fw fa-thumbs-up {% if not data|keyvalue:"flags"|keyvalue:"flag_segregates" %}text-muted" style="opacity: 0.3{% endif %}"></i>
-                <i class="fa fa-fw fa-thumbs-down {% if not data|keyvalue:"flags"|keyvalue:"flag_doesnt_segregate" %}text-muted" style="opacity: 0.3{% endif %}"></i>
+                <i class="fa fa-fw fa-star {% if not data.flags.flag_bookmarked %}text-muted" style="opacity: 0.3{% endif %}"></i>
+                <i class="fa fa-fw fa-flask {% if not data.flags.flag_for_validation %}text-muted" style="opacity: 0.3{% endif %}"></i>
+                <i class="fa fa-fw fa-heart {% if not data.flags.flag_candidate %}text-muted" style="opacity: 0.3{% endif %}"></i>
+                <i class="fa fa-fw fa-flag-checkered {% if not data.flags.flag_final_causative %}text-muted" style="opacity: 0.3{% endif %}"></i>
+                <i class="fa fa-fw fa-chain-broken {% if not data.flags.flag_no_disease_association %}text-muted" style="opacity: 0.3{% endif %}"></i>
+                <i class="fa fa-fw fa-thumbs-up {% if not data.flags.flag_segregates %}text-muted" style="opacity: 0.3{% endif %}"></i>
+                <i class="fa fa-fw fa-thumbs-down {% if not data.flags.flag_doesnt_segregate %}text-muted" style="opacity: 0.3{% endif %}"></i>
               </td>
               <td class="text-center">
-                <i class="fa fa-fw {{ data|keyvalue:"flags"|keyvalue:"flag_visual"|flag_value_to_fa }} {{ data|keyvalue:"flags"|keyvalue:"flag_visual"|flag_value_to_color }}"></i>
+                <i class="fa fa-fw {{ data.flags.flag_visual|flag_value_to_fa }} {{ data.flags.flag_visual|flag_value_to_color }}"></i>
               </td>
               <td class="text-center">
-                <i class="fa fa-fw {{ data|keyvalue:"flags"|keyvalue:"flag_molecular"|flag_value_to_fa }} {{ data|keyvalue:"flags"|keyvalue:"flag_molecular"|flag_value_to_color }}"></i>
+                <i class="fa fa-fw {{ data.flags.flag_molecular|flag_value_to_fa }} {{ data.flags.flag_molecular|flag_value_to_color }}"></i>
               </td>
               <td class="text-center">
-                <i class="fa fa-fw {{ data|keyvalue:"flags"|keyvalue:"flag_validation"|flag_value_to_fa }} {{ data|keyvalue:"flags"|keyvalue:"flag_validation"|flag_value_to_color }}"></i>
+                <i class="fa fa-fw {{ data.flags.flag_validation|flag_value_to_fa }} {{ data.flags.flag_validation|flag_value_to_color }}"></i>
               </td>
               <td class="text-center">
-                <i class="fa fa-fw {{ data|keyvalue:"flags"|keyvalue:"flag_phenotype_match"|flag_value_to_fa }} {{ data|keyvalue:"flags"|keyvalue:"flag_phenotype_match"|flag_value_to_color }}"></i>
+                <i class="fa fa-fw {{ data.flags.flag_phenotype_match|flag_value_to_fa }} {{ data.flags.flag_phenotype_match|flag_value_to_color }}"></i>
               </td>
               <td class="text-center">
-                <i class="fa fa-fw {{ data|keyvalue:"flags"|keyvalue:"flag_summary"|flag_value_to_fa }} {{ data|keyvalue:"flags"|keyvalue:"flag_summary"|flag_value_to_color }}"></i>
+                <i class="fa fa-fw {{ data.flags.flag_summary|flag_value_to_fa }} {{ data.flags.flag_summary|flag_value_to_color }}"></i>
               </td>
             {% else %}
               <td colspan="6" class="text-center text-muted font-italic">No flags.</td>
             {% endif %}
             <td>
               <ul class="list-group list" style="width: 400px"
-                id="small-variant-comment-{{ variant.0 }}-{{ variant.1 }}-{{ variant.2 }}-{{ variant.3 }}"
+                id="small-variant-comment-{{ data.variants.0.chromosome }}-{{ data.variants.0.start }}-{{ data.variants.0.reference }}-{{ data.variants.0.alternative }}"
                 data-url-submit="{% url 'variants:small-variant-comment-api' project=project.sodar_uuid case=object.sodar_uuid %}"
                 data-url-delete="{% url 'variants:small-variant-comment-delete-api' project=project.sodar_uuid case=object.sodar_uuid %}">
-              {% for comment in data|keyvalue:"comments" %}
-                <li class="list-group-item list-item" id="comment-{{ comment|keyvalue:"sodar_uuid" }}" data-sodar-uuid="{{ comment|keyvalue:"sodar_uuid" }}">
-                  <div id="display-comment-{{ comment|keyvalue:"sodar_uuid" }}">
+              {% for comment in data.comments %}
+                {{ comment }}
+                <li class="list-group-item list-item" id="comment-{{ comment.sodar_uuid }}" data-sodar-uuid="{{ comment.sodar_uuid }}">
+                  <div id="display-comment-{{ comment.sodar_uuid }}">
                     <span class="small text-muted">
-                      <strong>{{ comment|keyvalue:"username" }}</strong>
-                      {{ comment|keyvalue:"date_created"|date:"Y/m/d H:i" }}:
+                      <strong>{{ comment.user.username }}</strong>
+                      {{ comment.date_created|date:"Y/m/d H:i" }}:
                     </span>
-                    <em>{{ comment|keyvalue:"text" }}</em>
-                    {% if comment|keyvalue:"user" == request.user or request.user.is_superuser %}
+                    <em>{{ comment.text }}</em>
+                    {% if comment.user == request.user or request.user.is_superuser %}
                       <a href="#" class="pl-2 text-secondary comment-button-edit"><i class="fa fa-pencil"></i></a>
                       <a href="#" class="pl-2 text-secondary comment-button-delete"><i class="fa fa-times-circle"></i></a>
                     {% endif %}
                   </div>
-                  <div id="edit-comment-{{ comment|keyvalue:"sodar_uuid" }}" style="display: none;">
+                  <div id="edit-comment-{{ comment.sodar_uuid }}" style="display: none;">
                     <form>
-                      <textarea id="text-comment-{{ comment|keyvalue:"sodar_uuid" }}" name="variant_comment" rows="1" cols="40" class="form-control"></textarea>
+                      <textarea id="text-comment-{{ comment.sodar_uuid }}" name="variant_comment" rows="1" cols="40" class="form-control"></textarea>
                       <span class="btn-group d-flex">
                         <button
                             type="button"
@@ -115,7 +137,7 @@ <h4>
                       </span>
                     </form>
                   </div>
-                  <div id="delete-comment-{{ comment|keyvalue:"sodar_uuid" }}" style="display: none;">
+                  <div id="delete-comment-{{ comment.sodar_uuid }}" style="display: none;">
                     <span class="btn-group d-flex">
                       <button
                           type="button"
@@ -132,11 +154,11 @@ <h4>
                 </li>
               {% endfor %}
               </ul>
-              <p id="small-variant-comment-{{ variant.0 }}-{{ variant.1 }}-{{ variant.2 }}-{{ variant.3 }}-empty" class="text-muted font-italic text-center"{% if data|keyvalue:"comments" %} style="display: none;"{% endif %}>No comments.</p>
+              <p id="small-variant-comment-{{ data.variants.0.chromosome }}-{{ data.variants.0.start }}-{{ data.variants.0.reference }}-{{ data.variants.0.alternative }}-empty" class="text-muted font-italic text-center"{% if data.comments %} style="display: none;"{% endif %}>No comments.</p>
             </td>
             <td>
               <a
-                onclick="javascript:$.ajax({url: 'http://127.0.0.1:60151/goto?locus=chr{{ variant.0 }}:{{ variant.1 }}-{{ variant.1 }}'})"
+                onclick="javascript:$.ajax({url: 'http://127.0.0.1:60151/goto?locus=chr{{ data.variants.0.chromosome }}:{{ data.variants.0.start }}-{{ data.variants.0.end }}'})"
                 href="#"
                 class="btn btn-sm btn-secondary">
                 IGV

variants/templates/variants/case_list/annotation.html

@@ -11,6 +11,25 @@
       <h4>
         <i class="fa fa-flag"></i>
         Annotated Variants
+
+        <div class="float-right">
+          <div class="dropdown">
+            <button class="btn btn-secondary dropdown-toggle" type="button" data-toggle="dropdown">
+              <i class="fa fa-cloud-download"></i>
+              Download
+            </button>
+            <div class="dropdown-menu">
+              <a class="dropdown-item" href="{% url 'variants:project-download-annotations' project=project.sodar_uuid %}?format=xlsx">
+                <i class="fa fa-file-excel-o"></i>
+                Excel Format
+              </a>
+              <a class="dropdown-item" href="{% url 'variants:project-download-annotations' project=project.sodar_uuid %}?format=txt">
+                <i class="fa fa-file-text"></i>
+                TSV Format
+              </a>
+            </div>
+          </div>
+        </div>
       </h4>
     </div>
     <table class="table table-striped table-hover">