Merge branch 'main' into python-3.10-in-ci

HISTORY.rst

@@ -39,7 +39,8 @@ End-User Summary
 - Improving performance of case listing (#304)
 - Adding shortcut buttons to phenotype annotation (#289)
 - Fixing issue with multiple added variants (#283)
-- Implementing several usability improvements for clinvar submission editor (#286).
+- Implementing several usability improvements for clinvar submission editor (#286)
+- Make clinvar UI work with many annotations (#302)
 
 Full Change List
 ================
@@ -81,6 +82,7 @@ Full Change List
 - Adding shortcut buttons to phenotype annotation (#289)
 - Fixing issue with multiple added variants (#283)
 - Implementing several usability improvements for clinvar submission editor (#286).
+- Make clinvar UI work with many annotations by making it load them lazily for one case at a time (#302)
 - Adding CI builds for Python 3.10 in Github actions, bumping numpy/pandas dependencies.
   Dropping support for Python 3.7.
 

clinvar_export/tests/test_permissions_ajax.py

@@ -5,7 +5,7 @@
 from projectroles.tests.test_permissions_api import TestProjectAPIPermissionBase
 
 from geneinfo.tests.factories import HpoNameFactory, HpoFactory
-from variants.tests.factories import SmallVariantCommentFactory
+from variants.tests.factories import CaseFactory, SmallVariantCommentFactory
 from .factories import (
     SubmissionSetFactory,
     SubmissionFactory,
@@ -18,6 +18,7 @@
     SubmittingOrgFactory,
     SubmitterFactory,
 )
+from ..models import Family
 
 
 class TestOrganisationAjaxViews(TestProjectAPIPermissionBase):
@@ -681,10 +682,16 @@ def test(self):
 class TestAnnotatedSmallVariantsAjaxViews(TestProjectAPIPermissionBase):
     """Permission tests for the AJAX views for querying user-annotated small variants."""
 
+    def setUp(self):
+        super().setUp()
+        self.case = CaseFactory(project=self.project)
+        self.family = Family.objects.get_or_create_in_project(project=self.project, case=self.case)
+
     def test(self):
         SmallVariantCommentFactory()
         url = reverse(
-            "clinvar_export:user-annotations", kwargs={"project": self.project.sodar_uuid,},
+            "clinvar_export:user-annotations",
+            kwargs={"project": self.project.sodar_uuid, "family": self.family.sodar_uuid},
         )
         good_users = [
             self.superuser,

clinvar_export/tests/test_views_ajax.py

@@ -876,6 +876,7 @@ class TestAnnotatedSmallVariantsAjaxViews(TestProjectAPIPermissionBase):
     def setUp(self):
         super().setUp()
         self.case = CaseFactory(project=self.project)
+        self.family = Family.objects.get_or_create_in_project(project=self.project, case=self.case)
         self.small_variant = SmallVariantFactory(case_id=self.case.id)
         kwargs = {
             key: getattr(self.small_variant, key)
@@ -887,7 +888,8 @@ def setUp(self):
 
     def test_query(self):
         url = reverse(
-            "clinvar_export:user-annotations", kwargs={"project": self.project.sodar_uuid,},
+            "clinvar_export:user-annotations",
+            kwargs={"project": self.project.sodar_uuid, "family": self.family.sodar_uuid},
         )
         with self.login(self.contributor_as.user):
             response = self.client.get(url)

clinvar_export/urls.py

@@ -102,7 +102,7 @@
         name="query-hpo-term",
     ),
     url(
-        regex=r"^ajax/(?P<project>[0-9a-f-]+)/user-annotations/?$",
+        regex=r"^ajax/(?P<project>[0-9a-f-]+)/user-annotations/(?P<family>[0-9a-f-]+)/?$",
         view=views_ajax.AnnotatedSmallVariantsApiView.as_view(),
         name="user-annotations",
     ),

clinvar_export/views_ajax.py

@@ -31,6 +31,7 @@
 from variants.queries import SmallVariantUserAnnotationQuery
 from .clinvar_xml import SubmissionXmlGenerator, XSD_URL_1_7
 from .models import (
+    Case,
     SubmissionSet,
     Submission,
     Organisation,
@@ -424,8 +425,9 @@ class AnnotatedSmallVariantsApiView(
     permission_required = "clinvar_export.view_data"
     allowed_methods = ("GET",)
 
-    def get(self, *_args, **_kwargs):
+    def get(self, *_args, **kwargs):
+        family = Family.objects.get(project=self.get_project(), sodar_uuid=kwargs.get("family"))
         serializer = AnnotatedSmallVariantsSerializer(
-            SmallVariantUserAnnotationQuery(get_engine()).run(project=self.get_project())
+            SmallVariantUserAnnotationQuery(get_engine()).run(case=family.case)
         )
         return Response(serializer.data)

varfish/static/js/variant_validator.js

@@ -4,7 +4,7 @@ function queryVariantValidatorApi(obj, release, chromosome, start, reference, al
     let box = button.closest('.modal-content').find('.variant-validator-results');
     button.attr('disabled', true);
     icon.attr('src', '/icons/fa-solid/refresh.svg').addClass('spin');
-    box.html('<div class="text-center"><i class="iconify spin" height="5x" data-icon="fa-solid:spinner"></i></div>');
+    box.html('<div class="text-center"><i class="iconify spin" data-icon="fa-solid:spinner"></i></div>');
     $.ajax({
         type: 'POST',
         url: variant_validator_url,

varfish/vueapp/src/api/clinvarExport.js

@@ -203,8 +203,8 @@ export default {
     return await apiDelete('submitting_org', submittingOrg.sodar_uuid, appContext)
   },
 
-  async getUserAnnotations (appContext) {
-    const response = await fetch(`${appContext.baseUrl}/user-annotations/`, {
+  async getUserAnnotations (appContext, familyUuid) {
+    const response = await fetch(`${appContext.baseUrl}/user-annotations/${familyUuid}`, {
       method: 'GET',
       credentials: 'same-origin',
       headers: {

varfish/vueapp/src/components/SubmissionList.vue

@@ -47,12 +47,27 @@
       </p>
       <ul class="list-group mb-3">
         <li
-          class="list-group-item list-group-item-action list-group-item-dark"
+          class="list-group-item list-group-item-action list-group-item-dark pl-2 pr-2"
         >
           <div class="form-inline">
+            <div class="form-group mr-3" style="width: 200px; max-width: 200px;">
+               <multiselect
+                 id="input-family"
+                 placeholder="Select Case"
+                 @input="fetchRawModalUserAnnotations()"
+                 @close="fetchRawModalUserAnnotations()"
+                 @select="fetchRawModalUserAnnotations()"
+                 @remove="fetchRawModalUserAnnotations()"
+                 :options="familyUuids"
+                 :customLabel="getFamilyLabel"
+                 v-model="familyUuid"
+                 selectLabel=""
+                 deselectLabel=""
+               ></multiselect>
+            </div>
             <div class="form-group">
               <span
-                :class="{ 'cursor-pointer badge badge-light': !modalIncludeAll, 'cursor-pointer ml-2 badge badge-success': modalIncludeAll }"
+                :class="{ 'cursor-pointer ml-2 badge badge-light': !modalIncludeAll, 'cursor-pointer ml-2 badge badge-success': modalIncludeAll }"
                 @click="toggleData('modalIncludeAll')"
               >
                 all: <i :class="{ 'fa fa-check-circle': modalIncludeAll, 'fa fa-times-circle': !modalIncludeAll }"></i>
@@ -97,15 +112,12 @@
                 P5 <i :class="{ 'fa fa-check-circle': modalIncludeAcmg5, 'fa fa-times-circle': !modalIncludeAcmg5 }"></i>
               </span>
             </div>
-            <div class="form-group ml-3">
-              <b-form-input v-model="individualFilter" placeholder="Sample ID" size="sm"></b-form-input>
-            </div>
-            <div class="form-group ml-3">
+            <div class="form-group ml-3" style="height: 43px;">
               <b-form-checkbox v-model="onlyAddAffected">
                 affecteds only
               </b-form-checkbox>
             </div>
-            <div class="form-group ml-3">
+            <div class="form-group ml-auto">
               <b-button @click="onCreateSubmissionClicked()" size="sm" variant="primary">
                 <i class="iconify" data-icon="fa-solid:plus"></i>
               </b-button>
@@ -175,25 +187,53 @@
         </li>
         <li
           class="list-group-item list-group-item-action text-muted font-italic text-center"
-          v-if="modalUserAnnotations.length === 0"
+          v-if="(modalUserAnnotations.length === 0)"
         >
-          There is no matching user annotation for variants in this project.
+          <span v-if="(familyUuid === '')">
+            <span>
+              Please select a case to display the variant user annotations.
+            </span>
+          </span>
+          <span v-else-if="loadingVariants">
+            <span>
+              Fetching variants from case from server...
+            </span>
+          </span>
+          <span v-else-if="fetchError">
+            <span>
+              Ouch, an error occured while fetching the variants!
+            </span>
+          </span>
+          <span v-else-if="rawModalUserAnnotationsCount > 0">
+            <span>
+              None of the {{ rawModalUserAnnotationsCount }} user annotations matched your selection criteria.
+            </span>
+          </span>
+          <span v-else>
+            <span>
+              There is no matching user annotation for variants in this project.
+            </span>
+          </span>
         </li>
       </ul>
     </b-modal>
   </div>
 </template>
 
 <script>
+import Vue from 'vue'
 import { mapActions, mapState } from 'vuex'
 import draggable from 'vuedraggable'
 import SubmissionEditor from './SubmissionEditor'
 import { isDiseaseTerm, getSubmissionLabel, validConfirmed, removeItemAll, HPO_INHERITANCE_MODE, HPO_AGE_OF_ONSET } from '@/helpers'
+import Multiselect from 'vue-multiselect'
+import clinvarExport from '../api/clinvarExport'
 
 export default {
-  components: { draggable, SubmissionEditor },
+  components: { draggable, SubmissionEditor, Multiselect },
   data () {
     return {
+      familyUuid: '',
       modalIncludeAll: false,
       modalIncludeComments: false,
       modalIncludeCandidates: true,
@@ -203,20 +243,28 @@ export default {
       modalIncludeAcmg5: true,
       individualFilter: '',
       onlyAddAffected: true,
+      loadingVariants: false,
+      fetchError: false,
+      rawModalUserAnnotations: null,
+      rawModalUserAnnotationsCount: 0,
       selectedSmallVariants: []
     }
   },
   computed: {
     ...mapState({
+      appContext: state => state.clinvarExport.appContext,
+      families: state => state.clinvarExport.families,
       individuals: state => state.clinvarExport.individuals,
       submissionIndividuals: state => state.clinvarExport.submissionIndividuals,
       submissions: state => state.clinvarExport.submissions,
       currentSubmissionSet: state => state.clinvarExport.currentSubmissionSet,
       currentSubmission: state => state.clinvarExport.currentSubmission,
-      userAnnotations: state => state.clinvarExport.userAnnotations,
       assertionMethods: state => state.clinvarExport.assertionMethods
     }),
 
+    familyUuids: function () {
+      return Array.from(Object.values(this.families), f => f.sodar_uuid)
+    },
     submissionList: {
       get () {
         const lst = this.currentSubmissionSet.submissions.map(k => this.submissions[k])
@@ -229,11 +277,14 @@ export default {
     },
 
     /**
-     * Return data to display in the annotated variant modal.
+     * Return filtered data to display in the annotated variant modal.
      */
     modalUserAnnotations () {
       const c = 300_000_000 // longer than longest chromosome
-      const ua = this.userAnnotations
+      const ua = this.rawModalUserAnnotations
+      if (!ua) {
+        return []
+      }
 
       const smallVariants = Object.values(ua.smallVariants)
         .map(smallVar => {
@@ -277,13 +328,101 @@ export default {
       'applySubmissionListSortOrder'
     ]),
 
+    /**
+     * Fetch model user annotations.
+     */
+    fetchRawModalUserAnnotations () {
+      if (this.familyUuid) {
+        this.fetchError = false
+        this.loadingVariants = true
+        Vue.set(this, 'rawModalUserAnnotations', null)
+        Vue.set(this, 'rawModalUserAnnotationsCount', 0)
+
+        clinvarExport
+          .getUserAnnotations(this.appContext, this.familyUuid)
+          .then((res) => {
+            this.loadingVariants = false
+            this.fetchError = false
+
+            const getVariantId = (obj) => {
+              return `${obj.release}-${obj.chromosome}-${obj.start}-${obj.reference}-${obj.alternative}`
+            }
+
+            const collect = (arr) => {
+              const result = {}
+              for (const obj of arr) {
+                if (getVariantId(obj) in result) {
+                  result[getVariantId(obj)].push(obj)
+                } else {
+                  Vue.set(result, getVariantId(obj), [obj])
+                }
+              }
+              for (const arr of Object.values(result)) {
+                arr.sort((a, b) => (a.case_name < b.case_name) ? -1 : 1)
+              }
+              return result
+            }
+
+            const smallVariants = {}
+            for (const smallVar of res.small_variants) {
+              if (getVariantId(smallVar) in smallVariants) {
+                smallVariants[getVariantId(smallVar)].caseNames.push(smallVar.case_name)
+                Vue.set(
+                  smallVariants[getVariantId(smallVar)],
+                  'genotype',
+                  {
+                    ...smallVariants[getVariantId(smallVar)].genotype,
+                    ...smallVar.genotype
+                  }
+                )
+              } else {
+                Vue.set(
+                  smallVariants,
+                  getVariantId(smallVar),
+                  {
+                    ...smallVar,
+                    caseNames: [smallVar.case_name],
+                    variantId: getVariantId(smallVar)
+                  }
+                )
+                Vue.delete(smallVariants[getVariantId(smallVar)], 'case_name')
+              }
+            }
+            for (const arr of Object.values(smallVariants)) {
+              arr.caseNames = [...new Set(arr.caseNames)]
+              arr.caseNames.sort((a, b) => (a.case_name < b.case_name) ? -1 : 1)
+            }
+
+            Vue.set(
+              this,
+              'rawModalUserAnnotations',
+              {
+                smallVariants,
+                smallVariantFlags: collect(res.small_variant_flags),
+                smallVariantComments: collect(res.small_variant_comments),
+                acmgCriteriaRating: collect(res.acmg_criteria_rating)
+              }
+            )
+            Vue.set(this, 'rawModalUserAnnotationsCount', Object.keys(smallVariants).length)
+          })
+          .catch((error) => {
+            this.loadingVariants = false
+            this.fetchError = true
+            console.error(error)
+          })
+      }
+    },
+
     toggleData (name) {
       this.$set(this, name, !this[name])
     },
 
     getSubmissionLabel,
     validConfirmed,
 
+    getFamilyLabel (familyUuid) {
+      return this.families[familyUuid].case_name
+    },
     getSubmissionIndividualsCount (item) {
       return item.submission_individuals.length
     },
@@ -304,7 +443,7 @@ export default {
       return `${item.release}-${item.chromosome}-${item.start}-${item.reference}-${item.alternative}`
     },
     getVariantLabel (item) {
-      return `${item.refseq_gene_symbol}:${item.refseq_hgvs_p || '<none>'}`
+      return `${item.refseq_gene_symbol}:${item.refseq_hgvs_p || '--none--'}`
     },
     getVariantExtraLabel (item) {
       if (!item) {