Dev

.github/workflows/unit-tests.yml

@@ -21,7 +21,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.9]
+        python-version: [3.9, 3.11]
 
     services:
       # Label used to access the service container

clickhouse_search/managers.py

@@ -1117,11 +1117,7 @@ def search_padded_interval(self, chrom, pos, padding):
         return self.filter(interval_q).result_values()
 
     @staticmethod
-    def _interval_query(chrom, start, end, offset=None, **kwargs):
-        if offset:
-            offset_pos = int((end - start) * offset)
-            start = max(start - offset_pos, MIN_POS)
-            end = min(end + offset_pos, MAX_POS)
+    def _interval_query(chrom, start, end, **kwargs):
         return Q(xpos__range=(get_xpos(chrom, start), get_xpos(chrom, end)))
 
     def _filter_seqr_frequency(self, entries, ac=None, hh=None, **kwargs):

clickhouse_search/search.py

@@ -189,15 +189,6 @@ def _get_comp_het_results_queryset(annotations_cls, primary_q, secondary_q, num_
                     {2: ('arrayIntersect(primary_familyGuids, secondary_familyGuids)', 'has({value}, {field})')},
                 ]),
             }
-        elif results.has_annotation('secondary_genotypes'):
-            genotype_expressions = {
-                'primary_familyGenotypes': ArrayFilter('primary_familyGenotypes', conditions=[
-                    {1: ('secondary_familyGuids', 'has({value}, {field})')},
-                ]),
-                'secondary_genotypes': ArrayFilter('secondary_genotypes', conditions=[
-                    {2: (None, 'arrayExists(g -> g.1 = {field}, primary_familyGenotypes)')},
-                ]),
-            }
         else:
             genotype_expressions = {
                 'primary_familyGenotypes': ArrayFilter('primary_familyGenotypes', conditions=[
@@ -699,10 +690,11 @@ def delete_clickhouse_family(project, family_guid, dataset_type, sample_type=Non
     return f'Clickhouse does not support deleting individual families from project. Manually delete {dataset_type} data for {family_guid} in project {project.guid}'
 
 
+SV_DATASET_TYPES = {
+    Sample.SAMPLE_TYPE_WGS: Sample.DATASET_TYPE_SV_CALLS,
+    Sample.SAMPLE_TYPE_WES: 'GCNV',
+}
 def _clickhouse_dataset_type(dataset_type, sample_type):
     if dataset_type == Sample.DATASET_TYPE_SV_CALLS:
-        if not sample_type:
-            raise ValueError('sample_type is required when dataset_type is SV')
-        if sample_type == Sample.SAMPLE_TYPE_WES:
-            return 'GCNV'
+        dataset_type = SV_DATASET_TYPES[sample_type]
     return dataset_type

clickhouse_search/search_tests.py

@@ -599,7 +599,7 @@ def test_location_search(self):
             [SV_VARIANT1, SV_VARIANT2], locus=sv_locus,
         )
         self._assert_expected_search(
-            [SV_VARIANT1], locus={'rawItems': 'chr1:9292894-9369532'}
+            [SV_VARIANT1], locus={'rawItems': 'chr1:9297894-9369732%10'}
         )
 
         self._reset_search_families()

requirements-dev.in

@@ -1,5 +1,5 @@
 -c requirements.txt # use the generated reqs as a constraint
-coverage<5.2
+coverage<6.6
 django-compressor
 django-debug-toolbar      # https://github.com/jazzband/django-debug-toolbar
 mock                      # mock objects for unit tests

requirements-dev.txt

@@ -20,7 +20,7 @@ charset-normalizer==3.0.1
     #   requests
 click==8.1.3
     # via pip-tools
-coverage==5.1
+coverage==6.5
     # via -r requirements-dev.in
 django==4.2.24
     # via

seqr/management/commands/check_for_new_samples_from_pipeline.py

@@ -36,7 +36,6 @@
     'GCNV': f'{Sample.DATASET_TYPE_SV_CALLS}_WES',
     Sample.DATASET_TYPE_SV_CALLS: f'{Sample.DATASET_TYPE_SV_CALLS}_WGS',
 }
-USER_EMAIL = 'manage_command'
 RELATEDNESS_CHECK_NAME = 'relatedness_check'
 
 PDO_COPY_FIELDS = [
@@ -255,7 +254,7 @@ def _load_new_samples(cls, metadata_path, genome_version, dataset_type, run_vers
         # Reload saved variant JSON
         update_projects_saved_variant_json([
             (project.id, project.guid, project.name, project.genome_version, families) for project, families in families_by_project.items()
-        ], user_email=USER_EMAIL, dataset_type=dataset_type, update_function=cls._update_project_saved_variant_genotypes, samples=updated_samples, clickhouse_dataset_type=clickhouse_dataset_type)
+        ], dataset_type=dataset_type, update_function=cls._update_project_saved_variant_genotypes, samples=updated_samples, clickhouse_dataset_type=clickhouse_dataset_type)
 
     @classmethod
     def _is_internal_project(cls, project):
@@ -390,7 +389,7 @@ def _update_individuals_sample_qc(cls, sample_type, family_guids, sample_qc_map)
             )
 
     @classmethod
-    def _update_project_saved_variant_genotypes(cls, project_id, genome_version, user_email, family_guids, project_guid, samples=None, clickhouse_dataset_type=None, **kwargs):
+    def _update_project_saved_variant_genotypes(cls, project_id, genome_version, family_guids, project_guid, samples=None, clickhouse_dataset_type=None, **kwargs):
         updates = {}
         for family_guid in family_guids:
             variant_models_by_key = {

seqr/management/commands/reload_saved_variant_json.py

@@ -30,5 +30,5 @@ def handle(self, *args, **options):
 
         family_ids = [family_guid] if family_guid else None
         project_list = [(*project, family_ids) for project in projects.order_by('id').values_list('id', 'guid', 'name', 'genome_version')]
-        update_projects_saved_variant_json(project_list, user_email='manage_command', update_function=update_project_saved_variant_json)
+        update_projects_saved_variant_json(project_list, update_function=update_project_saved_variant_json)
         logger.info("Done")

seqr/utils/communication_utils.py

@@ -25,10 +25,9 @@ def _post_to_slack(channel, message):
         return None
 
     slack = Slacker(SLACK_TOKEN)
-    response = slack.chat.post_message(
+    slack.chat.post_message(
         channel, message, as_user=False, icon_emoji=":beaker:", username="Beaker (engineering-minion)",
     )
-    return response.raw
 
 
 def send_welcome_email(user, referrer):

seqr/utils/search/elasticsearch/es_utils_tests.py

@@ -39,6 +39,11 @@ def replace_json(self, url, *args, **kwargs):
     def call_request_json(self, index=-1):
         return json.loads(self.calls[index].request.body)
 
+    def activate(self, func):
+        def wrapper(*args, **kwargs):
+            with self:
+                return func(*args, **kwargs)
+        return wrapper
 
 urllib3_responses = Urllib3Responses()
 

seqr/utils/search/utils.py

@@ -560,14 +560,6 @@ def _filter_inheritance_family_samples(samples, inheritance_filter):
     ]
 
 
-def _format_interval(chrom=None, start=None, end=None, offset=None, **kwargs):
-    if offset:
-        offset_pos = int((end - start) * offset)
-        start = max(start - offset_pos, MIN_POS)
-        end = min(end + offset_pos, MAX_POS)
-    return [chrom, start, end]
-
-
 LIFTOVERS = {
     GENOME_VERSION_GRCh38: None,
     GENOME_VERSION_GRCh37: None,

seqr/views/apis/data_manager_api_tests.py

@@ -1274,9 +1274,6 @@ def _test_expected_vcf_responses(self, response, url):
         response = self.client.get(url, content_type='application/json')
         self.assertEqual(response.status_code, 200)
 
-    def _assert_expected_read_vcf_header_subprocess_calls(self, body):
-        return True
-
     def test_validate_callset(self):
         url = reverse(validate_callset)
         self.check_pm_login(url)

seqr/views/apis/dataset_api_tests.py

@@ -451,12 +451,6 @@ class LocalDatasetAPITest(AuthenticationTestCase, DatasetAPITest):
     fixtures = ['users', '1kg_project']
 
 
-def assert_no_anvil_calls(self):
-    self.mock_list_workspaces.assert_not_called()
-    self.mock_get_ws_access_level.assert_not_called()
-    self.assert_no_extra_anvil_calls()
-
-
 # Test for permissions from AnVIL only
 class AnvilDatasetAPITest(AnvilAuthenticationTestCase, DatasetAPITest):
     fixtures = ['users', 'social_auth', '1kg_project']

seqr/views/apis/project_api_tests.py

@@ -390,7 +390,7 @@ def test_project_families(self):
         gene_ids = self._assert_expected_project_families(url, response_keys)
         self.assertSetEqual(gene_ids, {'ENSG00000135953', 'ENSG00000240361'})
 
-    def _assert_expected_project_families(self, url, response_keys):
+    def _assert_expected_project_families(self, url, response_keys, no_discovery_tags=False):
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200)
 
@@ -436,12 +436,14 @@ def _assert_expected_project_families(self, url, response_keys):
 
         self.assertListEqual(family_3['discoveryTags'], [])
         self.assertListEqual(empty_family['discoveryTags'], [])
-        self.assertListEqual(family_1['discoveryTags'], [{
+        family_1_tags = [] if no_discovery_tags else [{
             'transcripts': {'ENSG00000135953': [mock.ANY, mock.ANY, mock.ANY, mock.ANY, mock.ANY, mock.ANY]},
             'mainTranscriptId': 'ENST00000258436',
             'selectedMainTranscriptId': None,
-        }])
-        self.assertListEqual(response_json['familiesByGuid']['F000002_2']['discoveryTags'], [self.DISCOVERY_TAG])
+        }]
+        self.assertListEqual(family_1['discoveryTags'], family_1_tags)
+        family_2_tags = [] if no_discovery_tags else [self.DISCOVERY_TAG]
+        self.assertListEqual(response_json['familiesByGuid']['F000002_2']['discoveryTags'], family_2_tags)
         no_discovery_families = set(response_json['familiesByGuid'].keys()) - {'F000001_1', 'F000002_2'}
         self.assertSetEqual({
             len(response_json['familiesByGuid'][family_guid]['discoveryTags']) for family_guid in no_discovery_families
@@ -802,8 +804,8 @@ def _assert_expected_project_families(self, *args, **kwargs):
         self.reset_logs()
         connections['clickhouse'].close()
         self.DISCOVERY_TAG = {**DISCOVERY_TAG, 'transcripts': {}}
-        no_clickhouse_gene_ids = super()._assert_expected_project_families(*args, **kwargs)
-        self.assertSetEqual(no_clickhouse_gene_ids, {'ENSG00000135953'})
+        no_clickhouse_gene_ids = super()._assert_expected_project_families(*args, **kwargs, no_discovery_tags=True)
+        self.assertSetEqual(no_clickhouse_gene_ids, set())
         self.assert_json_logs(None, [
             ("Error loading discovery genes from clickhouse: An error occurred in the current transaction. You can't execute queries until the end of the 'atomic' block.", {
                 'severity': 'ERROR',

seqr/views/apis/variant_search_api_tests.py

@@ -327,20 +327,17 @@ def _assert_expected_rnaseq_response(self, response_json):
               'start': 132885746, 'strand': '*', 'tissueType': 'F', 'type': 'psi5', 'deltaIntronJaccardIndex': 12.34}]
         )
 
-    def _assert_expected_results_family_context(self, response_json, locus_list_detail=False, skip_gene_context=False):
-        if not skip_gene_context:
-            self._assert_expected_results_context(response_json, locus_list_detail=locus_list_detail)
+    def _assert_expected_results_family_context(self, response_json, locus_list_detail=False):
+        self._assert_expected_results_context(response_json, locus_list_detail=locus_list_detail)
 
         family_fields = {'individualGuids'}
         family_fields.update(FAMILY_FIELDS)
         if len(response_json['familiesByGuid']) > 1:
             self.assertSetEqual(set(response_json['familiesByGuid']['F000002_2'].keys()), family_fields)
 
-        if not skip_gene_context:
-            family_fields.add('tpmGenes')
+        family_fields.add('tpmGenes')
         self.assertSetEqual(set(response_json['familiesByGuid']['F000001_1'].keys()), family_fields)
-        if not skip_gene_context:
-            self.assertSetEqual(set(response_json['familiesByGuid']['F000001_1']['tpmGenes']), {'ENSG00000227232'})
+        self.assertSetEqual(set(response_json['familiesByGuid']['F000001_1']['tpmGenes']), {'ENSG00000227232'})
 
         self.assertEqual(len(response_json['individualsByGuid']), len(response_json['familiesByGuid'])*3)
         individual_fields = {'igvSampleGuids'}
@@ -353,8 +350,7 @@ def _assert_expected_results_family_context(self, response_json, locus_list_deta
         self.assertEqual(len(response_json['familyNotesByGuid']), 3)
         self.assertSetEqual(set(response_json['familyNotesByGuid']['FAN000001_1'].keys()), FAMILY_NOTE_FIELDS)
 
-        if not skip_gene_context:
-            self._assert_expected_rnaseq_response(response_json)
+        self._assert_expected_rnaseq_response(response_json)
 
     def _assert_expected_results_context(self, response_json, has_pa_detail=True, locus_list_detail=False, rnaseq=True):
         gene_fields = {'locusListGuids'}
@@ -869,7 +865,7 @@ def test_query_single_variant(self, mock_get_variant):
         self.assertEqual(response.status_code, 400)
         self.assertEqual(response.json()['error'], 'Variant not found')
 
-    def _assert_expected_single_variant_results_context(self, response_json, omit_fields=None, no_metadata=False, **expected_response):
+    def _assert_expected_single_variant_results_context(self, response_json, omit_fields=None, **expected_response):
         omit_fields = {'search', *(omit_fields or [])}
 
         expected_search_response = deepcopy({**self.EXPECTED_SEARCH_RESPONSE, **EXPECTED_SEARCH_FAMILY_CONTEXT})
@@ -879,25 +875,18 @@ def _assert_expected_single_variant_results_context(self, response_json, omit_fi
         })
         for k in omit_fields:
             expected_search_response.pop(k)
-        if no_metadata:
-            expected_search_response.update({k: {} for k in {
-                'savedVariantsByGuid', 'variantTagsByGuid', 'variantFunctionalDataByGuid', 'genesById',
-                'rnaSeqData', 'phenotypeGeneScores', 'mmeSubmissionsByGuid'
-            }})
-            expected_search_response.pop('transcriptsById', None)
-        else:
-            expected_search_response['savedVariantsByGuid'].pop('SV0000002_1248367227_r0390_100')
-            expected_search_response['variantTagsByGuid'] = {
-                k: EXPECTED_SEARCH_RESPONSE['variantTagsByGuid'][k]
-                for k in {'VT1708633_2103343353_r0390_100', 'VT1726961_2103343353_r0390_100'}
-            }
+        expected_search_response['savedVariantsByGuid'].pop('SV0000002_1248367227_r0390_100')
+        expected_search_response['variantTagsByGuid'] = {
+            k: EXPECTED_SEARCH_RESPONSE['variantTagsByGuid'][k]
+            for k in {'VT1708633_2103343353_r0390_100', 'VT1726961_2103343353_r0390_100'}
+        }
         expected_search_response['variantNotesByGuid'] = {}
         expected_search_response['genesById'] = {
             k: v for k, v in expected_search_response['genesById'].items() if k in {'ENSG00000227232', 'ENSG00000268903'}
         }
         self.assertSetEqual(set(response_json.keys()), set(expected_search_response.keys()))
         self.assertDictEqual(response_json, expected_search_response)
-        self._assert_expected_results_family_context(response_json, locus_list_detail=True, skip_gene_context=no_metadata)
+        self._assert_expected_results_family_context(response_json, locus_list_detail=True)
         self.assertSetEqual(set(response_json['projectsByGuid'][PROJECT_GUID].keys()), PROJECT_TAG_TYPE_FIELDS)
         self.assertSetEqual(set(response_json['familiesByGuid'].keys()), {'F000001_1'})