From 7bd8edf590ca19988558ccc6ff6b406705daf1c8 Mon Sep 17 00:00:00 2001 From: Manuel Holtgrewe Date: Tue, 6 Sep 2022 17:17:26 +0200 Subject: [PATCH] Switch linting from codacy to default python tools (#639) Related-Issue: #639 Closes: #639 Projected-Results-Impact: require-revalidation --- .github/workflows/ci.yml | 19 +- Makefile | 15 + README.rst | 4 - beaconsite/forms.py | 2 +- beaconsite/migrations/0001_initial.py | 5 +- .../migrations/0003_auto_20210304_1052.py | 2 +- beaconsite/models.py | 4 +- beaconsite/models_api.py | 1 - beaconsite/plugins.py | 1 - beaconsite/rules.py | 2 - beaconsite/tests/factories.py | 5 +- beaconsite/tests/test_models.py | 4 +- beaconsite/tests/test_permissions_ajax.py | 13 +- beaconsite/tests/test_permissions_api.py | 15 +- beaconsite/tests/test_views.py | 20 +- beaconsite/tests/test_views_ajax.py | 1 - beaconsite/tests/test_views_api.py | 11 +- beaconsite/urls.py | 27 +- beaconsite/views.py | 44 +- beaconsite/views_ajax.py | 2 +- beaconsite/views_api.py | 18 +- clinvar/clinvar_models.py | 3 +- .../migrations/0002_clinvarpathogenicgenes.py | 3 +- clinvar/migrations/0004_delete_clinvar.py | 8 +- clinvar/migrations/0005_auto_20200611_1719.py | 5 +- .../migrations/0006_clinvarpathogenicgenes.py | 3 +- .../migrations/0007_alter_clinvar_details.py | 5 +- clinvar/migrations/0008_auto_20220829_1132.py | 17 +- clinvar/models.py | 3 +- clinvar_export/admin.py | 8 +- clinvar_export/clinvar_xml.py | 81 +++- clinvar_export/migrations/0001_initial.py | 58 ++- .../migrations/0002_bootstrap_records.py | 10 +- .../migrations/0003_auto_20211129_1443.py | 1 + clinvar_export/models.py | 9 +- clinvar_export/rules.py | 3 +- clinvar_export/serializers.py | 22 +- clinvar_export/tests/factories.py | 13 +- clinvar_export/tests/test_models.py | 17 +- clinvar_export/tests/test_permissions.py | 1 - clinvar_export/tests/test_permissions_ajax.py | 82 ++-- clinvar_export/tests/test_views.py | 1 - clinvar_export/tests/test_views_ajax.py | 56 ++- clinvar_export/urls.py | 1 + clinvar_export/views.py | 5 +- clinvar_export/views_ajax.py | 51 ++- cohorts/admin.py | 2 - cohorts/forms.py | 3 +- cohorts/migrations/0001_initial.py | 7 +- cohorts/migrations/0003_auto_20200701_1255.py | 5 +- cohorts/migrations/0004_auto_20200710_1548.py | 3 +- cohorts/models.py | 9 +- cohorts/templatetags/cohorts_tags.py | 2 - cohorts/tests/factories.py | 8 +- cohorts/tests/test_cohorts_tags.py | 3 +- cohorts/tests/test_models.py | 3 +- cohorts/tests/test_views.py | 63 ++- cohorts/urls.py | 7 +- cohorts/views.py | 16 +- config/celery.py | 1 + config/settings/base.py | 49 ++- config/settings/production.py | 6 +- config/urls.py | 6 +- .../migrations/0002_auto_20200714_1411.py | 5 +- .../0003_alter_extraanno_anno_data.py | 4 +- .../0004_alter_extraanno_anno_data.py | 5 +- extra_annos/models.py | 3 +- extra_annos/views.py | 2 +- .../migrations/0005_auto_20200123_1033.py | 12 +- .../migrations/0006_remove_mtdb_gap.py | 5 +- .../migrations/0007_auto_20200211_1625.py | 10 +- .../migrations/0008_alter_mtdb_synonymous.py | 4 +- frequencies/tests/factories.py | 2 +- frequencies/tests/test_models.py | 4 +- frequencies/views.py | 2 +- geneinfo/admin.py | 4 +- ...3_materialized_view_geneidtoinheritance.py | 3 +- geneinfo/migrations/0015_mgimapping.py | 3 +- geneinfo/migrations/0016_mgimapping_index.py | 4 +- .../0022_materialized_view_geneidinhpo.py | 2 +- .../0023_materialized_view_geneidinhpo_fix.py | 3 +- .../migrations/0025_auto_20211019_0829.py | 12 +- geneinfo/migrations/0026_hgnc_gtrnadb.py | 4 +- geneinfo/models.py | 7 +- geneinfo/tests/_test_views.py | 6 +- geneinfo/tests/factories.py | 20 +- geneinfo/urls.py | 3 - geneinfo/views.py | 13 +- genomicfeatures/admin.py | 4 +- genomicfeatures/migrations/0001_initial.py | 3 +- genomicfeatures/models.py | 3 - genomicfeatures/tests/factories.py | 10 +- genomicfeatures/tests/test_models.py | 8 +- importer/admin.py | 10 +- importer/management/commands/import_case.py | 8 +- .../management/commands/import_cases_bulk.py | 3 +- importer/management/commands/import_tables.py | 53 ++- importer/management/commands/pg_dump.py | 5 +- .../commands/rebuild_project_case_stats.py | 5 +- .../commands/rebuild_project_stats.py | 6 +- .../commands/rebuild_variant_summary.py | 6 +- .../migrations/0005_auto_20200129_1600.py | 35 +- .../migrations/0007_auto_20200203_1242.py | 1 + .../0009_alter_caseimportinfo_pedigree.py | 5 +- importer/models.py | 21 +- importer/qc_schemas.py | 2 +- importer/rules.py | 6 +- importer/serializers.py | 3 +- importer/tasks.py | 1 - importer/tests/factories.py | 8 +- importer/tests/test_views_api.py | 21 +- importer/urls.py | 1 + importer/views.py | 13 +- importer/views_api.py | 9 +- .../management/commands/check_installation.py | 10 +- .../commands/clear_expired_files.py | 3 +- .../commands/clear_inactive_variant_sets.py | 3 +- .../commands/clear_old_kiosk_cases.py | 3 +- pathways/tests/test_models.py | 4 +- regmaps/admin.py | 18 +- regmaps/cmds.py | 5 +- regmaps/migrations/0001_initial.py | 29 +- regmaps/migrations/0002_auto_20211008_1015.py | 5 +- regmaps/models.py | 45 ++- regmaps/tests/factories.py | 2 +- regmaps/tests/test_cmds.py | 6 +- regmaps/tests/test_models.py | 8 +- requirements/base.txt | 4 - requirements/local.txt | 2 +- requirements/test.txt | 10 +- setup.cfg | 34 +- svdbs/admin.py | 2 +- svdbs/models.py | 3 +- svdbs/tests/factories.py | 6 +- svdbs/tests/test_models.py | 6 +- svdbs/views.py | 2 - svs/admin.py | 7 +- svs/bg_db.py | 36 +- svs/forms.py | 20 +- .../commands/svs_bg_sv_set_cleanup.py | 2 +- svs/management/commands/svs_bg_sv_set_dump.py | 9 +- svs/management/commands/svs_bg_sv_set_list.py | 4 +- svs/management/commands/svs_sv_fill_nulls.py | 2 +- svs/management/commands/svs_svs_dump.py | 3 +- svs/migrations/0001_initial.py | 3 +- .../0002_pgplsql_overlapping_bins.py | 8 +- .../0005_importstructuralvariantbgjob.py | 4 +- svs/migrations/0006_auto_20190703_1709.py | 2 +- svs/migrations/0007_partition_sv_tables.py | 6 +- svs/migrations/0008_set_unlogged_table.py | 2 +- svs/migrations/0010_set_logged_table.py | 2 +- svs/migrations/0011_rename_partitions.py | 2 +- ...ucturalvariants_alter_unique_constraint.py | 2 +- svs/migrations/0015_set_logged_table.py | 2 +- ...roundsvsetjob_cleanupbackgroundsvsetjob.py | 13 +- .../0018_extend_structuralvariant.py | 3 +- svs/models.py | 40 +- svs/plugins.py | 8 +- svs/queries.py | 54 ++- svs/rules.py | 3 +- svs/tasks.py | 7 +- svs/tests/factories.py | 20 +- svs/tests/helpers.py | 3 +- svs/tests/test_bg_db.py | 49 ++- svs/tests/test_import.py | 2 +- svs/tests/test_models.py | 15 +- svs/tests/test_queries.py | 34 +- svs/tests/test_views.py | 3 +- svs/urls.py | 2 +- svs/views.py | 77 ++-- var_stats_qc/migrations/0001_initial.py | 1 + var_stats_qc/qc.py | 2 +- varfish/users/adapters.py | 2 +- varfish/users/admin.py | 1 + varfish/users/apps.py | 4 +- .../migrations/0002_auto_20181015_1316.py | 3 +- .../migrations/0003_auto_20200505_0724.py | 5 +- varfish/users/models.py | 2 +- varfish/users/tasks.py | 2 +- varfish/users/views.py | 1 - varfish/utils.py | 6 +- variants/admin.py | 18 +- variants/file_export.py | 31 +- variants/forms.py | 62 ++- variants/migrations/0001_initial.py | 3 +- ...002_exportfilebgjob_exportfilejobresult.py | 8 +- .../0005_load_btree_gin_extension.py | 3 +- .../migrations/0008_auto_20181113_1912.py | 3 +- .../migrations/0009_smallvariantcomment.py | 3 +- variants/migrations/0016_smallvariantquery.py | 8 +- .../0017_distillersubmissionbgjob.py | 8 +- .../migrations/0020_auto_20181126_1930.py | 3 +- .../migrations/0021_auto_20181127_0911.py | 8 +- .../migrations/0022_auto_20181127_1110.py | 8 +- .../migrations/0026_auto_20190128_1210.py | 3 +- .../migrations/0030_auto_20190307_0723.py | 8 +- .../migrations/0039_acmgcriteriarating.py | 3 +- variants/migrations/0041_array_cat_agg.py | 1 - ...synccaselistbgjob_synccaseresultmessage.py | 3 +- .../0046_partition_smallvariants_table.py | 5 +- .../migrations/0048_importvariantsbgjob.py | 3 +- .../migrations/0053_auto_20190809_1636.py | 3 +- .../migrations/0054_set_unlogged_table.py | 2 +- .../migrations/0056_add_info_and_distance.py | 5 +- .../migrations/0068_kioskannotatebgjob.py | 4 +- variants/migrations/0069_set_logged_table.py | 2 +- ...smallvariant_create_index_chromosome_no.py | 3 +- .../0071_smallvariantflags_flag_molecular.py | 1 + variants/migrations/0072_deletecasebgjob.py | 3 +- ...clearoldkioskcasesbgjob_refreshsmallvar.py | 23 +- .../migrations/0074_caddsubmissionbgjob.py | 8 +- .../migrations/0079_auto_20210204_1006.py | 12 +- .../migrations/0080_spanrsubmissionbgjob.py | 6 +- .../migrations/0081_set_logged_table_again.py | 2 +- .../migrations/0082_auto_20210617_1409.py | 4 +- .../migrations/0083_auto_20211129_1443.py | 11 +- variants/migrations/0085_add_variant_index.py | 2 +- ...mallvariantsummary_excludefrominhousedb.py | 9 +- variants/models.py | 135 ++++--- variants/plugins.py | 29 +- variants/queries.py | 60 +-- variants/query_presets.py | 171 ++++---- variants/query_schemas.py | 18 +- variants/rules.py | 3 +- variants/serializers.py | 9 +- variants/submit_external.py | 2 - variants/submit_filter.py | 16 +- variants/sync_upstream.py | 6 +- variants/tasks.py | 10 +- variants/templatetags/variants_tags.py | 16 +- variants/tests/_fixtures.py | 5 +- variants/tests/factories.py | 41 +- variants/tests/helpers.py | 9 +- variants/tests/test_file_export.py | 79 +++- variants/tests/test_forms.py | 46 ++- variants/tests/test_import.py | 4 +- variants/tests/test_models.py | 14 +- variants/tests/test_queries.py | 168 +++++--- variants/tests/test_query_presets.py | 74 +++- variants/tests/test_schemas.py | 2 +- variants/tests/test_submit_filter.py | 17 +- variants/tests/test_sync_upstream.py | 10 +- variants/tests/test_tasks.py | 13 +- variants/tests/test_templatetags.py | 2 +- variants/tests/test_ui.py | 22 +- variants/tests/test_views.py | 376 +++++++++++------- variants/tests/test_views_api.py | 43 +- variants/urls.py | 2 +- variants/variant_stats.py | 7 +- variants/views.py | 348 +++++++++------- variants/views_api.py | 20 +- 251 files changed, 2470 insertions(+), 1553 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1848b592a..93ae4e58f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,14 +87,17 @@ jobs: with: parallel: true flag-name: Unit Test - - name: run Codacy coverage reporter - uses: codacy/codacy-coverage-reporter-action@master - with: - project-token: ${{ secrets.CODACY_PROJECT_TOKEN }} - coverage-reports: coverage.xml - env: - secrets.CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }} - if: "${{ (matrix.python-version == '3.8') && (matrix.postgres-version == '12') && (!!env.CODACY_PROJECT_TOKEN) }}" + - name: Check import order with isort + run: | + rm -rf src + isort --force-sort-within-sections --profile=black --check . + if: ${{ matrix.python-version == '3.8' }} + - name: Check style with black + run: black --check --line-length 100 . + if: ${{ matrix.python-version == '3.8' }} + - name: Run linting + run: flake8 + if: ${{ matrix.python-version == '3.8' }} coveralls_finish: needs: build diff --git a/Makefile b/Makefile index 11d9bf3a3..1c9657475 100644 --- a/Makefile +++ b/Makefile @@ -62,3 +62,18 @@ test: collectstatic .PHONY: test-noselenium test-noselenium: VARFISH_KIOSK_MODE=0 SKIP_SELENIUM=1 coverage run manage.py test -v2 --settings=config.settings.test + +.PHONY: lint +lint: flake8 + +.PHONY: isort +isort: + isort --force-sort-within-sections --profile=black . + +.PHONY: flake8 +flake8: + flake8 + +coverage: + coverage report + coverage html diff --git a/README.rst b/README.rst index 3696bfa63..b07d9e68d 100644 --- a/README.rst +++ b/README.rst @@ -1,10 +1,6 @@ .. image:: https://readthedocs.org/projects/varfish-server/badge/?version=latest :target: https://varfish-server.readthedocs.io/en/latest/?badge=latest :alt: Documentation Status -.. image:: https://app.codacy.com/project/badge/Grade/f47216cf5a4349acbb9baf5ca1c91329 - :target: https://www.codacy.com/gh/bihealth/varfish-server/dashboard?utm_source=github.com&utm_medium=referral&utm_content=bihealth/varfish-server&utm_campaign=Badge_Grade -.. image:: https://app.codacy.com/project/badge/Coverage/f47216cf5a4349acbb9baf5ca1c91329 - :target: https://www.codacy.com/gh/bihealth/varfish-server/dashboard?utm_source=github.com&utm_medium=referral&utm_content=bihealth/varfish-server&utm_campaign=Badge_Coverage .. image:: https://coveralls.io/repos/github/bihealth/varfish-server/badge.svg?branch=main :target: https://coveralls.io/github/bihealth/varfish-server?branch=main .. image:: https://img.shields.io/badge/License-MIT-green.svg diff --git a/beaconsite/forms.py b/beaconsite/forms.py index eec37b6d9..e0803ecab 100644 --- a/beaconsite/forms.py +++ b/beaconsite/forms.py @@ -2,7 +2,7 @@ from django.db import transaction from projectroles.models import Project -from .models import Consortium, Site, ConsortiumMember, ConsortiumAssignment +from .models import Consortium, ConsortiumAssignment, ConsortiumMember, Site class ConsortiumForm(forms.ModelForm): diff --git a/beaconsite/migrations/0001_initial.py b/beaconsite/migrations/0001_initial.py index 183717422..152596335 100644 --- a/beaconsite/migrations/0001_initial.py +++ b/beaconsite/migrations/0001_initial.py @@ -2,11 +2,12 @@ # Generated by Django 1.11.29 on 2021-03-03 08:23 from __future__ import unicode_literals +import uuid + +import cryptographic_fields.fields from django.conf import settings from django.db import migrations, models import django.db.models.deletion -import cryptographic_fields.fields -import uuid class Migration(migrations.Migration): diff --git a/beaconsite/migrations/0003_auto_20210304_1052.py b/beaconsite/migrations/0003_auto_20210304_1052.py index 821be6856..50f1945f1 100644 --- a/beaconsite/migrations/0003_auto_20210304_1052.py +++ b/beaconsite/migrations/0003_auto_20210304_1052.py @@ -2,8 +2,8 @@ # Generated by Django 1.11.29 on 2021-03-04 10:52 from __future__ import unicode_literals -from django.db import migrations import cryptographic_fields.fields +from django.db import migrations class Migration(migrations.Migration): diff --git a/beaconsite/models.py b/beaconsite/models.py index 80a9cf25a..b813d8b5e 100644 --- a/beaconsite/models.py +++ b/beaconsite/models.py @@ -2,12 +2,12 @@ import uuid as uuid_object from Crypto.PublicKey import RSA +from cryptographic_fields.fields import EncryptedTextField +from django.conf import settings from django.core.exceptions import ValidationError from django.db import models -from django.conf import settings from django.urls import reverse from projectroles.models import Project -from cryptographic_fields.fields import EncryptedTextField #: Django user model. AUTH_USER_MODEL = getattr(settings, "AUTH_USER_MODEL", "auth.User") diff --git a/beaconsite/models_api.py b/beaconsite/models_api.py index dc9681618..8fe94d6af 100644 --- a/beaconsite/models_api.py +++ b/beaconsite/models_api.py @@ -2,7 +2,6 @@ import attr - #: Default assembly ID to assume. DEFAULT_ASSEMBLY_ID = "GRCh37" #: Indicated API version. diff --git a/beaconsite/plugins.py b/beaconsite/plugins.py index e79f5c3d9..a687a56bb 100644 --- a/beaconsite/plugins.py +++ b/beaconsite/plugins.py @@ -1,4 +1,3 @@ -from bgjobs.plugins import BackgroundJobsPluginPoint from projectroles.plugins import SiteAppPluginPoint from .urls import urlpatterns diff --git a/beaconsite/rules.py b/beaconsite/rules.py index 2a7c997a6..ea8fb73da 100644 --- a/beaconsite/rules.py +++ b/beaconsite/rules.py @@ -1,8 +1,6 @@ """Rule definitions for the ``beaconsites`` app.""" import rules -from projectroles import rules as pr_rules - # Rules ------------------------------------------------------------------------ diff --git a/beaconsite/tests/factories.py b/beaconsite/tests/factories.py index 6cfd842ea..7865afa69 100644 --- a/beaconsite/tests/factories.py +++ b/beaconsite/tests/factories.py @@ -3,8 +3,9 @@ from projectroles.models import Project from varfish.users.tests.factories import UserFactory -from variants.tests.factories import ProjectFactory, CaseFactory -from ..models import Site, Consortium, ConsortiumMember, Query, Response, ConsortiumAssignment +from variants.tests.factories import CaseFactory, ProjectFactory + +from ..models import Consortium, ConsortiumAssignment, ConsortiumMember, Query, Response, Site class SiteFactory(factory.django.DjangoModelFactory): diff --git a/beaconsite/tests/test_models.py b/beaconsite/tests/test_models.py index e3148cdd5..9ccd0a077 100644 --- a/beaconsite/tests/test_models.py +++ b/beaconsite/tests/test_models.py @@ -2,15 +2,15 @@ from django.core.exceptions import ValidationError from test_plus.test import TestCase +from ..models import Consortium, ConsortiumMember, Query, Response, Site from .factories import ( - SiteFactory, ConsortiumFactory, ConsortiumMemberFactory, ConsortiumWithLocalAndRemoteSiteFactory, QueryFactory, ResponseFactory, + SiteFactory, ) -from ..models import Site, Consortium, ConsortiumMember, Query, Response class TestSite(TestCase): diff --git a/beaconsite/tests/test_permissions_ajax.py b/beaconsite/tests/test_permissions_ajax.py index 90833dfe3..17c07eb49 100644 --- a/beaconsite/tests/test_permissions_ajax.py +++ b/beaconsite/tests/test_permissions_ajax.py @@ -2,13 +2,12 @@ import cattr from django.urls import reverse - from projectroles.tests.test_permissions_api import TestProjectAPIPermissionBase import requests_mock -from .factories import SiteFactory from ..models import Site from ..models_api import BeaconAlleleRequest +from .factories import SiteFactory class TestOrganisationAjaxViews(TestProjectAPIPermissionBase): @@ -16,7 +15,10 @@ class TestOrganisationAjaxViews(TestProjectAPIPermissionBase): def test_get(self, r_mock): _local_site = SiteFactory(role=Site.LOCAL) remote_site = SiteFactory() - url = reverse("beaconsite:ajax-beacon-info", kwargs={"site": remote_site.sodar_uuid},) + url = reverse( + "beaconsite:ajax-beacon-info", + kwargs={"site": remote_site.sodar_uuid}, + ) good_users = [ self.superuser, ] @@ -71,7 +73,10 @@ def test_get(self, r_mock): url_params = "&".join( "%s=%s" % (k, v) for k, v in cattr.unstructure(beacon_allele_request).items() ) - url = reverse("beaconsite:ajax-beacon-query", kwargs={"site": remote_site.sodar_uuid},) + url = reverse( + "beaconsite:ajax-beacon-query", + kwargs={"site": remote_site.sodar_uuid}, + ) good_users = [ self.superuser, ] diff --git a/beaconsite/tests/test_permissions_api.py b/beaconsite/tests/test_permissions_api.py index ac3464cfb..87e63e338 100644 --- a/beaconsite/tests/test_permissions_api.py +++ b/beaconsite/tests/test_permissions_api.py @@ -1,20 +1,21 @@ -import urllib import datetime +import urllib from wsgiref.handlers import format_date_time -import cattr -import httpsig from Crypto.PublicKey import RSA +import cattr from django.shortcuts import reverse from django.utils import timezone +import httpsig from projectroles.tests.test_permissions_api import TestProjectAPIPermissionBase from requests_http_signature import HTTPSignatureAuth from variants.tests.factories import SmallVariantFactory + from ..models import Site -from .factories import ConsortiumWithLocalAndRemoteSiteFactory, ConsortiumAssignmentFactory from ..models_api import BeaconAlleleRequest from ..views_api import _header_canonical +from .factories import ConsortiumAssignmentFactory, ConsortiumWithLocalAndRemoteSiteFactory class AcceptHeaderMixin: @@ -48,7 +49,8 @@ def setUp(self): self.consortium = ConsortiumWithLocalAndRemoteSiteFactory() self.remote_site = Site.objects.get(role=Site.REMOTE) ConsortiumAssignmentFactory( - consortium=self.consortium, project=self.project, + consortium=self.consortium, + project=self.project, ) def test_success(self): @@ -115,7 +117,8 @@ def setUp(self): self.consortium = ConsortiumWithLocalAndRemoteSiteFactory() self.remote_site = Site.objects.get(role=Site.REMOTE) ConsortiumAssignmentFactory( - consortium=self.consortium, project=self.project, + consortium=self.consortium, + project=self.project, ) self.small_variant = SmallVariantFactory(case__project=self.project) self.beacon_allele_request = BeaconAlleleRequest( diff --git a/beaconsite/tests/test_views.py b/beaconsite/tests/test_views.py index 5f2ab6e15..d7c41f5a8 100644 --- a/beaconsite/tests/test_views.py +++ b/beaconsite/tests/test_views.py @@ -187,7 +187,10 @@ class TestSiteUpdateView(TestViewsBase): def test_render(self): with self.login(self.superuser): response = self.client.get( - reverse("beaconsite:site-update", kwargs={"site": self.site.sodar_uuid},) + reverse( + "beaconsite:site-update", + kwargs={"site": self.site.sodar_uuid}, + ) ) self.assertEqual(response.status_code, 200) self.assertIsNotNone(response.context["object"]) @@ -211,7 +214,10 @@ def test_update(self): with self.login(self.superuser): response = self.client.post( - reverse("beaconsite:site-update", kwargs={"site": self.site.sodar_uuid},), + reverse( + "beaconsite:site-update", + kwargs={"site": self.site.sodar_uuid}, + ), post_data, ) @@ -243,7 +249,10 @@ class TestSiteDeleteView(TestViewsBase): def test_render(self): with self.login(self.superuser): response = self.client.get( - reverse("beaconsite:site-delete", kwargs={"site": self.site.sodar_uuid},) + reverse( + "beaconsite:site-delete", + kwargs={"site": self.site.sodar_uuid}, + ) ) self.assertEqual(response.status_code, 200) @@ -253,7 +262,10 @@ def test_delete(self): with self.login(self.superuser): response = self.client.post( - reverse("beaconsite:site-delete", kwargs={"site": self.site.sodar_uuid},) + reverse( + "beaconsite:site-delete", + kwargs={"site": self.site.sodar_uuid}, + ) ) self.assertEqual(response.status_code, 302) self.assertEqual(response.url, reverse("beaconsite:site-list")) diff --git a/beaconsite/tests/test_views_ajax.py b/beaconsite/tests/test_views_ajax.py index 1bfa6f5b9..313e996f8 100644 --- a/beaconsite/tests/test_views_ajax.py +++ b/beaconsite/tests/test_views_ajax.py @@ -5,7 +5,6 @@ from projectroles.tests.test_permissions_api import TestProjectAPIPermissionBase import requests_mock - RE_UUID4 = re.compile(r"^[0-9a-f-]+$") RE_DATETIME = re.compile(r"^\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\d\d\d\dZ$") diff --git a/beaconsite/tests/test_views_api.py b/beaconsite/tests/test_views_api.py index d7368b0b2..51163d17f 100644 --- a/beaconsite/tests/test_views_api.py +++ b/beaconsite/tests/test_views_api.py @@ -3,10 +3,11 @@ from variants.tests.factories import SmallVariantFactory from variants.tests.helpers import ApiViewTestBase -from .test_permissions_api import AcceptHeaderMixin + from ..models import Site -from .factories import ConsortiumWithLocalAndRemoteSiteFactory, ConsortiumAssignmentFactory from ..models_api import BeaconAlleleRequest +from .factories import ConsortiumAssignmentFactory, ConsortiumWithLocalAndRemoteSiteFactory +from .test_permissions_api import AcceptHeaderMixin class TestBeaconInfoApiView(AcceptHeaderMixin, ApiViewTestBase): @@ -16,7 +17,8 @@ def setUp(self): self.local_site = Site.objects.get(role=Site.LOCAL) self.remote_site = Site.objects.get(role=Site.REMOTE) ConsortiumAssignmentFactory( - consortium=self.consortium, project=self.project, + consortium=self.consortium, + project=self.project, ) def test_get_info(self): @@ -54,7 +56,8 @@ def setUp(self): self.local_site = Site.objects.get(role=Site.LOCAL) self.remote_site = Site.objects.get(role=Site.REMOTE) ConsortiumAssignmentFactory( - consortium=self.consortium, project=self.project, + consortium=self.consortium, + project=self.project, ) self.small_variant = SmallVariantFactory(case__project=self.project) self.beacon_allele_request = BeaconAlleleRequest( diff --git a/beaconsite/urls.py b/beaconsite/urls.py index 87e7e9093..1d6e97844 100644 --- a/beaconsite/urls.py +++ b/beaconsite/urls.py @@ -2,13 +2,22 @@ """ from django.conf.urls import url -from . import views, views_api, views_ajax + +from . import views, views_ajax, views_api app_name = "beaconsite" ui_urlpatterns = [ - url(regex=r"^$", view=views.IndexView.as_view(), name="index",), - url(regex=r"^consortium$", view=views.ConsortiumListView.as_view(), name="consortium-list",), + url( + regex=r"^$", + view=views.IndexView.as_view(), + name="index", + ), + url( + regex=r"^consortium$", + view=views.ConsortiumListView.as_view(), + name="consortium-list", + ), url( regex=r"^consortium/(?P[0-9a-f-]+)$", view=views.ConsortiumDetailView.as_view(), @@ -29,13 +38,21 @@ view=views.ConsortiumDeleteView.as_view(), name="consortium-delete", ), - url(regex=r"^site$", view=views.SiteListView.as_view(), name="site-list",), + url( + regex=r"^site$", + view=views.SiteListView.as_view(), + name="site-list", + ), url( regex=r"^site/(?P[0-9a-f-]+)$", view=views.SiteDetailView.as_view(), name="site-detail", ), - url(regex=r"^site/create/$", view=views.SiteCreateView.as_view(), name="site-create",), + url( + regex=r"^site/create/$", + view=views.SiteCreateView.as_view(), + name="site-create", + ), url( regex=r"^site/update/(?P[0-9a-f-]+)$", view=views.SiteUpdateView.as_view(), diff --git a/beaconsite/views.py b/beaconsite/views.py index a0531e9bc..51af241cd 100644 --- a/beaconsite/views.py +++ b/beaconsite/views.py @@ -1,24 +1,18 @@ from django.contrib import messages -from django.http import HttpResponseRedirect from django.shortcuts import redirect - from django.urls import reverse -from django.utils.safestring import mark_safe -from django.views.generic import ListView, TemplateView, DetailView -from django.views.generic.edit import CreateView, DeleteView, UpdateView, ModelFormMixin -from projectroles.views import ( - LoginRequiredMixin, - ProjectContextMixin, - ProjectPermissionMixin, - LoggedInPermissionMixin, -) +from django.views.generic import DetailView, ListView, TemplateView +from django.views.generic.edit import CreateView, DeleteView, ModelFormMixin, UpdateView +from projectroles.views import LoggedInPermissionMixin, LoginRequiredMixin, ProjectContextMixin from .forms import ConsortiumForm, SiteForm from .models import Consortium, Site class IndexView( - LoginRequiredMixin, LoggedInPermissionMixin, TemplateView, + LoginRequiredMixin, + LoggedInPermissionMixin, + TemplateView, ): """Display entry point into site-wide app.""" @@ -33,7 +27,9 @@ def get_context_data(self, **kwargs): class ConsortiumListView( - LoginRequiredMixin, LoggedInPermissionMixin, ListView, + LoginRequiredMixin, + LoggedInPermissionMixin, + ListView, ): """Display list of ``Consortium`` records.""" @@ -43,7 +39,9 @@ class ConsortiumListView( class ConsortiumDetailView( - LoginRequiredMixin, LoggedInPermissionMixin, DetailView, + LoginRequiredMixin, + LoggedInPermissionMixin, + DetailView, ): """Detail view of ``Site`` record.""" @@ -88,7 +86,10 @@ class ConsortiumUpdateView( class ConsortiumDeleteView( - LoginRequiredMixin, LoggedInPermissionMixin, ProjectContextMixin, DeleteView, + LoginRequiredMixin, + LoggedInPermissionMixin, + ProjectContextMixin, + DeleteView, ): permission_required = "beaconsite.delete_data" model = Consortium @@ -102,7 +103,9 @@ def get_success_url(self): class SiteListView( - LoginRequiredMixin, LoggedInPermissionMixin, ListView, + LoginRequiredMixin, + LoggedInPermissionMixin, + ListView, ): """Display list of ``Site`` records.""" @@ -112,7 +115,9 @@ class SiteListView( class SiteDetailView( - LoginRequiredMixin, LoggedInPermissionMixin, DetailView, + LoginRequiredMixin, + LoggedInPermissionMixin, + DetailView, ): """Detail view of ``Site`` record.""" @@ -152,7 +157,10 @@ class SiteUpdateView(LoginRequiredMixin, SiteModifyMixin, LoggedInPermissionMixi class SiteDeleteView( - LoginRequiredMixin, LoggedInPermissionMixin, ProjectContextMixin, DeleteView, + LoginRequiredMixin, + LoggedInPermissionMixin, + ProjectContextMixin, + DeleteView, ): permission_required = "beaconsite.delete_data" model = Site diff --git a/beaconsite/views_ajax.py b/beaconsite/views_ajax.py index ae5b1e7c4..a703cab0c 100644 --- a/beaconsite/views_ajax.py +++ b/beaconsite/views_ajax.py @@ -2,7 +2,7 @@ from django.http import JsonResponse from django.shortcuts import get_object_or_404 from django.views import View -from projectroles.views import LoginRequiredMixin, LoggedInPermissionMixin +from projectroles.views import LoggedInPermissionMixin, LoginRequiredMixin import requests from requests_http_signature import HTTPSignatureAuth diff --git a/beaconsite/views_api.py b/beaconsite/views_api.py index aaffbb5f6..754d30f54 100644 --- a/beaconsite/views_api.py +++ b/beaconsite/views_api.py @@ -4,22 +4,23 @@ from django.utils import timezone from django.utils.http import parse_http_date from httpsig import HeaderVerifier +from rest_framework import authentication, exceptions from rest_framework.permissions import BasePermission -from rest_framework.views import APIView from rest_framework.response import Response -from rest_framework import exceptions, authentication -from sqlalchemy import select, and_ +from rest_framework.views import APIView +from sqlalchemy import and_, select from variants.helpers import get_engine from variants.models import Case, SmallVariant + from .models import Site from .models_api import ( + API_VERSION, + BeaconAlleleRequest, + BeaconAlleleResponse, BeaconInfo, Dataset, Organisation, - BeaconAlleleRequest, - API_VERSION, - BeaconAlleleResponse, ) @@ -172,7 +173,10 @@ def _handle(self, request, *_args, **_kwargs): ) datasets = [ Dataset( - id=str(p.sodar_uuid), name=p.title, assembly="GRCh37", description=p.description, + id=str(p.sodar_uuid), + name=p.title, + assembly="GRCh37", + description=p.description, ) for p in remote_site.get_all_projects() ] diff --git a/clinvar/clinvar_models.py b/clinvar/clinvar_models.py index ad4148731..722d12537 100644 --- a/clinvar/clinvar_models.py +++ b/clinvar/clinvar_models.py @@ -5,11 +5,10 @@ import datetime import json +import typing import attr import cattr -from dateutil.parser import isoparse -import typing class DateTimeEncoder(json.JSONEncoder): diff --git a/clinvar/migrations/0002_clinvarpathogenicgenes.py b/clinvar/migrations/0002_clinvarpathogenicgenes.py index 56bf5cd99..f237c4b7d 100644 --- a/clinvar/migrations/0002_clinvarpathogenicgenes.py +++ b/clinvar/migrations/0002_clinvarpathogenicgenes.py @@ -2,9 +2,8 @@ # Generated by Django 1.11.20 on 2019-04-16 14:00 from __future__ import unicode_literals -from django.db import migrations, models from django.conf import settings - +from django.db import migrations, models operations = [ migrations.CreateModel( diff --git a/clinvar/migrations/0004_delete_clinvar.py b/clinvar/migrations/0004_delete_clinvar.py index 684ab279a..68fdbca9b 100644 --- a/clinvar/migrations/0004_delete_clinvar.py +++ b/clinvar/migrations/0004_delete_clinvar.py @@ -12,6 +12,10 @@ class Migration(migrations.Migration): ] operations = [ - migrations.DeleteModel(name="Clinvar",), - migrations.DeleteModel(name="ClinvarPathogenicGenes",), + migrations.DeleteModel( + name="Clinvar", + ), + migrations.DeleteModel( + name="ClinvarPathogenicGenes", + ), ] diff --git a/clinvar/migrations/0005_auto_20200611_1719.py b/clinvar/migrations/0005_auto_20200611_1719.py index 6ca161853..e044f3084 100644 --- a/clinvar/migrations/0005_auto_20200611_1719.py +++ b/clinvar/migrations/0005_auto_20200611_1719.py @@ -42,7 +42,10 @@ class Migration(migrations.Migration): base_field=models.CharField(max_length=32), size=None ), ), - ("vcv", models.CharField(max_length=32),), + ( + "vcv", + models.CharField(max_length=32), + ), ("point_rating", models.IntegerField()), ("pathogenicity", models.CharField(max_length=128)), ("review_status", models.CharField(max_length=128)), diff --git a/clinvar/migrations/0006_clinvarpathogenicgenes.py b/clinvar/migrations/0006_clinvarpathogenicgenes.py index 8a2f41bce..17b80773d 100644 --- a/clinvar/migrations/0006_clinvarpathogenicgenes.py +++ b/clinvar/migrations/0006_clinvarpathogenicgenes.py @@ -2,9 +2,8 @@ # Generated by Django 1.11.20 on 2019-04-16 14:00 from __future__ import unicode_literals -from django.db import migrations, models from django.conf import settings - +from django.db import migrations, models operations = [ migrations.CreateModel( diff --git a/clinvar/migrations/0007_alter_clinvar_details.py b/clinvar/migrations/0007_alter_clinvar_details.py index 0e6e7467f..edf67444a 100644 --- a/clinvar/migrations/0007_alter_clinvar_details.py +++ b/clinvar/migrations/0007_alter_clinvar_details.py @@ -1,6 +1,7 @@ # Generated by Django 3.2.9 on 2021-11-29 14:43 from django.db import migrations + import varfish.utils @@ -12,6 +13,8 @@ class Migration(migrations.Migration): operations = [ migrations.AlterField( - model_name="clinvar", name="details", field=varfish.utils.JSONField(), + model_name="clinvar", + name="details", + field=varfish.utils.JSONField(), ), ] diff --git a/clinvar/migrations/0008_auto_20220829_1132.py b/clinvar/migrations/0008_auto_20220829_1132.py index b413860c9..e3f07da02 100644 --- a/clinvar/migrations/0008_auto_20220829_1132.py +++ b/clinvar/migrations/0008_auto_20220829_1132.py @@ -11,14 +11,23 @@ class Migration(migrations.Migration): operations = [ migrations.RenameField( - model_name="clinvar", old_name="point_rating", new_name="gold_stars", + model_name="clinvar", + old_name="point_rating", + new_name="gold_stars", + ), + migrations.RemoveField( + model_name="clinvar", + name="pathogenicity_summary", ), - migrations.RemoveField(model_name="clinvar", name="pathogenicity_summary",), migrations.AddField( - model_name="clinvar", name="origin", field=models.CharField(max_length=32, null=True), + model_name="clinvar", + name="origin", + field=models.CharField(max_length=32, null=True), ), migrations.AddField( - model_name="clinvar", name="rcv", field=models.CharField(max_length=32, null=True), + model_name="clinvar", + name="rcv", + field=models.CharField(max_length=32, null=True), ), migrations.AddField( model_name="clinvar", diff --git a/clinvar/models.py b/clinvar/models.py index cafb3faf6..93b33b536 100644 --- a/clinvar/models.py +++ b/clinvar/models.py @@ -1,9 +1,10 @@ from django.conf import settings -from django.db import models, connection, transaction from django.contrib.postgres.fields import ArrayField +from django.db import connection, models, transaction from postgres_copy import CopyManager from varfish.utils import JSONField + from .clinvar_models import ClinVarSet diff --git a/clinvar_export/admin.py b/clinvar_export/admin.py index 1982a42de..c3d65b47e 100644 --- a/clinvar_export/admin.py +++ b/clinvar_export/admin.py @@ -1,15 +1,15 @@ from django.contrib import admin from .models import ( + AssertionMethod, Family, Individual, + Organisation, + Submission, SubmissionIndividual, - AssertionMethod, + SubmissionSet, Submitter, - Organisation, SubmittingOrg, - SubmissionSet, - Submission, ) # Register your models here. diff --git a/clinvar_export/clinvar_xml.py b/clinvar_export/clinvar_xml.py index 7fd07ec9d..edf8aa146 100644 --- a/clinvar_export/clinvar_xml.py +++ b/clinvar_export/clinvar_xml.py @@ -11,7 +11,11 @@ class SubmissionXmlGenerator: """Helper class to create ClinVar submission XML.""" def __init__(self): - self.em = ElementMaker(nsmap={"xsi": "http://www.w3.org/2001/XMLSchema-instance",}) + self.em = ElementMaker( + nsmap={ + "xsi": "http://www.w3.org/2001/XMLSchema-instance", + } + ) def generate_tree(self, submission_set): root = self.em( @@ -109,7 +113,10 @@ def build_assertion_method_tag(self, submission): self.em( "AttributeSet", self.em("MeasureTraitAttributeType", "AssertionMethod", **{"val_type": "name"}), - self.em("Attribute", submission.assertion_method.title,), + self.em( + "Attribute", + submission.assertion_method.title, + ), self.em("Citation", self.build_assertion_method_citation_child(submission)), ) ] @@ -168,7 +175,14 @@ def build_observed_in_tag(self, submission): ), *[self.build_citation(citation) for citation in sub_ind.citations], ), - self.em("Method", self.em("MethodType", sub_ind.source, **{"val_type": "name"},),), + self.em( + "Method", + self.em( + "MethodType", + sub_ind.source, + **{"val_type": "name"}, + ), + ), *self.build_variant_alleles(sub_ind), self.em( "ObservedData", @@ -197,12 +211,28 @@ def build_citation(self, citation): def build_phenotype_trait_set(self, submission_individual): def build_xref(term_id): if term_id.startswith("HP:"): - self.em("XRef", **{"db": "HP", "id": term_id,}), + self.em( + "XRef", + **{ + "db": "HP", + "id": term_id, + }, + ), elif term_id.startswith("OMIM:") or term_id.startswith("MIM:"): - self.em("XRef", **{"db": "OMIM", "id": ("MIM:%s" % term_id.split(":", 1)[1]),}), + self.em( + "XRef", + **{ + "db": "OMIM", + "id": ("MIM:%s" % term_id.split(":", 1)[1]), + }, + ), elif term_id.startswith("ORPHA:"): self.em( - "XRef", **{"db": "Orphanet", "id": ("ORPHA:%s" % term_id.split(":", 1)[1]),} + "XRef", + **{ + "db": "Orphanet", + "id": ("ORPHA:%s" % term_id.split(":", 1)[1]), + }, ), else: return self.em("Invalid-XRef", term_id) @@ -223,7 +253,13 @@ def build_xref(term_id): self.em("ElementValueType", "Preferred", **{"val_type": "name"}), self.em("ElementValue", phenotype["term_name"]), ), - self.em("XRef", **{"db": "HP", "id": phenotype["term_id"],}), + self.em( + "XRef", + **{ + "db": "HP", + "id": phenotype["term_id"], + }, + ), ) for phenotype in submission_individual.phenotypes ], @@ -267,19 +303,34 @@ def build_variant_measure_set(self, submission): seq_location_dict["alternateAllele"] = submission.variant_alternative return self.em( "MeasureSet", - self.em("MeasureSetType", "Variant", **{"val_type": "name"},), + self.em( + "MeasureSetType", + "Variant", + **{"val_type": "name"}, + ), self.em( "Measure", self.em("MeasureType", submission.variant_type, **{"val_type": "name"}), - self.em("SequenceLocation", **seq_location_dict,), + self.em( + "SequenceLocation", + **seq_location_dict, + ), self.em( "MeasureRelationship", - self.em("MeasureRelationshipType", "variant in gene", **{"val_type": "name"},), + self.em( + "MeasureRelationshipType", + "variant in gene", + **{"val_type": "name"}, + ), *[ self.em( "Symbol", ET.Comment(" HGVS: %s:%s " % (gene_symbol, gene_hgvs)), - self.em("ElementValueType", "Preferred", **{"val_type": "name"},), + self.em( + "ElementValueType", + "Preferred", + **{"val_type": "name"}, + ), self.em("ElementValue", gene_symbol), ) for (gene_symbol, gene_hgvs) in zip( @@ -315,7 +366,13 @@ def build_disease_trait_set(self, submission): ] return self.em( - "TraitSet", self.em("TraitSetType", "Disease", **{"val_type": "name"},), *traits + "TraitSet", + self.em( + "TraitSetType", + "Disease", + **{"val_type": "name"}, + ), + *traits, ) def build_disease_xref(self, term_id): diff --git a/clinvar_export/migrations/0001_initial.py b/clinvar_export/migrations/0001_initial.py index 19fd1cfa6..450f1b405 100644 --- a/clinvar_export/migrations/0001_initial.py +++ b/clinvar_export/migrations/0001_initial.py @@ -2,10 +2,11 @@ # Generated by Django 1.11.29 on 2021-03-30 13:17 from __future__ import unicode_literals +import uuid + import django.contrib.postgres.fields from django.db import migrations, models import django.db.models.deletion -import uuid class Migration(migrations.Migration): @@ -42,7 +43,9 @@ class Migration(migrations.Migration): ("title", models.TextField(max_length=200)), ("reference", models.TextField(max_length=200)), ], - options={"ordering": ("reference",),}, + options={ + "ordering": ("reference",), + }, ), migrations.CreateModel( name="Family", @@ -88,7 +91,9 @@ class Migration(migrations.Migration): ), ), ], - options={"ordering": ("date_created",),}, + options={ + "ordering": ("date_created",), + }, ), migrations.CreateModel( name="Individual", @@ -122,7 +127,9 @@ class Migration(migrations.Migration): ), ), ], - options={"ordering": ("date_created",),}, + options={ + "ordering": ("date_created",), + }, ), migrations.CreateModel( name="Organisation", @@ -148,7 +155,9 @@ class Migration(migrations.Migration): ("clinvar_id", models.IntegerField()), ("name", models.TextField()), ], - options={"ordering": ("clinvar_id",),}, + options={ + "ordering": ("clinvar_id",), + }, ), migrations.CreateModel( name="Submission", @@ -198,7 +207,10 @@ class Migration(migrations.Migration): base_field=models.TextField(), size=None ), ), - ("diseases", models.JSONField(blank=True, default=list, null=True),), + ( + "diseases", + models.JSONField(blank=True, default=list, null=True), + ), ( "assertion_method", models.ForeignKey( @@ -207,7 +219,9 @@ class Migration(migrations.Migration): ), ), ], - options={"ordering": ("sort_order",),}, + options={ + "ordering": ("sort_order",), + }, ), migrations.CreateModel( name="SubmissionIndividual", @@ -231,7 +245,10 @@ class Migration(migrations.Migration): models.UUIDField(default=uuid.uuid4, help_text="SODAR UUID", unique=True), ), ("sort_order", models.IntegerField()), - ("phenotypes", models.JSONField(blank=True, default=list, null=True),), + ( + "phenotypes", + models.JSONField(blank=True, default=list, null=True), + ), ("variant_origin", models.TextField(default="not-reported", max_length=100)), ("variant_allele_count", models.IntegerField(blank=True, default=None, null=True)), ( @@ -265,7 +282,9 @@ class Migration(migrations.Migration): ), ), ], - options={"ordering": ("sort_order",),}, + options={ + "ordering": ("sort_order",), + }, ), migrations.CreateModel( name="SubmissionSet", @@ -291,7 +310,9 @@ class Migration(migrations.Migration): ("state", models.TextField(max_length=32)), ("title", models.TextField()), ], - options={"ordering": ("date_created",),}, + options={ + "ordering": ("date_created",), + }, ), migrations.CreateModel( name="Submitter", @@ -317,7 +338,9 @@ class Migration(migrations.Migration): ("clinvar_id", models.IntegerField()), ("name", models.TextField()), ], - options={"ordering": ("name",),}, + options={ + "ordering": ("name",), + }, ), migrations.CreateModel( name="SubmittingOrg", @@ -357,7 +380,9 @@ class Migration(migrations.Migration): ), ), ], - options={"ordering": ("sort_order",),}, + options={ + "ordering": ("sort_order",), + }, ), migrations.AddField( model_name="submissionset", @@ -400,12 +425,15 @@ class Migration(migrations.Migration): ), ), migrations.AlterUniqueTogether( - name="submittingorg", unique_together=set([("organisation", "submission_set")]), + name="submittingorg", + unique_together=set([("organisation", "submission_set")]), ), migrations.AlterUniqueTogether( - name="submissionindividual", unique_together=set([("individual", "submission")]), + name="submissionindividual", + unique_together=set([("individual", "submission")]), ), migrations.AlterUniqueTogether( - name="family", unique_together=set([("case_name", "project")]), + name="family", + unique_together=set([("case_name", "project")]), ), ] diff --git a/clinvar_export/migrations/0002_bootstrap_records.py b/clinvar_export/migrations/0002_bootstrap_records.py index 8280cdefa..1312be97b 100644 --- a/clinvar_export/migrations/0002_bootstrap_records.py +++ b/clinvar_export/migrations/0002_bootstrap_records.py @@ -7,7 +7,8 @@ def add_submitters(apps, _schema_editor): Submitter = apps.get_model("clinvar_export", "Submitter") Submitter.objects.create( - clinvar_id=9131, name="Manuel Holtgrewe", + clinvar_id=9131, + name="Manuel Holtgrewe", ) @@ -21,14 +22,17 @@ def add_organisations(apps, _schema_editor): ), ) Organisation.objects.create( - clinvar_id=507461, name="CUBI - Core Unit Bioinformatics (Berlin Institute of Health)", + clinvar_id=507461, + name="CUBI - Core Unit Bioinformatics (Berlin Institute of Health)", ) def add_assertion_methods(apps, _schema_editor): AssertionMethod = apps.get_model("clinvar_export", "AssertionMethod") AssertionMethod.objects.create( - is_builtin=True, title="ACMG Guidelines, 2015", reference="PMID:25741868", + is_builtin=True, + title="ACMG Guidelines, 2015", + reference="PMID:25741868", ) diff --git a/clinvar_export/migrations/0003_auto_20211129_1443.py b/clinvar_export/migrations/0003_auto_20211129_1443.py index aa476e43e..7730bc9df 100644 --- a/clinvar_export/migrations/0003_auto_20211129_1443.py +++ b/clinvar_export/migrations/0003_auto_20211129_1443.py @@ -1,6 +1,7 @@ # Generated by Django 3.2.9 on 2021-11-29 14:43 from django.db import migrations + import varfish.utils diff --git a/clinvar_export/models.py b/clinvar_export/models.py index f828cadce..5e8424404 100644 --- a/clinvar_export/models.py +++ b/clinvar_export/models.py @@ -6,13 +6,13 @@ import uuid as uuid_object +from django.conf import settings from django.contrib.postgres.fields import ArrayField from django.db import models, transaction -from django.conf import settings from projectroles.models import Project from varfish.utils import JSONField -from variants.models import CaseAwareProject, Case +from variants.models import Case, CaseAwareProject #: Django user model. AUTH_USER_MODEL = getattr(settings, "AUTH_USER_MODEL", "auth.User") @@ -352,7 +352,10 @@ class Submission(models.Model): significance_last_evaluation = models.DateField() #: The used assertion method. - assertion_method = models.ForeignKey(AssertionMethod, on_delete=models.CASCADE,) + assertion_method = models.ForeignKey( + AssertionMethod, + on_delete=models.CASCADE, + ) #: Mode of Inheritance inheritance = models.TextField(max_length=100, blank=True, null=True) #: Age of onset diff --git a/clinvar_export/rules.py b/clinvar_export/rules.py index 562b95a06..1a88fb9d4 100644 --- a/clinvar_export/rules.py +++ b/clinvar_export/rules.py @@ -1,6 +1,5 @@ -import rules from projectroles import rules as pr_rules - +import rules rules.add_perm( "clinvar_export.view_data", diff --git a/clinvar_export/serializers.py b/clinvar_export/serializers.py index b30f0e18a..c322466b8 100644 --- a/clinvar_export/serializers.py +++ b/clinvar_export/serializers.py @@ -1,28 +1,28 @@ -from django.contrib.postgres.aggregates import ArrayAgg from django.shortcuts import get_object_or_404 -from projectroles.serializers import SODARProjectModelSerializer, SODARModelSerializer +from projectroles.serializers import SODARModelSerializer, SODARProjectModelSerializer from rest_framework import serializers from rest_framework.fields import SerializerMethodField -from rest_framework.serializers import BaseSerializer, Serializer +from rest_framework.serializers import Serializer -from geneinfo.models import HpoName, Hpo, Hgnc +from geneinfo.models import Hgnc, Hpo, HpoName from variants.models import ( + AcmgCriteriaRating, + Case, CasePhenotypeTerms, SmallVariant, - SmallVariantFlags, SmallVariantComment, - AcmgCriteriaRating, - Case, + SmallVariantFlags, ) + from .models import ( - SubmissionSet, - Organisation, + AssertionMethod, Family, Individual, - AssertionMethod, - Submitter, + Organisation, Submission, SubmissionIndividual, + SubmissionSet, + Submitter, SubmittingOrg, ) diff --git a/clinvar_export/tests/factories.py b/clinvar_export/tests/factories.py index 856e613d7..0bfba5b43 100644 --- a/clinvar_export/tests/factories.py +++ b/clinvar_export/tests/factories.py @@ -1,20 +1,21 @@ """Factory Boy factory classes for ``clinvar_export``.""" -import factory from django.utils import timezone +import factory + +from variants.tests.factories import CaseFactory, ProjectFactory from ..models import ( + AssertionMethod, Family, Individual, - AssertionMethod, - Submitter, Organisation, - SubmittingOrg, - SubmissionSet, Submission, SubmissionIndividual, + SubmissionSet, + Submitter, + SubmittingOrg, ) -from variants.tests.factories import CaseFactory, ProjectFactory class FamilyFactory(factory.django.DjangoModelFactory): diff --git a/clinvar_export/tests/test_models.py b/clinvar_export/tests/test_models.py index e1a31832a..3643f0f8e 100644 --- a/clinvar_export/tests/test_models.py +++ b/clinvar_export/tests/test_models.py @@ -2,28 +2,29 @@ from test_plus.test import TestCase +from variants.models import Case +from variants.tests.factories import CaseFactory + from ..models import ( + AssertionMethod, Family, Individual, - AssertionMethod, - Submitter, Organisation, + Submission, SubmissionSet, + Submitter, SubmittingOrg, - Submission, ) from .factories import ( + AssertionMethodFactory, FamilyFactory, IndividualFactory, - AssertionMethodFactory, - SubmitterFactory, OrganisationFactory, + SubmissionFactory, SubmissionSetFactory, + SubmitterFactory, SubmittingOrgFactory, - SubmissionFactory, ) -from variants.models import Case -from variants.tests.factories import CaseFactory class TestFamily(TestCase): diff --git a/clinvar_export/tests/test_permissions.py b/clinvar_export/tests/test_permissions.py index ad905a19b..7965b73ab 100644 --- a/clinvar_export/tests/test_permissions.py +++ b/clinvar_export/tests/test_permissions.py @@ -1,5 +1,4 @@ from django.urls import reverse - from projectroles.tests.test_permissions import TestProjectPermissionBase diff --git a/clinvar_export/tests/test_permissions_ajax.py b/clinvar_export/tests/test_permissions_ajax.py index 97dcfca26..e935ff203 100644 --- a/clinvar_export/tests/test_permissions_ajax.py +++ b/clinvar_export/tests/test_permissions_ajax.py @@ -1,24 +1,24 @@ from unittest import skip from django.urls import reverse - from projectroles.tests.test_permissions_api import TestProjectAPIPermissionBase -from geneinfo.tests.factories import HpoNameFactory, HpoFactory +from geneinfo.tests.factories import HpoFactory, HpoNameFactory from variants.tests.factories import CaseFactory, SmallVariantCommentFactory + +from ..models import Family from .factories import ( - SubmissionSetFactory, - SubmissionFactory, - SubmissionWithIndividualFactory, + AssertionMethodFactory, FamilyFactory, IndividualFactory, - AssertionMethodFactory, - SubmissionSetWithOrgFactory, OrganisationFactory, - SubmittingOrgFactory, + SubmissionFactory, + SubmissionSetFactory, + SubmissionSetWithOrgFactory, + SubmissionWithIndividualFactory, SubmitterFactory, + SubmittingOrgFactory, ) -from ..models import Family class TestOrganisationAjaxViews(TestProjectAPIPermissionBase): @@ -26,7 +26,8 @@ class TestOrganisationAjaxViews(TestProjectAPIPermissionBase): def test_list(self): url = reverse( - "clinvar_export:ajax-organisation-list", kwargs={"project": self.project.sodar_uuid}, + "clinvar_export:ajax-organisation-list", + kwargs={"project": self.project.sodar_uuid}, ) good_users = [ self.superuser, @@ -45,7 +46,8 @@ class TestSubmitterAjaxViews(TestProjectAPIPermissionBase): def test_list(self): url = reverse( - "clinvar_export:ajax-submitter-list", kwargs={"project": self.project.sodar_uuid}, + "clinvar_export:ajax-submitter-list", + kwargs={"project": self.project.sodar_uuid}, ) good_users = [ self.superuser, @@ -64,7 +66,8 @@ class TestAssertionMethodAjaxViews(TestProjectAPIPermissionBase): def test_list(self): url = reverse( - "clinvar_export:ajax-assertionmethod-list", kwargs={"project": self.project.sodar_uuid}, + "clinvar_export:ajax-assertionmethod-list", + kwargs={"project": self.project.sodar_uuid}, ) good_users = [ self.superuser, @@ -83,7 +86,8 @@ class TestIndividualAjaxViews(TestProjectAPIPermissionBase): def test_list(self): url = reverse( - "clinvar_export:ajax-individual-list", kwargs={"project": self.project.sodar_uuid}, + "clinvar_export:ajax-individual-list", + kwargs={"project": self.project.sodar_uuid}, ) good_users = [ self.superuser, @@ -102,7 +106,8 @@ class TestFamilyAjaxViews(TestProjectAPIPermissionBase): def test_list(self): url = reverse( - "clinvar_export:ajax-family-list", kwargs={"project": self.project.sodar_uuid}, + "clinvar_export:ajax-family-list", + kwargs={"project": self.project.sodar_uuid}, ) good_users = [ self.superuser, @@ -223,7 +228,10 @@ def test_delete(self): submissionset = SubmissionSetFactory(project=self.project) url = reverse( "clinvar_export:ajax-submissionset-retrieve-update-destroy", - kwargs={"project": self.project.sodar_uuid, "submissionset": submissionset.sodar_uuid,}, + kwargs={ + "project": self.project.sodar_uuid, + "submissionset": submissionset.sodar_uuid, + }, ) for user in bad_users: self.assert_response(url, [user], 403, method="DELETE") @@ -284,20 +292,29 @@ def test_create(self): good_users, 201, method="POST", - data={"submission_set": self.submission.submission_set.sodar_uuid, **data,}, + data={ + "submission_set": self.submission.submission_set.sodar_uuid, + **data, + }, ) self.assert_response( url, bad_users, 403, method="POST", - data={"submission_set": self.submission.submission_set.sodar_uuid, **data,}, + data={ + "submission_set": self.submission.submission_set.sodar_uuid, + **data, + }, ) def test_retrieve(self): url = reverse( "clinvar_export:ajax-submission-retrieve-update-destroy", - kwargs={"project": self.project.sodar_uuid, "submission": self.submission.sodar_uuid,}, + kwargs={ + "project": self.project.sodar_uuid, + "submission": self.submission.sodar_uuid, + }, ) good_users = [ self.superuser, @@ -313,7 +330,10 @@ def test_retrieve(self): def test_update(self): url = reverse( "clinvar_export:ajax-submission-retrieve-update-destroy", - kwargs={"project": self.project.sodar_uuid, "submission": self.submission.sodar_uuid,}, + kwargs={ + "project": self.project.sodar_uuid, + "submission": self.submission.sodar_uuid, + }, ) good_users = [ self.superuser, @@ -341,14 +361,20 @@ def test_delete(self): submission = SubmissionFactory(submission_set__project=self.project) url = reverse( "clinvar_export:ajax-submission-retrieve-update-destroy", - kwargs={"project": self.project.sodar_uuid, "submission": submission.sodar_uuid,}, + kwargs={ + "project": self.project.sodar_uuid, + "submission": submission.sodar_uuid, + }, ) self.assert_response(url, [user], 204, method="DELETE") submission = SubmissionFactory(submission_set__project=self.project) url = reverse( "clinvar_export:ajax-submission-retrieve-update-destroy", - kwargs={"project": self.project.sodar_uuid, "submission": submission.sodar_uuid,}, + kwargs={ + "project": self.project.sodar_uuid, + "submission": submission.sodar_uuid, + }, ) for user in bad_users: self.assert_response(url, [user], 403, method="DELETE") @@ -639,7 +665,12 @@ class TestQueryOmimAjaxViews(TestProjectAPIPermissionBase): def test(self): hpo_record = HpoFactory() url = ( - reverse("clinvar_export:query-omim-term", kwargs={"project": self.project.sodar_uuid,},) + reverse( + "clinvar_export:query-omim-term", + kwargs={ + "project": self.project.sodar_uuid, + }, + ) + "?query=" + hpo_record.database_id ) @@ -662,7 +693,12 @@ class TestQueryHpoAjaxViews(TestProjectAPIPermissionBase): def test(self): hpo_record = HpoNameFactory() url = ( - reverse("clinvar_export:query-hpo-term", kwargs={"project": self.project.sodar_uuid,},) + reverse( + "clinvar_export:query-hpo-term", + kwargs={ + "project": self.project.sodar_uuid, + }, + ) + "?query=" + hpo_record.hpo_id ) diff --git a/clinvar_export/tests/test_views.py b/clinvar_export/tests/test_views.py index 4460643e7..a6566dbde 100644 --- a/clinvar_export/tests/test_views.py +++ b/clinvar_export/tests/test_views.py @@ -1,5 +1,4 @@ from django.urls import reverse - from projectroles.tests.test_permissions import TestProjectPermissionBase diff --git a/clinvar_export/tests/test_views_ajax.py b/clinvar_export/tests/test_views_ajax.py index d58b71ae7..7e08d5c33 100644 --- a/clinvar_export/tests/test_views_ajax.py +++ b/clinvar_export/tests/test_views_ajax.py @@ -7,29 +7,30 @@ from geneinfo.tests.factories import HpoFactory, HpoNameFactory from variants.tests.factories import ( + AcmgCriteriaRatingFactory, CaseFactory, SmallVariantCommentFactory, SmallVariantFactory, SmallVariantFlagsFactory, - AcmgCriteriaRatingFactory, -) -from .factories import ( - SubmissionSetFactory, - SubmissionFactory, - SubmissionIndividualFactory, - OrganisationFactory, - SubmittingOrgFactory, - SubmitterFactory, ) + from ..models import ( - Individual, - Family, - SubmissionSet, AssertionMethod, + Family, + Individual, Submission, - create_families_and_individuals, SubmissionIndividual, + SubmissionSet, SubmittingOrg, + create_families_and_individuals, +) +from .factories import ( + OrganisationFactory, + SubmissionFactory, + SubmissionIndividualFactory, + SubmissionSetFactory, + SubmitterFactory, + SubmittingOrgFactory, ) RE_UUID4 = re.compile(r"^[0-9a-f-]+$") @@ -449,7 +450,10 @@ def setUp(self): def test_retrieve(self): url = reverse( "clinvar_export:ajax-submission-retrieve-update-destroy", - kwargs={"project": self.project.sodar_uuid, "submission": self.submission.sodar_uuid,}, + kwargs={ + "project": self.project.sodar_uuid, + "submission": self.submission.sodar_uuid, + }, ) with self.login(self.contributor_as.user): response = self.client.get(url) @@ -491,7 +495,10 @@ def test_retrieve(self): def test_update(self): url = reverse( "clinvar_export:ajax-submission-retrieve-update-destroy", - kwargs={"project": self.project.sodar_uuid, "submission": self.submission.sodar_uuid,}, + kwargs={ + "project": self.project.sodar_uuid, + "submission": self.submission.sodar_uuid, + }, ) data = { "age_of_onset": "Antenatal", @@ -530,7 +537,10 @@ def test_update(self): def test_destroy(self): url = reverse( "clinvar_export:ajax-submission-retrieve-update-destroy", - kwargs={"project": self.project.sodar_uuid, "submission": self.submission.sodar_uuid,}, + kwargs={ + "project": self.project.sodar_uuid, + "submission": self.submission.sodar_uuid, + }, ) with self.login(self.contributor_as.user): response = self.client.delete(url) @@ -825,7 +835,12 @@ class TestQueryOmimAjaxViews(TestProjectAPIPermissionBase): def test_query(self): hpo_record = HpoFactory() url = ( - reverse("clinvar_export:query-omim-term", kwargs={"project": self.project.sodar_uuid,},) + reverse( + "clinvar_export:query-omim-term", + kwargs={ + "project": self.project.sodar_uuid, + }, + ) + "?query=" + hpo_record.database_id ) @@ -853,7 +868,12 @@ class TestQueryHpoAjaxViews(TestProjectAPIPermissionBase): def test_query(self): hpo_name_record = HpoNameFactory() url = ( - reverse("clinvar_export:query-hpo-term", kwargs={"project": self.project.sodar_uuid,},) + reverse( + "clinvar_export:query-hpo-term", + kwargs={ + "project": self.project.sodar_uuid, + }, + ) + "?query=" + hpo_name_record.hpo_id ) diff --git a/clinvar_export/urls.py b/clinvar_export/urls.py index d6059c068..b78d7abf1 100644 --- a/clinvar_export/urls.py +++ b/clinvar_export/urls.py @@ -1,6 +1,7 @@ """URL configuration for the ``clinvar_export`` app.""" from django.conf.urls import url + from . import views, views_ajax app_name = "clinvar_export" diff --git a/clinvar_export/views.py b/clinvar_export/views.py index f3b34ed1f..9ffd3fbbc 100644 --- a/clinvar_export/views.py +++ b/clinvar_export/views.py @@ -9,10 +9,10 @@ from django.urls import reverse from django.views.generic import ListView from projectroles.views import ( + LoggedInPermissionMixin, LoginRequiredMixin, ProjectContextMixin, ProjectPermissionMixin, - LoggedInPermissionMixin, ) from .models import SubmissionSet @@ -25,8 +25,7 @@ class SubmissionSetView( ProjectContextMixin, ListView, ): - """Display list of cohorts. - """ + """Display list of cohorts.""" permission_required = "variants.view_data" template_name = "submission_set/entrypoint.html" diff --git a/clinvar_export/views_ajax.py b/clinvar_export/views_ajax.py index 210291b7a..7da164a38 100644 --- a/clinvar_export/views_ajax.py +++ b/clinvar_export/views_ajax.py @@ -8,52 +8,48 @@ from django.contrib.postgres.aggregates import ArrayAgg from django.db.models import Q -from django.http import JsonResponse, HttpResponse +from django.http import HttpResponse, JsonResponse from django.views.generic import DetailView from lxml import etree from projectroles.views import ( - LoginRequiredMixin, LoggedInPermissionMixin, - ProjectPermissionMixin, + LoginRequiredMixin, ProjectContextMixin, + ProjectPermissionMixin, ) -from projectroles.views_ajax import SODARBaseProjectAjaxView, SODARBaseAjaxView +from projectroles.views_ajax import SODARBaseAjaxView, SODARBaseProjectAjaxView from projectroles.views_api import APIProjectContextMixin -from rest_framework.generics import ( - ListCreateAPIView, - RetrieveUpdateDestroyAPIView, - ListAPIView, -) +from rest_framework.generics import ListAPIView, ListCreateAPIView, RetrieveUpdateDestroyAPIView from rest_framework.response import Response -from variants.helpers import get_engine from geneinfo.models import Hpo, HpoName +from variants.helpers import get_engine from variants.queries import SmallVariantUserAnnotationQuery -from .clinvar_xml import SubmissionXmlGenerator, XSD_URL_1_7 + +from .clinvar_xml import XSD_URL_1_7, SubmissionXmlGenerator from .models import ( - Case, - SubmissionSet, - Submission, - Organisation, - Submitter, AssertionMethod, - SubmissionIndividual, - Individual, Family, - create_families_and_individuals, + Individual, + Organisation, + Submission, + SubmissionIndividual, + SubmissionSet, + Submitter, SubmittingOrg, + create_families_and_individuals, ) from .serializers import ( - SubmissionSetSerializer, - SubmissionSerializer, - OrganisationSerializer, - SubmitterSerializer, + AnnotatedSmallVariantsSerializer, AssertionMethodSerializer, - SubmissionIndividualSerializer, - IndividualSerializer, FamilySerializer, + IndividualSerializer, + OrganisationSerializer, + SubmissionIndividualSerializer, + SubmissionSerializer, + SubmissionSetSerializer, + SubmitterSerializer, SubmittingOrgSerializer, - AnnotatedSmallVariantsSerializer, ) @@ -418,7 +414,8 @@ def get(self, *_args, **_kwargs): class AnnotatedSmallVariantsApiView( - APIProjectContextMixin, SODARBaseProjectAjaxView, + APIProjectContextMixin, + SODARBaseProjectAjaxView, ): """Retrieve user annotations for all families in the current project.""" diff --git a/cohorts/admin.py b/cohorts/admin.py index 8c38f3f3d..846f6b406 100644 --- a/cohorts/admin.py +++ b/cohorts/admin.py @@ -1,3 +1 @@ -from django.contrib import admin - # Register your models here. diff --git a/cohorts/forms.py b/cohorts/forms.py index 6df8e7122..6a7c6d87f 100644 --- a/cohorts/forms.py +++ b/cohorts/forms.py @@ -1,7 +1,8 @@ from django import forms -from django.forms import ModelMultipleChoiceField, MultipleChoiceField +from django.forms import ModelMultipleChoiceField from django.forms.models import ModelChoiceIterator from projectroles.app_settings import AppSettingAPI + from cohorts.models import Cohort from variants.models import Case diff --git a/cohorts/migrations/0001_initial.py b/cohorts/migrations/0001_initial.py index b52251615..c35cbd3ca 100644 --- a/cohorts/migrations/0001_initial.py +++ b/cohorts/migrations/0001_initial.py @@ -2,10 +2,11 @@ # Generated by Django 1.11.29 on 2020-06-26 12:43 from __future__ import unicode_literals +import uuid + from django.conf import settings from django.db import migrations, models import django.db.models.deletion -import uuid class Migration(migrations.Migration): @@ -63,6 +64,8 @@ class Migration(migrations.Migration): ), ), ], - options={"ordering": ["date_modified"],}, + options={ + "ordering": ["date_modified"], + }, ), ] diff --git a/cohorts/migrations/0003_auto_20200701_1255.py b/cohorts/migrations/0003_auto_20200701_1255.py index 09fa6caba..ac3db8037 100644 --- a/cohorts/migrations/0003_auto_20200701_1255.py +++ b/cohorts/migrations/0003_auto_20200701_1255.py @@ -12,5 +12,8 @@ class Migration(migrations.Migration): ] operations = [ - migrations.AlterModelOptions(name="cohort", options={"ordering": ["-date_modified"]},), + migrations.AlterModelOptions( + name="cohort", + options={"ordering": ["-date_modified"]}, + ), ] diff --git a/cohorts/migrations/0004_auto_20200710_1548.py b/cohorts/migrations/0004_auto_20200710_1548.py index 7fa7443af..85c8df07d 100644 --- a/cohorts/migrations/0004_auto_20200710_1548.py +++ b/cohorts/migrations/0004_auto_20200710_1548.py @@ -2,9 +2,10 @@ # Generated by Django 1.11.29 on 2020-07-10 15:48 from __future__ import unicode_literals -from django.db import migrations, models import uuid +from django.db import migrations, models + class Migration(migrations.Migration): diff --git a/cohorts/models.py b/cohorts/models.py index c6f6461e1..987b99430 100644 --- a/cohorts/models.py +++ b/cohorts/models.py @@ -1,21 +1,18 @@ from collections import defaultdict +import uuid as uuid_object +from django.conf import settings from django.db import models # Create your models here. from django.urls import reverse -from django.conf import settings -import uuid as uuid_object - - #: Django user model. AUTH_USER_MODEL = getattr(settings, "AUTH_USER_MODEL", "auth.User") class Cohort(models.Model): - """Class for ``Cohort`` model. - """ + """Class for ``Cohort`` model.""" #: DateTime of creation. date_created = models.DateTimeField(auto_now_add=True, help_text="DateTime of creation") diff --git a/cohorts/templatetags/cohorts_tags.py b/cohorts/templatetags/cohorts_tags.py index db8694e58..bcd21b5b1 100644 --- a/cohorts/templatetags/cohorts_tags.py +++ b/cohorts/templatetags/cohorts_tags.py @@ -1,7 +1,5 @@ from django import template -from variants.models import Case - register = template.Library() diff --git a/cohorts/tests/factories.py b/cohorts/tests/factories.py index 2929e8958..2e5c50120 100644 --- a/cohorts/tests/factories.py +++ b/cohorts/tests/factories.py @@ -1,16 +1,12 @@ import factory -from projectroles.forms import PROJECT_ROLE_OWNER, PROJECT_ROLE_CONTRIBUTOR +from projectroles.forms import PROJECT_ROLE_CONTRIBUTOR, PROJECT_ROLE_OWNER from projectroles.models import Role from projectroles.tests.test_models import RoleAssignmentMixin from test_plus import TestCase from cohorts.models import Cohort from variants.models import Case -from variants.tests.factories import ( - ProjectFactory, - SmallVariantFactory, - CaseWithVariantSetFactory, -) +from variants.tests.factories import CaseWithVariantSetFactory, ProjectFactory, SmallVariantFactory class CohortFactory(factory.django.DjangoModelFactory): diff --git a/cohorts/tests/test_cohorts_tags.py b/cohorts/tests/test_cohorts_tags.py index 7ddcebd92..e27f42f77 100644 --- a/cohorts/tests/test_cohorts_tags.py +++ b/cohorts/tests/test_cohorts_tags.py @@ -1,6 +1,7 @@ +from django.template import Context, Template + from cohorts.models import Cohort from cohorts.tests.factories import TestCohortBase -from django.template import Context, Template class TestTemplateTagsCohortsTags(TestCohortBase): diff --git a/cohorts/tests/test_models.py b/cohorts/tests/test_models.py index b65ea16e8..b787841a4 100644 --- a/cohorts/tests/test_models.py +++ b/cohorts/tests/test_models.py @@ -1,5 +1,4 @@ -from cohorts.tests.factories import CohortFactory -from cohorts.tests.factories import TestCohortBase +from cohorts.tests.factories import CohortFactory, TestCohortBase class TestTemplateTagsCohortsTags(TestCohortBase): diff --git a/cohorts/tests/test_views.py b/cohorts/tests/test_views.py index 7287d92db..fc5fc6e99 100644 --- a/cohorts/tests/test_views.py +++ b/cohorts/tests/test_views.py @@ -163,10 +163,15 @@ def test_cohort_create_submit_form_as_superuser_project1_all_possible_cases(self } with self.login(user): response = self.client.post( - reverse("cohorts:create", kwargs={"project": project.sodar_uuid}), form_data, + reverse("cohorts:create", kwargs={"project": project.sodar_uuid}), + form_data, ) self.assertRedirects( - response, reverse("cohorts:list", kwargs={"project": project.sodar_uuid},), + response, + reverse( + "cohorts:list", + kwargs={"project": project.sodar_uuid}, + ), ) self.assertEqual(Cohort.objects.count(), 1) cohort = Cohort.objects.first() @@ -185,7 +190,8 @@ def test_cohort_create_submit_form_as_superuser_project2_all_possible_cases(self } with self.login(user): self.client.post( - reverse("cohorts:create", kwargs={"project": project.sodar_uuid}), form_data, + reverse("cohorts:create", kwargs={"project": project.sodar_uuid}), + form_data, ) self.assertEqual(Cohort.objects.count(), 1) cohort = Cohort.objects.first() @@ -204,7 +210,8 @@ def test_cohort_create_submit_form_as_contributor_project2_all_possible_cases(se } with self.login(user): self.client.post( - reverse("cohorts:create", kwargs={"project": project.sodar_uuid}), form_data, + reverse("cohorts:create", kwargs={"project": project.sodar_uuid}), + form_data, ), self.assertEqual(Cohort.objects.count(), 1) cohort = Cohort.objects.first() @@ -276,7 +283,11 @@ def test_cohort_update_submit_form_as_superuser(self): form_data, ) self.assertRedirects( - response, reverse("cohorts:list", kwargs={"project": project.sodar_uuid},), + response, + reverse( + "cohorts:list", + kwargs={"project": project.sodar_uuid}, + ), ) self.assertEqual(Cohort.objects.count(), 1) cohort_altered = Cohort.objects.first() @@ -308,7 +319,11 @@ def test_cohort_update_submit_form_as_contributor(self): form_data, ) self.assertRedirects( - response, reverse("cohorts:list", kwargs={"project": project.sodar_uuid},), + response, + reverse( + "cohorts:list", + kwargs={"project": project.sodar_uuid}, + ), ) self.assertEqual(Cohort.objects.count(), 1) cohort_altered = Cohort.objects.first() @@ -384,7 +399,11 @@ def test_cohort_update_submit_form_as_superuser_for_cohort_by_contributor(self): form_data, ) self.assertRedirects( - response, reverse("cohorts:list", kwargs={"project": project.sodar_uuid},), + response, + reverse( + "cohorts:list", + kwargs={"project": project.sodar_uuid}, + ), ) self.assertEqual(Cohort.objects.count(), 1) cohort_altered = Cohort.objects.first() @@ -416,7 +435,11 @@ def test_cohort_update_submit_form_as_contributor_for_cohort_by_superuser(self): form_data, ) self.assertRedirects( - response, reverse("cohorts:list", kwargs={"project": project.sodar_uuid},), + response, + reverse( + "cohorts:list", + kwargs={"project": project.sodar_uuid}, + ), ) messages = [m.message for m in get_messages(response.wsgi_request)] self.assertEqual("Can't update other user's cohort.", messages[0]) @@ -469,7 +492,11 @@ def test_cohort_delete_submit_delete_as_superuser(self): ), ) self.assertRedirects( - response, reverse("cohorts:list", kwargs={"project": project.sodar_uuid},) + response, + reverse( + "cohorts:list", + kwargs={"project": project.sodar_uuid}, + ), ) self.assertEqual(Cohort.objects.count(), 0) @@ -486,7 +513,11 @@ def test_cohort_delete_submit_delete_as_contributor(self): ), ) self.assertRedirects( - response, reverse("cohorts:list", kwargs={"project": project.sodar_uuid},) + response, + reverse( + "cohorts:list", + kwargs={"project": project.sodar_uuid}, + ), ) self.assertEqual(Cohort.objects.count(), 0) @@ -533,7 +564,11 @@ def test_cohort_delete_submit_delete_as_superuser_for_cohort_by_contributor(self ), ) self.assertRedirects( - response, reverse("cohorts:list", kwargs={"project": project.sodar_uuid},) + response, + reverse( + "cohorts:list", + kwargs={"project": project.sodar_uuid}, + ), ) messages = [m.message for m in get_messages(response.wsgi_request)] self.assertEqual("Cohort %s deleted." % cohort.name, messages[0]) @@ -551,7 +586,11 @@ def test_cohort_delete_submit_delete_as_contributor_for_cohort_by_superuser(self ), ) self.assertRedirects( - response, reverse("cohorts:list", kwargs={"project": project.sodar_uuid},) + response, + reverse( + "cohorts:list", + kwargs={"project": project.sodar_uuid}, + ), ) messages = [m.message for m in get_messages(response.wsgi_request)] self.assertEqual("Can't delete other user's cohort.", messages[0]) diff --git a/cohorts/urls.py b/cohorts/urls.py index d5632f2d5..6fc577aec 100644 --- a/cohorts/urls.py +++ b/cohorts/urls.py @@ -2,12 +2,17 @@ """ from django.conf.urls import url + from . import views app_name = "cohorts" urlpatterns = [ - url(regex=r"^(?P[0-9a-f-]+)/$", view=views.CohortView.as_view(), name="list",), + url( + regex=r"^(?P[0-9a-f-]+)/$", + view=views.CohortView.as_view(), + name="list", + ), url( regex=r"^(?P[0-9a-f-]+)/create/$", view=views.CohortCreateView.as_view(), diff --git a/cohorts/views.py b/cohorts/views.py index 872515eeb..2fce676fe 100644 --- a/cohorts/views.py +++ b/cohorts/views.py @@ -1,16 +1,14 @@ from django.contrib import messages from django.http import HttpResponseRedirect - - from django.urls import reverse from django.utils.safestring import mark_safe from django.views.generic import ListView from django.views.generic.edit import CreateView, DeleteView, UpdateView from projectroles.views import ( + LoggedInPermissionMixin, LoginRequiredMixin, ProjectContextMixin, ProjectPermissionMixin, - LoggedInPermissionMixin, ) from cohorts.forms import CohortForm @@ -25,8 +23,7 @@ class CohortView( ProjectContextMixin, ListView, ): - """Display list of cohorts. - """ + """Display list of cohorts.""" permission_required = "variants.view_data" template_name = "cohorts/cohort_list.html" @@ -40,8 +37,7 @@ class CohortCreateView( ProjectContextMixin, CreateView, ): - """Create cohort. - """ + """Create cohort.""" permission_required = "variants.view_data" template_name = "cohorts/cohort_create.html" @@ -83,8 +79,7 @@ class CohortUpdateView( ProjectContextMixin, UpdateView, ): - """Update cohort. - """ + """Update cohort.""" permission_required = "variants.view_data" template_name = "cohorts/cohort_update.html" @@ -132,8 +127,7 @@ class CohortDeleteView( ProjectContextMixin, DeleteView, ): - """Delete cohort. - """ + """Delete cohort.""" permission_required = "variants.view_data" template_name = "cohorts/cohort_confirm_delete.html" diff --git a/config/celery.py b/config/celery.py index e2aa22254..798687551 100644 --- a/config/celery.py +++ b/config/celery.py @@ -1,4 +1,5 @@ import os + from celery import Celery # Set the default Django settings module for the 'celery' program. diff --git a/config/settings/base.py b/config/settings/base.py index 76a2c6e60..708d0e0ae 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -10,11 +10,12 @@ import logging import os -import environ import re import sys from dotenv import load_dotenv +import environ + from varfish import __version__ as varfish_version logger = logging.getLogger(__name__) @@ -203,7 +204,13 @@ # List of apps to include in logging LOGGING_APPS = env.list( "LOGGING_APPS", - default=["projectroles", "siteinfo", "sodarcache", "taskflowbackend", "timeline",], + default=[ + "projectroles", + "siteinfo", + "sodarcache", + "taskflowbackend", + "timeline", + ], ) # Path for file logging. If not set, will log only to console @@ -219,7 +226,11 @@ def set_logging(level=None): "propagate": True, } log_handlers = { - "console": {"level": level, "class": "logging.StreamHandler", "formatter": "simple",} + "console": { + "level": level, + "class": "logging.StreamHandler", + "formatter": "simple", + } } if LOGGING_FILE_PATH: log_handlers["file"] = { @@ -279,9 +290,9 @@ def set_logging(level=None): # ------------------------------------------------------------------------------ if env.bool("ENABLE_SENTRY", default=False): import sentry_sdk + from sentry_sdk.integrations.celery import CeleryIntegration from sentry_sdk.integrations.django import DjangoIntegration from sentry_sdk.integrations.redis import RedisIntegration - from sentry_sdk.integrations.celery import CeleryIntegration from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration SENTRY_DSN = "%s?verify_ssl=0" % env.str("SENTRY_DSN") @@ -543,7 +554,8 @@ def set_logging(level=None): # Varfish: MutationTaster URL VARFISH_MUTATIONTASTER_REST_API_URL = env.str( - "VARFISH_MUTATIONTASTER_REST_API_URL", "https://www.genecascade.org/MTc85/MT_API.cgi", + "VARFISH_MUTATIONTASTER_REST_API_URL", + "https://www.genecascade.org/MTc85/MT_API.cgi", ) VARFISH_MUTATIONTASTER_BATCH_VARS = env.int("VARFISH_MUTATIONTASTER_BATCH_VARS", 50) VARFISH_MUTATIONTASTER_MAX_VARS = env.int("VARFISH_MUTATIONTASTER_MAX_VARS", 500) @@ -707,7 +719,11 @@ def set_logging(debug): "propagate": True, } log_handlers = { - "console": {"level": "DEBUG", "class": "logging.StreamHandler", "formatter": "simple",} + "console": { + "level": "DEBUG", + "class": "logging.StreamHandler", + "formatter": "simple", + } } if LOGGING_FILE_PATH: log_handlers["file"] = { @@ -733,8 +749,9 @@ def set_logging(debug): if ENABLE_LDAP: import itertools - import ldap + from django_auth_ldap.config import LDAPSearch + import ldap # Default values LDAP_DEFAULT_CONN_OPTIONS = {ldap.OPT_REFERRALS: 0} @@ -752,7 +769,9 @@ def set_logging(debug): AUTH_LDAP_CONNECTION_OPTIONS = LDAP_DEFAULT_CONN_OPTIONS AUTH_LDAP_USER_SEARCH = LDAPSearch( - env.str("AUTH_LDAP_USER_SEARCH_BASE", None), ldap.SCOPE_SUBTREE, LDAP_DEFAULT_FILTERSTR, + env.str("AUTH_LDAP_USER_SEARCH_BASE", None), + ldap.SCOPE_SUBTREE, + LDAP_DEFAULT_FILTERSTR, ) AUTH_LDAP_USER_ATTR_MAP = LDAP_DEFAULT_ATTR_MAP AUTH_LDAP_USERNAME_DOMAIN = env.str("AUTH_LDAP_USERNAME_DOMAIN", None) @@ -780,7 +799,8 @@ def set_logging(debug): AUTHENTICATION_BACKENDS = tuple( itertools.chain( - ("projectroles.auth_backends.SecondaryLDAPBackend",), AUTHENTICATION_BACKENDS, + ("projectroles.auth_backends.SecondaryLDAPBackend",), + AUTHENTICATION_BACKENDS, ) ) @@ -826,7 +846,10 @@ def set_logging(debug): }, "service": { "sp": { - "idp": env.str("SAML_CLIENT_IDP", "https://sso.hpc.bihealth.org/auth/realms/cubi",), + "idp": env.str( + "SAML_CLIENT_IDP", + "https://sso.hpc.bihealth.org/auth/realms/cubi", + ), # Keycloak expects client signature "authn_requests_signed": "true", # Enforce POST binding which is required by keycloak @@ -916,7 +939,11 @@ def set_logging(debug): # WEBPACK / VUE.JS CONFIGURATION # ------------------------------------------------------------------------------ -WEBPACK_LOADER = {"VARFISH_VUE": {"STATS_FILE": ROOT_DIR("varfish/vueapp/webpack-stats.json"),}} +WEBPACK_LOADER = { + "VARFISH_VUE": { + "STATS_FILE": ROOT_DIR("varfish/vueapp/webpack-stats.json"), + } +} # ICONIFY CONFIGURATION # ------------------------------------------------------------------------------ diff --git a/config/settings/production.py b/config/settings/production.py index 74b1a4f81..185795d62 100644 --- a/config/settings/production.py +++ b/config/settings/production.py @@ -11,11 +11,7 @@ """ - -import logging - - -from .base import * # noqa +from .base import * # SECRET CONFIGURATION # ------------------------------------------------------------------------------ diff --git a/config/urls.py b/config/urls.py index 5b8484060..f43552481 100644 --- a/config/urls.py +++ b/config/urls.py @@ -5,15 +5,15 @@ from django.contrib.auth import views as auth_views from django.http import HttpResponse from django.shortcuts import render -from django.views.generic import TemplateView from django.views import defaults as default_views - +from django.views.generic import TemplateView import django_saml2_auth.views from djproxy.views import HttpProxy from projectroles.views import HomeView as ProjectRolesHomeView -from variants.views import KioskHomeView from sentry_sdk import last_event_id +from variants.views import KioskHomeView + def handler500(request, *args, **argv): if request.user and "User" in str(type(request.user)): diff --git a/extra_annos/migrations/0002_auto_20200714_1411.py b/extra_annos/migrations/0002_auto_20200714_1411.py index 8b9b80a3e..41dffce57 100644 --- a/extra_annos/migrations/0002_auto_20200714_1411.py +++ b/extra_annos/migrations/0002_auto_20200714_1411.py @@ -12,5 +12,8 @@ class Migration(migrations.Migration): ] operations = [ - migrations.AlterModelOptions(name="extraannofield", options={"ordering": ("field",)},), + migrations.AlterModelOptions( + name="extraannofield", + options={"ordering": ("field",)}, + ), ] diff --git a/extra_annos/migrations/0003_alter_extraanno_anno_data.py b/extra_annos/migrations/0003_alter_extraanno_anno_data.py index a9eca4d8a..a1dfa425e 100644 --- a/extra_annos/migrations/0003_alter_extraanno_anno_data.py +++ b/extra_annos/migrations/0003_alter_extraanno_anno_data.py @@ -11,6 +11,8 @@ class Migration(migrations.Migration): operations = [ migrations.AlterField( - model_name="extraanno", name="anno_data", field=models.JSONField(default=dict), + model_name="extraanno", + name="anno_data", + field=models.JSONField(default=dict), ), ] diff --git a/extra_annos/migrations/0004_alter_extraanno_anno_data.py b/extra_annos/migrations/0004_alter_extraanno_anno_data.py index 35bd5607d..f5fcd55aa 100644 --- a/extra_annos/migrations/0004_alter_extraanno_anno_data.py +++ b/extra_annos/migrations/0004_alter_extraanno_anno_data.py @@ -1,6 +1,7 @@ # Generated by Django 3.2.9 on 2021-11-29 14:43 from django.db import migrations + import varfish.utils @@ -12,6 +13,8 @@ class Migration(migrations.Migration): operations = [ migrations.AlterField( - model_name="extraanno", name="anno_data", field=varfish.utils.JSONField(default=dict), + model_name="extraanno", + name="anno_data", + field=varfish.utils.JSONField(default=dict), ), ] diff --git a/extra_annos/models.py b/extra_annos/models.py index 21d0b5505..3d3d8ff6d 100644 --- a/extra_annos/models.py +++ b/extra_annos/models.py @@ -1,10 +1,9 @@ import typing +import attr from django.db import models from postgres_copy import CopyManager -import attr - from varfish.utils import JSONField diff --git a/extra_annos/views.py b/extra_annos/views.py index 3977bbbeb..6fa1b52f0 100644 --- a/extra_annos/views.py +++ b/extra_annos/views.py @@ -1,6 +1,6 @@ import typing -from .models import ExtraAnnoField, ExtraAnno, AugmentedExtraAnno +from .models import AugmentedExtraAnno, ExtraAnno, ExtraAnnoField class ExtraAnnosMixin: diff --git a/frequencies/migrations/0005_auto_20200123_1033.py b/frequencies/migrations/0005_auto_20200123_1033.py index 97b64703c..8010b5efb 100644 --- a/frequencies/migrations/0005_auto_20200123_1033.py +++ b/frequencies/migrations/0005_auto_20200123_1033.py @@ -37,7 +37,9 @@ class Migration(migrations.Migration): ("ac_het_min", models.FloatField(null=True)), ("ac_het_max", models.FloatField(null=True)), ], - options={"abstract": False,}, + options={ + "abstract": False, + }, ), migrations.CreateModel( name="Mitomap", @@ -59,7 +61,9 @@ class Migration(migrations.Migration): ("an", models.IntegerField()), ("af", models.FloatField()), ], - options={"abstract": False,}, + options={ + "abstract": False, + }, ), migrations.CreateModel( name="MtDb", @@ -87,7 +91,9 @@ class Migration(migrations.Migration): ("aa_change", models.CharField(max_length=32, null=True)), ("synonymous", models.NullBooleanField()), ], - options={"abstract": False,}, + options={ + "abstract": False, + }, ), migrations.AlterUniqueTogether( name="mtdb", diff --git a/frequencies/migrations/0006_remove_mtdb_gap.py b/frequencies/migrations/0006_remove_mtdb_gap.py index 666e40229..70ac9de58 100644 --- a/frequencies/migrations/0006_remove_mtdb_gap.py +++ b/frequencies/migrations/0006_remove_mtdb_gap.py @@ -12,5 +12,8 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RemoveField(model_name="mtdb", name="gap",), + migrations.RemoveField( + model_name="mtdb", + name="gap", + ), ] diff --git a/frequencies/migrations/0007_auto_20200211_1625.py b/frequencies/migrations/0007_auto_20200211_1625.py index e9e6eac8c..3a2a9ae18 100644 --- a/frequencies/migrations/0007_auto_20200211_1625.py +++ b/frequencies/migrations/0007_auto_20200211_1625.py @@ -12,8 +12,14 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RenameField(model_name="helixmtdb", old_name="ac", new_name="ac_hom",), + migrations.RenameField( + model_name="helixmtdb", + old_name="ac", + new_name="ac_hom", + ), migrations.AddField( - model_name="helixmtdb", name="is_triallelic", field=models.BooleanField(default=False), + model_name="helixmtdb", + name="is_triallelic", + field=models.BooleanField(default=False), ), ] diff --git a/frequencies/migrations/0008_alter_mtdb_synonymous.py b/frequencies/migrations/0008_alter_mtdb_synonymous.py index f8ad792f9..caeaac990 100644 --- a/frequencies/migrations/0008_alter_mtdb_synonymous.py +++ b/frequencies/migrations/0008_alter_mtdb_synonymous.py @@ -11,6 +11,8 @@ class Migration(migrations.Migration): operations = [ migrations.AlterField( - model_name="mtdb", name="synonymous", field=models.BooleanField(null=True), + model_name="mtdb", + name="synonymous", + field=models.BooleanField(null=True), ), ] diff --git a/frequencies/tests/factories.py b/frequencies/tests/factories.py index 1e17a82a7..1d0f79ba6 100644 --- a/frequencies/tests/factories.py +++ b/frequencies/tests/factories.py @@ -2,7 +2,7 @@ import binning import factory -from ..models import Exac, GnomadExomes, GnomadGenomes, ThousandGenomes, Mitomap, HelixMtDb, MtDb +from ..models import Exac, GnomadExomes, GnomadGenomes, HelixMtDb, Mitomap, MtDb, ThousandGenomes class MacArthurFrequenciesFactoryBase(factory.django.DjangoModelFactory): diff --git a/frequencies/tests/test_models.py b/frequencies/tests/test_models.py index 1ac131472..0ebff5de9 100644 --- a/frequencies/tests/test_models.py +++ b/frequencies/tests/test_models.py @@ -2,12 +2,12 @@ from variants.tests.helpers import QueryTestBase -from ..models import Exac, ThousandGenomes, GnomadExomes, GnomadGenomes +from ..models import Exac, GnomadExomes, GnomadGenomes, ThousandGenomes from .factories import ( ExacFactory, - ThousandGenomesFactory, GnomadExomesFactory, GnomadGenomesFactory, + ThousandGenomesFactory, ) #: Symbol of gene to query the variants for. diff --git a/frequencies/views.py b/frequencies/views.py index d14116d41..5846d9a40 100644 --- a/frequencies/views.py +++ b/frequencies/views.py @@ -1,4 +1,4 @@ -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned +from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist from .models import FREQUENCY_DB_INFO diff --git a/geneinfo/admin.py b/geneinfo/admin.py index 198b324ef..3c9213dd3 100644 --- a/geneinfo/admin.py +++ b/geneinfo/admin.py @@ -1,11 +1,11 @@ from django.contrib import admin from .models import ( + Acmg, Hgnc, - Mim2geneMedgen, Hpo, - Acmg, HpoName, + Mim2geneMedgen, NcbiGeneInfo, NcbiGeneRif, RefseqToHgnc, diff --git a/geneinfo/migrations/0013_materialized_view_geneidtoinheritance.py b/geneinfo/migrations/0013_materialized_view_geneidtoinheritance.py index cb930d235..4a9cf6ad2 100644 --- a/geneinfo/migrations/0013_materialized_view_geneidtoinheritance.py +++ b/geneinfo/migrations/0013_materialized_view_geneidtoinheritance.py @@ -2,9 +2,8 @@ # Generated by Django 1.11.20 on 2019-04-30 14:28 from __future__ import unicode_literals -from django.db import migrations, models from django.conf import settings - +from django.db import migrations, models operations = [ migrations.CreateModel( diff --git a/geneinfo/migrations/0015_mgimapping.py b/geneinfo/migrations/0015_mgimapping.py index 4f4269d05..5f284c360 100644 --- a/geneinfo/migrations/0015_mgimapping.py +++ b/geneinfo/migrations/0015_mgimapping.py @@ -2,11 +2,10 @@ # Generated by Django 1.11.20 on 2019-08-13 13:39 from __future__ import unicode_literals -import django.contrib.postgres.fields from django.conf import settings +import django.contrib.postgres.fields from django.db import migrations, models - operations = [ migrations.CreateModel( name="MgiMapping", diff --git a/geneinfo/migrations/0016_mgimapping_index.py b/geneinfo/migrations/0016_mgimapping_index.py index 816e25702..e96fc18c7 100644 --- a/geneinfo/migrations/0016_mgimapping_index.py +++ b/geneinfo/migrations/0016_mgimapping_index.py @@ -2,10 +2,8 @@ # Generated by Django 1.11.20 on 2019-08-13 13:39 from __future__ import unicode_literals -import django.contrib.postgres.fields from django.conf import settings -from django.db import migrations, models - +from django.db import migrations if not settings.IS_TESTING: operations = [ diff --git a/geneinfo/migrations/0022_materialized_view_geneidinhpo.py b/geneinfo/migrations/0022_materialized_view_geneidinhpo.py index 2a0949cbf..3768e3b3d 100644 --- a/geneinfo/migrations/0022_materialized_view_geneidinhpo.py +++ b/geneinfo/migrations/0022_materialized_view_geneidinhpo.py @@ -2,8 +2,8 @@ # Generated by Django 1.11.29 on 2020-07-10 16:03 from __future__ import unicode_literals -from django.db import migrations, models from django.conf import settings +from django.db import migrations, models operations = [ migrations.CreateModel( diff --git a/geneinfo/migrations/0023_materialized_view_geneidinhpo_fix.py b/geneinfo/migrations/0023_materialized_view_geneidinhpo_fix.py index 8e902139c..fbe715b76 100644 --- a/geneinfo/migrations/0023_materialized_view_geneidinhpo_fix.py +++ b/geneinfo/migrations/0023_materialized_view_geneidinhpo_fix.py @@ -2,9 +2,8 @@ # Generated by Django 1.11.29 on 2020-07-10 16:03 from __future__ import unicode_literals -from django.db import migrations, models from django.conf import settings - +from django.db import migrations operations = [] diff --git a/geneinfo/migrations/0025_auto_20211019_0829.py b/geneinfo/migrations/0025_auto_20211019_0829.py index 7a17e4c9e..98b8c4b10 100644 --- a/geneinfo/migrations/0025_auto_20211019_0829.py +++ b/geneinfo/migrations/0025_auto_20211019_0829.py @@ -11,12 +11,18 @@ class Migration(migrations.Migration): operations = [ migrations.AddField( - model_name="hgnc", name="agr", field=models.CharField(max_length=32, null=True), + model_name="hgnc", + name="agr", + field=models.CharField(max_length=32, null=True), ), migrations.AddField( - model_name="hgnc", name="lncipedia", field=models.CharField(max_length=32, null=True), + model_name="hgnc", + name="lncipedia", + field=models.CharField(max_length=32, null=True), ), migrations.AddField( - model_name="hgnc", name="mane_select", field=models.CharField(max_length=64, null=True), + model_name="hgnc", + name="mane_select", + field=models.CharField(max_length=64, null=True), ), ] diff --git a/geneinfo/migrations/0026_hgnc_gtrnadb.py b/geneinfo/migrations/0026_hgnc_gtrnadb.py index 5b7eb2c12..ac6e36659 100644 --- a/geneinfo/migrations/0026_hgnc_gtrnadb.py +++ b/geneinfo/migrations/0026_hgnc_gtrnadb.py @@ -11,6 +11,8 @@ class Migration(migrations.Migration): operations = [ migrations.AddField( - model_name="hgnc", name="gtrnadb", field=models.CharField(max_length=32, null=True), + model_name="hgnc", + name="gtrnadb", + field=models.CharField(max_length=32, null=True), ), ] diff --git a/geneinfo/models.py b/geneinfo/models.py index 06b6600ba..6e1caab8f 100644 --- a/geneinfo/models.py +++ b/geneinfo/models.py @@ -1,6 +1,6 @@ -from django.contrib.postgres.fields import ArrayField -from django.db import models, connection, transaction from django.conf import settings +from django.contrib.postgres.fields import ArrayField +from django.db import connection, models, transaction from postgres_copy import CopyManager @@ -482,8 +482,7 @@ def refresh_geneinfo_geneidtoinheritance(): class GeneIdInHpo(models.Model): - """Materialized view with a list of gene ids that have an HPO entry. - """ + """Materialized view with a list of gene ids that have an HPO entry.""" #: RefSeq gene ID (the view requires entrez id not to be null.) entrez_id = models.CharField(max_length=16, null=False) diff --git a/geneinfo/tests/_test_views.py b/geneinfo/tests/_test_views.py index a127b04b8..c540c0441 100644 --- a/geneinfo/tests/_test_views.py +++ b/geneinfo/tests/_test_views.py @@ -1,12 +1,12 @@ """Tests for the filter view""" from django.urls import reverse +from projectroles.models import Project -from pathways.models import EnsemblToKegg, RefseqToKegg, KeggInfo +from pathways.models import EnsemblToKegg, KeggInfo, RefseqToKegg from variants.tests.test_views import TestViewBase -from projectroles.models import Project -from ..models import Hgnc, Mim2geneMedgen, Hpo +from ..models import Hgnc, Hpo, Mim2geneMedgen PROJECT_DICT = { "title": "project", diff --git a/geneinfo/tests/factories.py b/geneinfo/tests/factories.py index 820720fb2..edf80c5d3 100644 --- a/geneinfo/tests/factories.py +++ b/geneinfo/tests/factories.py @@ -3,21 +3,21 @@ import factory from ..models import ( - Hgnc, - Hpo, - Mim2geneMedgen, Acmg, - RefseqToHgnc, - HpoName, - ExacConstraints, - GnomadConstraints, + EnsemblToGeneSymbol, EnsemblToRefseq, - RefseqToEnsembl, + ExacConstraints, + GeneIdInHpo, GeneIdToInheritance, + GnomadConstraints, + Hgnc, + Hpo, + HpoName, MgiMapping, + Mim2geneMedgen, + RefseqToEnsembl, RefseqToGeneSymbol, - EnsemblToGeneSymbol, - GeneIdInHpo, + RefseqToHgnc, ) diff --git a/geneinfo/urls.py b/geneinfo/urls.py index bce67f491..ef2ec1d0d 100644 --- a/geneinfo/urls.py +++ b/geneinfo/urls.py @@ -1,5 +1,2 @@ -from django.conf.urls import url -from . import views - app_name = "geneinfo" urlpatterns = [] diff --git a/geneinfo/views.py b/geneinfo/views.py index afdb8a791..192091021 100644 --- a/geneinfo/views.py +++ b/geneinfo/views.py @@ -4,20 +4,19 @@ from clinvar.models import ClinvarPathogenicGenes from geneinfo.models import ( - RefseqToHgnc, - Hgnc, - GnomadConstraints, + EnsemblToGeneSymbol, ExacConstraints, - Mim2geneMedgen, + GnomadConstraints, + Hgnc, Hpo, HpoName, + Mim2geneMedgen, RefseqToEnsembl, RefseqToGeneSymbol, - EnsemblToGeneSymbol, + RefseqToHgnc, ) - -RE_OMIM_PARSER = re.compile("^(?:#\d+ )?(.+)$") +RE_OMIM_PARSER = re.compile(r"^(?:#\d+ )?(.+)$") # Modes of inheritance in HPO: https://hpo.jax.org/app/browse/term/HP:0000005 diff --git a/genomicfeatures/admin.py b/genomicfeatures/admin.py index 1a4eedcad..9b6ad534f 100644 --- a/genomicfeatures/admin.py +++ b/genomicfeatures/admin.py @@ -3,9 +3,9 @@ from .models import ( EnsemblRegulatoryFeature, GeneInterval, - TadSet, - TadInterval, TadBoundaryInterval, + TadInterval, + TadSet, VistaEnhancer, ) diff --git a/genomicfeatures/migrations/0001_initial.py b/genomicfeatures/migrations/0001_initial.py index 4c2f164f8..2ffc2c6ac 100644 --- a/genomicfeatures/migrations/0001_initial.py +++ b/genomicfeatures/migrations/0001_initial.py @@ -2,11 +2,12 @@ # Generated by Django 1.11.20 on 2019-04-23 09:40 from __future__ import unicode_literals +import uuid + import django.contrib.postgres.fields import django.contrib.postgres.indexes from django.db import migrations, models import django.db.models.deletion -import uuid class Migration(migrations.Migration): diff --git a/genomicfeatures/models.py b/genomicfeatures/models.py index f9d356b4b..1faeadc1b 100644 --- a/genomicfeatures/models.py +++ b/genomicfeatures/models.py @@ -1,9 +1,6 @@ import uuid as uuid_object -from django.contrib.postgres.fields import ArrayField -from django.contrib.postgres.indexes import GinIndex from django.db import models -from django.db.models import Index from postgres_copy import CopyManager diff --git a/genomicfeatures/tests/factories.py b/genomicfeatures/tests/factories.py index 607b5d327..88f041de9 100644 --- a/genomicfeatures/tests/factories.py +++ b/genomicfeatures/tests/factories.py @@ -4,13 +4,13 @@ import factory from ..models import ( - TadSet, - TadInterval, - TadBoundaryInterval, + VISTA_POSITIVE, EnsemblRegulatoryFeature, - VistaEnhancer, GeneInterval, - VISTA_POSITIVE, + TadBoundaryInterval, + TadInterval, + TadSet, + VistaEnhancer, ) diff --git a/genomicfeatures/tests/test_models.py b/genomicfeatures/tests/test_models.py index ffba5ce6d..7b752d9bd 100644 --- a/genomicfeatures/tests/test_models.py +++ b/genomicfeatures/tests/test_models.py @@ -3,12 +3,12 @@ from django.test import TestCase from .factories import ( - TadSetFactory, - TadIntervalFactory, - TadBoundaryIntervalFactory, EnsemblRegulatoryFeatureFactory, - VistaEnhancerFactory, GeneIntervalFactory, + TadBoundaryIntervalFactory, + TadIntervalFactory, + TadSetFactory, + VistaEnhancerFactory, ) diff --git a/importer/admin.py b/importer/admin.py index 2268cc2c3..e2de21e8b 100644 --- a/importer/admin.py +++ b/importer/admin.py @@ -1,14 +1,14 @@ from django.contrib import admin from .models import ( - ImportInfo, - CaseImportInfo, - VariantSetImportInfo, BamQcFile, - GenotypeFile, - EffectFile, + CaseImportInfo, DatabaseInfoFile, + EffectFile, + GenotypeFile, ImportCaseBgJob, + ImportInfo, + VariantSetImportInfo, ) # Register your models here. diff --git a/importer/management/commands/import_case.py b/importer/management/commands/import_case.py index 800ead503..927e4d0a7 100644 --- a/importer/management/commands/import_case.py +++ b/importer/management/commands/import_case.py @@ -1,23 +1,21 @@ """Django command for importing a case after annotation with ``varfish-annotator``.""" -import os.path import gzip import itertools +import os.path from bgjobs.models import BackgroundJob +from django.conf import settings from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand, CommandError from django.db import transaction -from django.conf import settings - -from svs.models import ImportStructuralVariantBgJob from projectroles.models import Project +from svs.models import ImportStructuralVariantBgJob from svs.tasks import run_import_structural_variants_bg_job from variants.models import ImportVariantsBgJob from variants.tasks import run_import_variants_bg_job - #: The User model to use. User = get_user_model() diff --git a/importer/management/commands/import_cases_bulk.py b/importer/management/commands/import_cases_bulk.py index eb038495b..590a19a6d 100644 --- a/importer/management/commands/import_cases_bulk.py +++ b/importer/management/commands/import_cases_bulk.py @@ -1,13 +1,12 @@ """Django command for importing a case after annotation with ``varfish-annotator``.""" -import json import gzip +import json from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand, CommandError from importer.management.commands.import_case import CaseImportBase - #: The User model to use. User = get_user_model() diff --git a/importer/management/commands/import_tables.py b/importer/management/commands/import_tables.py index a1afecdf5..b4086d872 100644 --- a/importer/management/commands/import_tables.py +++ b/importer/management/commands/import_tables.py @@ -1,62 +1,61 @@ from contextlib import contextmanager +from multiprocessing.pool import ThreadPool import os import sys -import traceback -from multiprocessing.pool import ThreadPool import tempfile +import traceback -from variants.helpers import get_engine from django.core.management.base import BaseCommand, CommandError from django.db import connection, transaction -from clinvar.models import Clinvar, refresh_clinvar_clinvarpathogenicgenes +from clinvar.models import Clinvar from conservation.models import KnowngeneAA from dbsnp.models import Dbsnp +from extra_annos.models import ExtraAnno, ExtraAnnoField from frequencies.models import ( Exac, GnomadExomes, GnomadGenomes, - ThousandGenomes, + HelixMtDb, Mitomap, MtDb, - HelixMtDb, + ThousandGenomes, ) from geneinfo.models import ( + Acmg, + EnsemblToGeneSymbol, + EnsemblToRefseq, + ExacConstraints, + GnomadConstraints, Hgnc, + Hpo, + HpoName, MgiHomMouseHumanSequence, Mim2geneMedgen, - Hpo, NcbiGeneInfo, NcbiGeneRif, - HpoName, - RefseqToHgnc, - Acmg, - GnomadConstraints, - ExacConstraints, - EnsemblToRefseq, RefseqToEnsembl, - refresh_geneinfo_geneidtoinheritance, - refresh_geneinfo_mgimapping, RefseqToGeneSymbol, - EnsemblToGeneSymbol, + RefseqToHgnc, refresh_geneinfo_geneidinhpo, + refresh_geneinfo_geneidtoinheritance, + refresh_geneinfo_mgimapping, ) from genomicfeatures.models import ( - GeneInterval, EnsemblRegulatoryFeature, - TadSet, - TadInterval, + GeneInterval, TadBoundaryInterval, + TadInterval, + TadSet, VistaEnhancer, ) from hgmd.models import HgmdPublicLocus -from extra_annos.models import ExtraAnnoField, ExtraAnno +from pathways.models import EnsemblToKegg, KeggInfo, RefseqToKegg +from svdbs.models import DbVarSv, DgvGoldStandardSvs, DgvSvs, ExacCnv, GnomAdSv, ThousandGenomesSv +from variants.helpers import get_engine, get_meta + from ...models import ImportInfo -from pathways.models import EnsemblToKegg, RefseqToKegg, KeggInfo from ..helpers import tsv_reader -from svdbs.models import DgvGoldStandardSvs, DgvSvs, ExacCnv, ThousandGenomesSv, DbVarSv, GnomAdSv -from variants.helpers import get_meta - #: Tables in both GRCh37 and GRCh38. _TABLES_BOTH = { @@ -133,8 +132,7 @@ class Command(BaseCommand): - """Command class for importing all external databases into Varfish tables. - """ + """Command class for importing all external databases into Varfish tables.""" #: Help message displayed on the command line. help = "Bulk import all external databases into Varfish tables." @@ -177,8 +175,7 @@ def add_arguments(self, parser): ) def handle(self, *args, **options): - """Iterate over genomebuilds, database folders and versions to gather all required information for import. - """ + """Iterate over genomebuilds, database folders and versions to gather all required information for import.""" if not options["service"] and options["tables_path"] is None: raise CommandError("Please set either --tables-path or --service") diff --git a/importer/management/commands/pg_dump.py b/importer/management/commands/pg_dump.py index bf8bf1c38..09f6ec3e4 100644 --- a/importer/management/commands/pg_dump.py +++ b/importer/management/commands/pg_dump.py @@ -5,8 +5,8 @@ import subprocess import sys -from django.core.management.base import BaseCommand, CommandError from django.conf import settings +from django.core.management.base import BaseCommand, CommandError #: The available dump modes. DUMP_MODES = ("full", "backup-large", "backup-small") @@ -34,8 +34,7 @@ class Command(BaseCommand): - """Implementation wrapping ``pg_dump`` to support creating dumps of the underlying PostgreSQL database. - """ + """Implementation wrapping ``pg_dump`` to support creating dumps of the underlying PostgreSQL database.""" #: Help message displayed on the command line. help = "Easily create database dumps with ``pg_dump``" diff --git a/importer/management/commands/rebuild_project_case_stats.py b/importer/management/commands/rebuild_project_case_stats.py index 0c8d76194..1c1317747 100644 --- a/importer/management/commands/rebuild_project_case_stats.py +++ b/importer/management/commands/rebuild_project_case_stats.py @@ -1,16 +1,15 @@ """Django command for rebuilding cohort statistics after import.""" +from django.conf import settings from django.contrib.auth import get_user_model from django.core.exceptions import ObjectDoesNotExist from django.core.management.base import BaseCommand, CommandError from django.db import transaction -from django.conf import settings - from projectroles.plugins import get_backend_api +from variants.helpers import get_engine from variants.models import CaseAwareProject from variants.variant_stats import rebuild_case_variant_stats -from variants.helpers import get_engine timeline = get_backend_api("timeline_backend") diff --git a/importer/management/commands/rebuild_project_stats.py b/importer/management/commands/rebuild_project_stats.py index ae30e22b7..373a29f6d 100644 --- a/importer/management/commands/rebuild_project_stats.py +++ b/importer/management/commands/rebuild_project_stats.py @@ -1,15 +1,15 @@ """Django command for rebuilding cohort statistics after import.""" +from django.conf import settings from django.contrib.auth import get_user_model from django.core.exceptions import ObjectDoesNotExist from django.core.management.base import BaseCommand, CommandError from django.db import transaction -from django.conf import settings - from projectroles.models import Project from projectroles.plugins import get_backend_api -from variants.variant_stats import rebuild_project_variant_stats + from variants.helpers import get_engine +from variants.variant_stats import rebuild_project_variant_stats timeline = get_backend_api("timeline_backend") diff --git a/importer/management/commands/rebuild_variant_summary.py b/importer/management/commands/rebuild_variant_summary.py index cb71623c9..57a8d75bb 100644 --- a/importer/management/commands/rebuild_variant_summary.py +++ b/importer/management/commands/rebuild_variant_summary.py @@ -3,13 +3,13 @@ from django.core.management.base import BaseCommand from django.db import transaction -from ...tasks import refresh_variants_smallvariantsummary import variants.models as models +from ...tasks import refresh_variants_smallvariantsummary + class Command(BaseCommand): - """Implementation of rebuilding variant summary. - """ + """Implementation of rebuilding variant summary.""" #: Help message displayed on the command line. help = "Rebuild the variants summary." diff --git a/importer/migrations/0005_auto_20200129_1600.py b/importer/migrations/0005_auto_20200129_1600.py index 7452fa4e2..d81d9368e 100644 --- a/importer/migrations/0005_auto_20200129_1600.py +++ b/importer/migrations/0005_auto_20200129_1600.py @@ -2,11 +2,13 @@ # Generated by Django 1.11.27 on 2020-01-29 16:00 from __future__ import unicode_literals +import uuid + from django.conf import settings import django.contrib.postgres.fields from django.db import migrations, models import django.db.models.deletion -import uuid + import variants.models @@ -143,7 +145,9 @@ class Migration(migrations.Migration): ), ), ], - options={"abstract": False,}, + options={ + "abstract": False, + }, ), migrations.CreateModel( name="DatabaseInfoFile", @@ -173,7 +177,9 @@ class Migration(migrations.Migration): models.CharField(help_text="MD5 checksum of original file.", max_length=32), ), ], - options={"abstract": False,}, + options={ + "abstract": False, + }, ), migrations.CreateModel( name="EffectFile", @@ -203,7 +209,9 @@ class Migration(migrations.Migration): models.CharField(help_text="MD5 checksum of original file.", max_length=32), ), ], - options={"abstract": False,}, + options={ + "abstract": False, + }, ), migrations.CreateModel( name="GenotypeFile", @@ -233,7 +241,9 @@ class Migration(migrations.Migration): models.CharField(help_text="MD5 checksum of original file.", max_length=32), ), ], - options={"abstract": False,}, + options={ + "abstract": False, + }, ), migrations.CreateModel( name="ImportCaseBgJob", @@ -374,18 +384,23 @@ class Migration(migrations.Migration): unique_together=set([("case_import_info", "variant_type")]), ), migrations.AlterUniqueTogether( - name="genotypefile", unique_together=set([("variant_set_import_info", "md5")]), + name="genotypefile", + unique_together=set([("variant_set_import_info", "md5")]), ), migrations.AlterUniqueTogether( - name="effectfile", unique_together=set([("variant_set_import_info", "md5")]), + name="effectfile", + unique_together=set([("variant_set_import_info", "md5")]), ), migrations.AlterUniqueTogether( - name="databaseinfofile", unique_together=set([("variant_set_import_info", "md5")]), + name="databaseinfofile", + unique_together=set([("variant_set_import_info", "md5")]), ), migrations.AlterUniqueTogether( - name="caseimportinfo", unique_together=set([("project", "name")]), + name="caseimportinfo", + unique_together=set([("project", "name")]), ), migrations.AlterUniqueTogether( - name="bamqcfile", unique_together=set([("case_import_info", "md5")]), + name="bamqcfile", + unique_together=set([("case_import_info", "md5")]), ), ] diff --git a/importer/migrations/0007_auto_20200203_1242.py b/importer/migrations/0007_auto_20200203_1242.py index 238859a67..05539aa43 100644 --- a/importer/migrations/0007_auto_20200203_1242.py +++ b/importer/migrations/0007_auto_20200203_1242.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals from django.db import migrations, models + import importer.models diff --git a/importer/migrations/0009_alter_caseimportinfo_pedigree.py b/importer/migrations/0009_alter_caseimportinfo_pedigree.py index 2a1904073..ca88b1bf6 100644 --- a/importer/migrations/0009_alter_caseimportinfo_pedigree.py +++ b/importer/migrations/0009_alter_caseimportinfo_pedigree.py @@ -1,6 +1,7 @@ # Generated by Django 3.2.9 on 2021-11-29 14:43 from django.db import migrations + import varfish.utils @@ -12,6 +13,8 @@ class Migration(migrations.Migration): operations = [ migrations.AlterField( - model_name="caseimportinfo", name="pedigree", field=varfish.utils.JSONField(), + model_name="caseimportinfo", + name="pedigree", + field=varfish.utils.JSONField(), ), ] diff --git a/importer/models.py b/importer/models.py index e8d598d93..013371117 100644 --- a/importer/models.py +++ b/importer/models.py @@ -4,7 +4,7 @@ import tempfile import uuid as uuid_object -from bgjobs.models import BackgroundJob, LOG_LEVEL_ERROR, JobModelMessageMixin +from bgjobs.models import LOG_LEVEL_ERROR, BackgroundJob, JobModelMessageMixin from django.contrib import auth from django.db import models, transaction from django.db.models.signals import post_delete @@ -16,17 +16,16 @@ from importer.management.helpers import open_file, tsv_reader from svs.models import StructuralVariant, StructuralVariantGeneAnnotation, SvAnnotationReleaseInfo from varfish.utils import receiver_subclasses +from variants.helpers import get_engine, get_meta from variants.models import ( + AnnotationReleaseInfo, Case, + CaseAlignmentStats, CoreCase, - update_variant_counts, SmallVariant, - AnnotationReleaseInfo, - CaseAlignmentStats, SmallVariantSet, + update_variant_counts, ) -from variants.helpers import get_engine -from variants.helpers import get_meta User = auth.get_user_model() @@ -146,7 +145,9 @@ class VariantSetImportInfo(models.Model): ) case_import_info = models.ForeignKey( - CaseImportInfo, on_delete=models.CASCADE, help_text="The import info for the case.", + CaseImportInfo, + on_delete=models.CASCADE, + help_text="The import info for the case.", ) variant_type = models.CharField( @@ -573,7 +574,11 @@ def _perform_import(self, variant_set, variant_set_info): else: assert variant_set_info.variant_type == CaseVariantType.STRUCTURAL.name self._import_table( - variant_set_info, variant_set, "SVs", "genotypefile_set", StructuralVariant, + variant_set_info, + variant_set, + "SVs", + "genotypefile_set", + StructuralVariant, ) self._import_annotation_release_info( variant_set_info, variant_set, SvAnnotationReleaseInfo diff --git a/importer/qc_schemas.py b/importer/qc_schemas.py index 8b9959ca1..2a84d3e5d 100644 --- a/importer/qc_schemas.py +++ b/importer/qc_schemas.py @@ -1,7 +1,7 @@ """Utility code for QC schemas.""" -import os.path import json +import os.path def load_json(path): diff --git a/importer/rules.py b/importer/rules.py index b960c5286..fe73e4a9d 100644 --- a/importer/rules.py +++ b/importer/rules.py @@ -1,8 +1,7 @@ """Rule definitions for the ``bgjobs`` app.""" -import rules from projectroles import rules as pr_rules - +import rules # Predicates ------------------------------------------------------------------- @@ -20,7 +19,8 @@ ) rules.add_perm( - "importer.view_import", is_allowed_to_modify, + "importer.view_import", + is_allowed_to_modify, ) rules.add_perm("importer.add_import", is_allowed_to_modify) diff --git a/importer/serializers.py b/importer/serializers.py index 8f0fb31ca..2c871d796 100644 --- a/importer/serializers.py +++ b/importer/serializers.py @@ -1,9 +1,10 @@ """Serializers for the importer app.""" from django.db.models import Q +from projectroles.serializers import SODARProjectModelSerializer from rest_framework import serializers -from projectroles.serializers import SODARProjectModelSerializer from variants.serializers import CoreCaseSerializerMixin + from .models import ( BamQcFile, CaseImportInfo, diff --git a/importer/tasks.py b/importer/tasks.py index b12f6712b..9561a5534 100644 --- a/importer/tasks.py +++ b/importer/tasks.py @@ -1,7 +1,6 @@ """Tasks for the importer module.""" from config.celery import app - import variants.models from . import models diff --git a/importer/tests/factories.py b/importer/tests/factories.py index ae8b284e5..eb60087d5 100644 --- a/importer/tests/factories.py +++ b/importer/tests/factories.py @@ -6,13 +6,13 @@ from variants.tests.factories import CoreCaseFactory from ..models import ( + BamQcFile, CaseImportInfo, - VariantSetImportInfo, CaseVariantType, - BamQcFile, - GenotypeFile, - EffectFile, DatabaseInfoFile, + EffectFile, + GenotypeFile, + VariantSetImportInfo, ) diff --git a/importer/tests/test_views_api.py b/importer/tests/test_views_api.py index 260ad566a..24761426f 100644 --- a/importer/tests/test_views_api.py +++ b/importer/tests/test_views_api.py @@ -1,21 +1,20 @@ -from varfish import __version__ as varfish_version - -import os from itertools import chain +import os from django.contrib.auth import get_user_model -from django.urls import reverse from django.forms import model_to_dict +from django.urls import reverse +from varfish import __version__ as varfish_version from variants.tests.helpers import ApiViewTestBase from variants.tests.test_views_api import transmogrify_pedigree -from ..models import CaseImportInfo, VariantSetImportInfo, CaseVariantType, BamQcFile, GenotypeFile +from ..models import BamQcFile, CaseImportInfo, CaseVariantType, GenotypeFile, VariantSetImportInfo from .factories import ( - CaseImportInfoFactory, - VariantSetImportInfoFactory, BamQcFileFactory, + CaseImportInfoFactory, GenotypeFileFactory, + VariantSetImportInfoFactory, ) #: A known invalid MIME type. @@ -384,7 +383,9 @@ def test_list(self): self.assertEquals(response_content, expected) def test_create(self): - obj = BamQcFileFactory(case_import_info=self.case_import_info,) + obj = BamQcFileFactory( + case_import_info=self.case_import_info, + ) obj.delete() post_data = bam_qc_file_to_dict(obj, self.case_import_info, exclude=("sodar_uuid",)) @@ -486,7 +487,9 @@ def test_list(self): self.assertEquals(response_content, expected) def test_create(self): - obj = GenotypeFileFactory(variant_set_import_info=self.variant_set_import_info,) + obj = GenotypeFileFactory( + variant_set_import_info=self.variant_set_import_info, + ) obj.delete() post_data = genotype_file_to_dict( obj, self.variant_set_import_info, exclude=("sodar_uuid",) diff --git a/importer/urls.py b/importer/urls.py index ef6e92402..3c1c1527c 100644 --- a/importer/urls.py +++ b/importer/urls.py @@ -2,6 +2,7 @@ """ from django.conf.urls import url + from . import views, views_api app_name = "importer" diff --git a/importer/views.py b/importer/views.py index 44a56fba7..51e38fadd 100644 --- a/importer/views.py +++ b/importer/views.py @@ -1,9 +1,12 @@ -from projectroles.views import LoginRequiredMixin, ProjectPermissionMixin, ProjectContextMixin -from django.views.generic import ListView, DetailView - -from projectroles.views import LoggedInPermissionMixin +from django.views.generic import DetailView, ListView +from projectroles.views import ( + LoggedInPermissionMixin, + LoginRequiredMixin, + ProjectContextMixin, + ProjectPermissionMixin, +) -from .models import ImportInfo, ImportCaseBgJob +from .models import ImportCaseBgJob, ImportInfo class ImportInfoView(LoginRequiredMixin, LoggedInPermissionMixin, ListView): diff --git a/importer/views_api.py b/importer/views_api.py index 6e938fcf5..98aed05fc 100644 --- a/importer/views_api.py +++ b/importer/views_api.py @@ -9,20 +9,21 @@ from projectroles.views_api import SODARAPIBaseProjectMixin, SODARAPIGenericProjectMixin from rest_framework.generics import ( ListCreateAPIView, - RetrieveUpdateDestroyAPIView, RetrieveDestroyAPIView, + RetrieveUpdateDestroyAPIView, ) from varfish.api_utils import VarfishApiRenderer, VarfishApiVersioning + from . import tasks -from .models import CaseImportInfo, VariantSetImportInfo, CaseImportState, ImportCaseBgJob +from .models import CaseImportInfo, CaseImportState, ImportCaseBgJob, VariantSetImportInfo from .serializers import ( - CaseImportInfoSerializer, - VariantSetImportInfoSerializer, BamQcFileSerializer, + CaseImportInfoSerializer, DatabaseInfoFileSerializer, EffectFileSerializer, GenotypeFileSerializer, + VariantSetImportInfoSerializer, ) #: Logger to use in this module. diff --git a/maintenance/management/commands/check_installation.py b/maintenance/management/commands/check_installation.py index 2e763ca65..d567050bf 100644 --- a/maintenance/management/commands/check_installation.py +++ b/maintenance/management/commands/check_installation.py @@ -2,11 +2,10 @@ from enum import Enum from django.core.management.base import BaseCommand -from django.db import transaction, connection, ProgrammingError +from django.db import ProgrammingError, connection, transaction - -ICON_CHECK_MARK = u"\u2714" -ICON_FAILED = u"\u2718" +ICON_CHECK_MARK = "\u2714" +ICON_FAILED = "\u2718" class State(Enum): @@ -201,8 +200,7 @@ class State(Enum): class Command(BaseCommand): - """Implementation of checking installation. - """ + """Implementation of checking installation.""" #: Help message displayed on the command line. help = "Sanity check installation." diff --git a/maintenance/management/commands/clear_expired_files.py b/maintenance/management/commands/clear_expired_files.py index d20f3efba..affbe96be 100644 --- a/maintenance/management/commands/clear_expired_files.py +++ b/maintenance/management/commands/clear_expired_files.py @@ -9,8 +9,7 @@ class Command(BaseCommand): - """Implementation of clearing expired exported files. - """ + """Implementation of clearing expired exported files.""" #: Help message displayed on the command line. help = "Clear expired exported files." diff --git a/maintenance/management/commands/clear_inactive_variant_sets.py b/maintenance/management/commands/clear_inactive_variant_sets.py index 88aa1ea03..1e20cb96b 100644 --- a/maintenance/management/commands/clear_inactive_variant_sets.py +++ b/maintenance/management/commands/clear_inactive_variant_sets.py @@ -9,8 +9,7 @@ class Command(BaseCommand): - """Implementation of clearing inactive variant sets. - """ + """Implementation of clearing inactive variant sets.""" #: Help message displayed on the command line. help = "Clear inactive variant sets." diff --git a/maintenance/management/commands/clear_old_kiosk_cases.py b/maintenance/management/commands/clear_old_kiosk_cases.py index c9de86865..5093c69a1 100644 --- a/maintenance/management/commands/clear_old_kiosk_cases.py +++ b/maintenance/management/commands/clear_old_kiosk_cases.py @@ -9,8 +9,7 @@ class Command(BaseCommand): - """Implementation of clearing old kiosk cases. - """ + """Implementation of clearing old kiosk cases.""" #: Help message displayed on the command line. help = "Clear old kiosk cases." diff --git a/pathways/tests/test_models.py b/pathways/tests/test_models.py index 7068928a4..ab498f27f 100644 --- a/pathways/tests/test_models.py +++ b/pathways/tests/test_models.py @@ -2,8 +2,8 @@ from variants.tests.helpers import QueryTestBase -from .factories import EnsemblToKeggFactory, RefseqToKeggFactory, KeggInfoFactory -from ..models import EnsemblToKegg, RefseqToKegg, KeggInfo +from ..models import EnsemblToKegg, KeggInfo, RefseqToKegg +from .factories import EnsemblToKeggFactory, KeggInfoFactory, RefseqToKeggFactory class TestPathwayQuery(QueryTestBase): diff --git a/regmaps/admin.py b/regmaps/admin.py index d6e6fc28e..4383315be 100644 --- a/regmaps/admin.py +++ b/regmaps/admin.py @@ -1,12 +1,14 @@ from django.contrib import admin -from .models import ( - RegMapCollection, - RegMap, - RegElementType, - RegElement, - RegInteraction, -) +from .models import RegElement, RegElementType, RegInteraction, RegMap, RegMapCollection # Register your models here. -admin.site.register((RegMapCollection, RegMap, RegElementType, RegElement, RegInteraction,)) +admin.site.register( + ( + RegMapCollection, + RegMap, + RegElementType, + RegElement, + RegInteraction, + ) +) diff --git a/regmaps/cmds.py b/regmaps/cmds.py index 27acd595b..a11881250 100644 --- a/regmaps/cmds.py +++ b/regmaps/cmds.py @@ -5,12 +5,13 @@ import sys import binning -from django.db import transaction from django.core.management.base import CommandError +from django.db import transaction import yaml from importer.models import ImportInfo -from .models import RegMapCollection, RegElementType, RegMap, RegElement, RegInteraction + +from .models import RegElement, RegElementType, RegInteraction, RegMap, RegMapCollection class CollectionListImpl: diff --git a/regmaps/migrations/0001_initial.py b/regmaps/migrations/0001_initial.py index 9d4e9dd17..05ec1db51 100644 --- a/regmaps/migrations/0001_initial.py +++ b/regmaps/migrations/0001_initial.py @@ -2,9 +2,10 @@ # Generated by Django 1.11.29 on 2020-10-26 16:05 from __future__ import unicode_literals +import uuid + from django.db import migrations, models import django.db.models.deletion -import uuid class Migration(migrations.Migration): @@ -37,7 +38,9 @@ class Migration(migrations.Migration): ("score", models.FloatField()), ("extra_data", models.JSONField(null=True)), ], - options={"ordering": ["chromosome", "start", "end"],}, + options={ + "ordering": ["chromosome", "start", "end"], + }, ), migrations.CreateModel( name="RegElementType", @@ -61,7 +64,9 @@ class Migration(migrations.Migration): ("slug", models.SlugField()), ("description", models.TextField(null=True)), ], - options={"ordering": ["title"],}, + options={ + "ordering": ["title"], + }, ), migrations.CreateModel( name="RegInteraction", @@ -92,7 +97,9 @@ class Migration(migrations.Migration): ("end2", models.IntegerField()), ("extra_data", models.JSONField(null=True)), ], - options={"ordering": ["chromosome", "start", "end"],}, + options={ + "ordering": ["chromosome", "start", "end"], + }, ), migrations.CreateModel( name="RegMap", @@ -114,7 +121,9 @@ class Migration(migrations.Migration): ("slug", models.SlugField()), ("description", models.TextField(null=True)), ], - options={"ordering": ["title"],}, + options={ + "ordering": ["title"], + }, ), migrations.CreateModel( name="RegMapCollection", @@ -140,7 +149,9 @@ class Migration(migrations.Migration): ("slug", models.SlugField(unique=True)), ("description", models.TextField(null=True)), ], - options={"ordering": ["title"],}, + options={ + "ordering": ["title"], + }, ), migrations.AddIndex( model_name="regmapcollection", @@ -186,7 +197,8 @@ class Migration(migrations.Migration): index=models.Index(fields=["sodar_uuid"], name="regmaps_reg_sodar_u_9ed6fa_idx"), ), migrations.AlterUniqueTogether( - name="regmap", unique_together=set([("collection", "slug")]), + name="regmap", + unique_together=set([("collection", "slug")]), ), migrations.AddIndex( model_name="reginteraction", @@ -203,7 +215,8 @@ class Migration(migrations.Migration): index=models.Index(fields=["sodar_uuid"], name="regmaps_reg_sodar_u_fb7c06_idx"), ), migrations.AlterUniqueTogether( - name="regelementtype", unique_together=set([("collection", "slug")]), + name="regelementtype", + unique_together=set([("collection", "slug")]), ), migrations.AddIndex( model_name="regelement", diff --git a/regmaps/migrations/0002_auto_20211008_1015.py b/regmaps/migrations/0002_auto_20211008_1015.py index 8ff8433e6..e5da1dc83 100644 --- a/regmaps/migrations/0002_auto_20211008_1015.py +++ b/regmaps/migrations/0002_auto_20211008_1015.py @@ -1,6 +1,7 @@ # Generated by Django 3.2.7 on 2021-10-08 10:15 from django.db import migrations + import varfish.utils @@ -12,7 +13,9 @@ class Migration(migrations.Migration): operations = [ migrations.AlterField( - model_name="regelement", name="extra_data", field=varfish.utils.JSONField(null=True), + model_name="regelement", + name="extra_data", + field=varfish.utils.JSONField(null=True), ), migrations.AlterField( model_name="reginteraction", diff --git a/regmaps/models.py b/regmaps/models.py index 67c04ac84..07a03b8b3 100644 --- a/regmaps/models.py +++ b/regmaps/models.py @@ -18,7 +18,9 @@ class RegMapCollection(models.Model): #: The UUID of this regulatory map collection. sodar_uuid = models.UUIDField( - default=uuid_object.uuid4, unique=True, help_text="UUID of regulatory map collection", + default=uuid_object.uuid4, + unique=True, + help_text="UUID of regulatory map collection", ) #: Genome build that this regulatory map collection is for. release = models.CharField(max_length=32) @@ -43,10 +45,15 @@ class RegMap(models.Model): #: The UUID of this regulatory map. sodar_uuid = models.UUIDField( - default=uuid_object.uuid4, unique=True, help_text="UUID of regulatory map", + default=uuid_object.uuid4, + unique=True, + help_text="UUID of regulatory map", ) #: The regulatory map collection that this map belongs to. - collection = models.ForeignKey(RegMapCollection, on_delete=models.CASCADE,) + collection = models.ForeignKey( + RegMapCollection, + on_delete=models.CASCADE, + ) #: The title of the map. title = models.TextField() #: The short title of the map. @@ -67,10 +74,15 @@ class RegElementType(models.Model): #: The UUID of this regulatory element type. sodar_uuid = models.UUIDField( - default=uuid_object.uuid4, unique=True, help_text="UUID of regulatory map collection", + default=uuid_object.uuid4, + unique=True, + help_text="UUID of regulatory map collection", ) #: The regulatory map collection that this map belongs to. - collection = models.ForeignKey(RegMapCollection, on_delete=models.CASCADE,) + collection = models.ForeignKey( + RegMapCollection, + on_delete=models.CASCADE, + ) #: The title of the map. title = models.TextField() #: The short title of the map. @@ -109,12 +121,20 @@ class RegElement(models.Model): #: The UUID of this regulatory element. sodar_uuid = models.UUIDField( - default=uuid_object.uuid4, unique=True, help_text="UUID of regulatory element", + default=uuid_object.uuid4, + unique=True, + help_text="UUID of regulatory element", ) #: The regulatory map. - reg_map = models.ForeignKey(RegMap, on_delete=models.CASCADE,) + reg_map = models.ForeignKey( + RegMap, + on_delete=models.CASCADE, + ) #: The element type. - elem_type = models.ForeignKey(RegElementType, on_delete=models.CASCADE,) + elem_type = models.ForeignKey( + RegElementType, + on_delete=models.CASCADE, + ) #: Genome build release = models.CharField(max_length=32) #: Variant coordinates - chromosome @@ -161,10 +181,15 @@ class RegInteraction(models.Model): #: The UUID of this raw signal entry. sodar_uuid = models.UUIDField( - default=uuid_object.uuid4, unique=True, help_text="UUID of interaction", + default=uuid_object.uuid4, + unique=True, + help_text="UUID of interaction", ) #: The regulatory map. - reg_map = models.ForeignKey(RegMap, on_delete=models.CASCADE,) + reg_map = models.ForeignKey( + RegMap, + on_delete=models.CASCADE, + ) #: Genome build release = models.CharField(max_length=32) #: Variant coordinates - chromosome diff --git a/regmaps/tests/factories.py b/regmaps/tests/factories.py index 5625095e3..4e9729e20 100644 --- a/regmaps/tests/factories.py +++ b/regmaps/tests/factories.py @@ -3,7 +3,7 @@ import binning import factory -from ..models import RegMapCollection, RegMap, RegElementType, RegElement, RegInteraction +from ..models import RegElement, RegElementType, RegInteraction, RegMap, RegMapCollection class RegMapCollectionFactory(factory.django.DjangoModelFactory): diff --git a/regmaps/tests/test_cmds.py b/regmaps/tests/test_cmds.py index e0c6aa46b..cc506c67f 100644 --- a/regmaps/tests/test_cmds.py +++ b/regmaps/tests/test_cmds.py @@ -1,15 +1,15 @@ """Tests for the admin command implementations in ``cmds``.""" -import pathlib import io +import pathlib import sys import types import uuid from django.test import TestCase -from ..cmds import CollectionListImpl, CollectionDeleteImpl, CollectionImportImpl -from ..models import RegMapCollection, RegMap, RegElementType, RegElement, RegInteraction +from ..cmds import CollectionDeleteImpl, CollectionImportImpl, CollectionListImpl +from ..models import RegElement, RegElementType, RegInteraction, RegMap, RegMapCollection from .factories import RegMapCollectionFactory diff --git a/regmaps/tests/test_models.py b/regmaps/tests/test_models.py index 96c4aaad4..0cebb10ca 100644 --- a/regmaps/tests/test_models.py +++ b/regmaps/tests/test_models.py @@ -2,13 +2,13 @@ from django.test import TestCase -from ..models import RegMapCollection, RegMap, RegElementType, RegElement, RegInteraction +from ..models import RegElement, RegElementType, RegInteraction, RegMap, RegMapCollection from .factories import ( - RegMapCollectionFactory, - RegMapFactory, - RegElementTypeFactory, RegElementFactory, + RegElementTypeFactory, RegInteractionFactory, + RegMapCollectionFactory, + RegMapFactory, ) diff --git a/requirements/base.txt b/requirements/base.txt index 3308bb113..f25c073a8 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -151,10 +151,6 @@ django-autocomplete-light >=3.8.2, <3.9 # SAML2 support for SSO django-saml2-auth-ai>=2.1.6, <2.2 -# Code formatting. -black ==19.10b0 -click >=8.0,<8.1 - # Editable interval trees intervaltree >=3.1.0,<4.0.0 diff --git a/requirements/local.txt b/requirements/local.txt index 2c3d87b39..3fa4d09cd 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -1,5 +1,5 @@ # Local development dependencies go here --r base.txt +-r test.txt django-extensions >=3.1.3, <3.2 Werkzeug >=2.0.1, <2.1 diff --git a/requirements/test.txt b/requirements/test.txt index e631ec6ab..5fa6b6723 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,7 +1,8 @@ # Test dependencies go here. -r base.txt -flake8 >=3.8.4, <3.9 +flake8 ==4.0.1 +flake8-import-order ==0.18.1 django-test-plus >=1.4.0, <1.5 factory-boy >=3.2.0, <3.3 django-coverage-plugin >=1.8.0, <1.9 @@ -24,5 +25,12 @@ parameterized >=0.8.1, <0.9 # Comparison of JSON objects. jsonmatch >=0.4, <0.5 +# Code formatting. +black ==22.8.0 +click >=8.0,<8.1 + +# isort code formatter - import order +isort ==5.10.1 + # Coveralls tooling coveralls diff --git a/setup.cfg b/setup.cfg index e7ebd5899..42e10eaed 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,11 +1,35 @@ +# Style Checking and Linting ------------------------------------------------ +[isort] +profile = "black" +multi_line_output = 3 +include_trailing_comma = True +force_grid_wrap = 0 +use_parentheses = True +ensure_newline_before_comments = True +line_length = 100 + [flake8] -max-line-length = 100 +exclude = + docs + tests + .*.py + .snakemake.*.wrapper.py + splitMNPsAndComplex.py + wrapper.py + snappy_pipeline/__init__.py + versioneer.py + docs_manual/src + src +max-complexity = 18 +select = B,C,E,F,W,T4,B9 ignore = E203, E266, E501, W503 -exclude = .tox,.git,*/migrations/*,*/static/CACHE/*,docs,node_modules - -[pycodestyle] max-line-length = 100 -exclude=.tox,.git,*/migrations/*,*/static/CACHE/*,docs,node_modules +per-file-ignores = + tests/**/*.py: E501 + docs/conf.py: ALL + config/settings/*.py: F401, F403, F405 +application_import_names = snappy_pipeline +import-order-style = google [versioneer] VCS = git diff --git a/svdbs/admin.py b/svdbs/admin.py index 71cb6edf2..51399a552 100644 --- a/svdbs/admin.py +++ b/svdbs/admin.py @@ -1,6 +1,6 @@ from django.contrib import admin -from .models import DgvGoldStandardSvs, DgvSvs, DbVarSv, ThousandGenomesSv, ExacCnv +from .models import DbVarSv, DgvGoldStandardSvs, DgvSvs, ExacCnv, ThousandGenomesSv # Register your models here. admin.site.register(DgvGoldStandardSvs) diff --git a/svdbs/models.py b/svdbs/models.py index 8ca72727b..8d0b8a80e 100644 --- a/svdbs/models.py +++ b/svdbs/models.py @@ -4,9 +4,8 @@ __author__ = "Manuel Holtgrewe " -from django.db import models from django.contrib.postgres.fields import ArrayField - +from django.db import models #: "gain" in DGV DGV_SV_SUB_TYPE_GAIN = "gain" diff --git a/svdbs/tests/factories.py b/svdbs/tests/factories.py index 4d18a820f..13a6df311 100644 --- a/svdbs/tests/factories.py +++ b/svdbs/tests/factories.py @@ -4,13 +4,13 @@ import factory from ..models import ( + EXAC_POP_CHOICES, + DbVarSv, DgvGoldStandardSvs, DgvSvs, ExacCnv, - ThousandGenomesSv, - DbVarSv, GnomAdSv, - EXAC_POP_CHOICES, + ThousandGenomesSv, ) diff --git a/svdbs/tests/test_models.py b/svdbs/tests/test_models.py index 6662c89ad..557de9d35 100644 --- a/svdbs/tests/test_models.py +++ b/svdbs/tests/test_models.py @@ -3,11 +3,11 @@ from django.test import TestCase from .factories import ( - DgvSvsFactory, - DgvGoldStandardSvsFactory, DbVarSvFactory, - GnomAdSvFactory, + DgvGoldStandardSvsFactory, + DgvSvsFactory, ExacCnvFactory, + GnomAdSvFactory, ) diff --git a/svdbs/views.py b/svdbs/views.py index 91ea44a21..60f00ef0e 100644 --- a/svdbs/views.py +++ b/svdbs/views.py @@ -1,3 +1 @@ -from django.shortcuts import render - # Create your views here. diff --git a/svs/admin.py b/svs/admin.py index f77c7f124..23feee37e 100644 --- a/svs/admin.py +++ b/svs/admin.py @@ -1,12 +1,11 @@ from django.contrib import admin - from .models import ( + ImportStructuralVariantBgJob, StructuralVariant, - StructuralVariantGeneAnnotation, - StructuralVariantFlags, StructuralVariantComment, - ImportStructuralVariantBgJob, + StructuralVariantFlags, + StructuralVariantGeneAnnotation, StructuralVariantSet, ) diff --git a/svs/bg_db.py b/svs/bg_db.py index 0b128400e..a91858c15 100644 --- a/svs/bg_db.py +++ b/svs/bg_db.py @@ -1,21 +1,20 @@ """Code that supports building the structural variant background database.""" -import gc -import math -import os -import tempfile - -import attrs from contextlib import contextmanager import enum +import gc import json import logging +import math +import os import pathlib import random import re import statistics import sys +import tempfile import typing +import attrs import binning import cattr from django.conf import settings @@ -24,23 +23,22 @@ from django.utils import timezone from intervaltree import Interval, IntervalTree from projectroles.plugins import get_backend_api -import psutil from projectroles.templatetags.projectroles_common_tags import get_app_setting +import psutil from sqlalchemy import delete from svs.models import ( - SV_SUB_TYPE_CHOICES as _SV_SUB_TYPE_CHOICES, - SV_SUB_TYPE_BND as _SV_SUB_TYPE_BND, - SV_SUB_TYPE_INS as _SV_SUB_TYPE_INS, - StructuralVariant, + BackgroundSv, BackgroundSvSet, BuildBackgroundSvSetJob, - BackgroundSv, CleanupBackgroundSvSetJob, + StructuralVariant, ) +from svs.models import SV_SUB_TYPE_BND as _SV_SUB_TYPE_BND +from svs.models import SV_SUB_TYPE_CHOICES as _SV_SUB_TYPE_CHOICES +from svs.models import SV_SUB_TYPE_INS as _SV_SUB_TYPE_INS from varfish import __version__ as varfish_version - #: Logger to use in this module. from variants.helpers import get_engine, get_meta from variants.models import CHROMOSOME_NAMES, CHROMOSOME_STR_TO_CHROMOSOME_INT, Case @@ -145,7 +143,7 @@ def does_overlap(self, other: "SvRecord", *, bnd_slack: typing.Optional[int] = N if self.sv_type != other.sv_type: # pragma: nocover raise ValueError(f"Incompatible sv_type values: {self.sv_type} vs {other.sv_type}") if (bnd_slack is None) == (self.is_bnd() or self.is_ins()): - raise ValueError(f"Should specify bnd_slack if and only if SV is a breakend (or INS)") + raise ValueError("Should specify bnd_slack if and only if SV is a breakend (or INS)") if self.is_bnd(): # break-end, potentially non-linear SV return ( self.chrom == other.chrom @@ -164,7 +162,7 @@ def jaccard_index(self, other: "SvRecord") -> typing.Optional[float]: Raises an ``ValueError` if ``release`` or ``sv_type`` are not the same. """ if self.is_bnd() or other.is_bnd() or self.is_ins() or other.is_ins(): - raise ValueError(f"Cannot compute Jaccard overlap for break-ends and INS!") + raise ValueError("Cannot compute Jaccard overlap for break-ends and INS!") if self.does_overlap(other): len_union = max(self.end, other.end) + 1 - min(self.pos, other.pos) len_intersect = min(self.end, other.end) + 1 - max(self.pos, other.pos) @@ -257,7 +255,11 @@ def _compute_mean(self) -> SvRecord: pos = math.floor(statistics.mean([r.pos for r in self.records])) end = math.ceil(statistics.mean([r.end for r in self.records])) end = max(end, pos + 1) - return attrs.evolve(self.records[0], pos=pos, end=end,) + return attrs.evolve( + self.records[0], + pos=pos, + end=end, + ) def sort_key(self): if self.mean is None: @@ -604,7 +606,7 @@ def build_bg_sv_set( log_to_stderr: bool = False, chromosomes: typing.Optional[typing.List[str]] = None, ) -> BackgroundSvSet: - """Construct a new ``BackgroundSvSet`` """ + """Construct a new ``BackgroundSvSet``""" job.mark_start() timeline = get_backend_api("timeline_backend") if timeline: diff --git a/svs/forms.py b/svs/forms.py index cebbb5853..8e4e4b74a 100644 --- a/svs/forms.py +++ b/svs/forms.py @@ -1,16 +1,11 @@ from functools import lru_cache from django import forms + from genomicfeatures.models import TadSet from regmaps.models import RegMapCollection -from svs.models import StructuralVariantFlags, StructuralVariantComment - -from variants.forms import ( - VariantGeneListFilterFormMixin, - FAIL, - only_source_name, - GenomicRegionFilterFormMixin, -) +from svs.models import StructuralVariantComment, StructuralVariantFlags +from variants.forms import FAIL, GenomicRegionFilterFormMixin, VariantGeneListFilterFormMixin FILTER_FORM_TRANSLATE_EFFECTS = { "effect_coding_sequence_variant": "coding_sequence_variant", @@ -525,7 +520,9 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.fields["regulatory_general_padding"] = forms.IntegerField( - label="padding (bp)", required=False, initial=100, + label="padding (bp)", + required=False, + initial=100, ) self.fields["regulatory_ensembl"] = forms.MultipleChoiceField( label="ENSEMBL feature", @@ -551,7 +548,10 @@ def __init__(self, *args, **kwargs): choices=[("__any__", "any")] + [(ret.slug, ret.short_title) for ret in coll.regmap_set.all()], ) - interaction_field = forms.BooleanField(label=coll.title, required=False,) + interaction_field = forms.BooleanField( + label=coll.title, + required=False, + ) self.fields["regmap_%s_element" % coll.slug] = element_field self.fields["regmap_%s_map" % coll.slug] = map_field self.fields["regmap_%s_interaction" % coll.slug] = interaction_field diff --git a/svs/management/commands/svs_bg_sv_set_cleanup.py b/svs/management/commands/svs_bg_sv_set_cleanup.py index 1edd44e80..c74672262 100644 --- a/svs/management/commands/svs_bg_sv_set_cleanup.py +++ b/svs/management/commands/svs_bg_sv_set_cleanup.py @@ -26,4 +26,4 @@ def handle(self, *args, **options): """The actual implementation is in ``_handle()``, splitting to get commit times.""" self.stderr.write("Removing background svs") tasks.cleanup_bg_sv_set_task(timeout_hours=options["timeout"]) - self.stderr.write(self.style.SUCCESS(f"All done, have a nice day!")) + self.stderr.write(self.style.SUCCESS("All done, have a nice day!")) diff --git a/svs/management/commands/svs_bg_sv_set_dump.py b/svs/management/commands/svs_bg_sv_set_dump.py index b79399cde..58a1c10b3 100644 --- a/svs/management/commands/svs_bg_sv_set_dump.py +++ b/svs/management/commands/svs_bg_sv_set_dump.py @@ -3,13 +3,12 @@ from django.core.management import BaseCommand from sqlalchemy import select -from svs.models import BackgroundSvSet, BackgroundSv -from variants.helpers import get_meta, get_engine +from svs.models import BackgroundSv, BackgroundSvSet +from variants.helpers import get_engine, get_meta class Command(BaseCommand): - """Dump a single ``BackgroundSvSet`` as BED file. - """ + """Dump a single ``BackgroundSvSet`` as BED file.""" #: Help message displayed on the command line. help = "List existing background SV sets" @@ -32,4 +31,4 @@ def handle(self, *args, **options): ) for row in get_engine().execute(query): print("\t".join(map(str, [row.chromosome, row.start - 1, row.end, row.sv_type]))) - self.stderr.write(self.style.SUCCESS(f"All done, have a nice day!")) + self.stderr.write(self.style.SUCCESS("All done, have a nice day!")) diff --git a/svs/management/commands/svs_bg_sv_set_list.py b/svs/management/commands/svs_bg_sv_set_list.py index 41e3c8c60..49b57cb62 100644 --- a/svs/management/commands/svs_bg_sv_set_list.py +++ b/svs/management/commands/svs_bg_sv_set_list.py @@ -3,13 +3,11 @@ from django.core.management import BaseCommand from prettytable import PrettyTable - from svs.models import BackgroundSvSet class Command(BaseCommand): - """List all existing ``BackgroundSvSet`` records - """ + """List all existing ``BackgroundSvSet`` records""" #: Help message displayed on the command line. help = "List existing background SV sets" diff --git a/svs/management/commands/svs_sv_fill_nulls.py b/svs/management/commands/svs_sv_fill_nulls.py index 65de2fd3a..a75bd8381 100644 --- a/svs/management/commands/svs_sv_fill_nulls.py +++ b/svs/management/commands/svs_sv_fill_nulls.py @@ -8,8 +8,8 @@ from django.core.management import BaseCommand from tqdm import tqdm -from svs.models import StructuralVariant from svs import bg_db +from svs.models import StructuralVariant from variants.models import Case diff --git a/svs/management/commands/svs_svs_dump.py b/svs/management/commands/svs_svs_dump.py index 7890401e7..150c21350 100644 --- a/svs/management/commands/svs_svs_dump.py +++ b/svs/management/commands/svs_svs_dump.py @@ -34,7 +34,8 @@ def add_arguments(self, parser): def handle(self, *args, **options): """The actual implementation is in ``_handle()``, splitting to get commit times.""" if options.get("output_file"): - self.stderr.write(f"Writing to {options[output_file]}") + output_file = options["output_file"] + self.stderr.write(f"Writing to {output_file}") with open(options["output_file"], "wt") as outputf: record_count = self._handle(options, outputf) else: diff --git a/svs/migrations/0001_initial.py b/svs/migrations/0001_initial.py index 04966bc8e..37db4338f 100644 --- a/svs/migrations/0001_initial.py +++ b/svs/migrations/0001_initial.py @@ -2,11 +2,12 @@ # Generated by Django 1.11.20 on 2019-04-23 09:40 from __future__ import unicode_literals +import uuid + from django.conf import settings import django.contrib.postgres.fields from django.db import migrations, models import django.db.models.deletion -import uuid class Migration(migrations.Migration): diff --git a/svs/migrations/0002_pgplsql_overlapping_bins.py b/svs/migrations/0002_pgplsql_overlapping_bins.py index a186ca4dc..880b3b0f4 100644 --- a/svs/migrations/0002_pgplsql_overlapping_bins.py +++ b/svs/migrations/0002_pgplsql_overlapping_bins.py @@ -22,7 +22,7 @@ class Migration(migrations.Migration): shift_next constant integer := 3; max_position constant integer := 1 << 29; max_bin constant integer := bin_offsets[0] + (max_position >> shift_first); - + start_bin integer; stop_bin integer; bin_offset integer; @@ -30,10 +30,10 @@ class Migration(migrations.Migration): IF start < 0 THEN RAISE EXCEPTION 'Invalid start %s', start; END IF; - + start_bin = start >> shift_first; stop_bin = stop >> shift_first; - + FOREACH bin_offset IN ARRAY bin_offsets LOOP bin_start := bin_offset + start_bin; @@ -48,7 +48,7 @@ class Migration(migrations.Migration): LANGUAGE 'plpgsql' IMMUTABLE COST 1; - + -- start and stop are 0-based positions CREATE OR REPLACE FUNCTION overlapping_bins(start integer, stop integer) RETURNS TABLE (bin integer) diff --git a/svs/migrations/0005_importstructuralvariantbgjob.py b/svs/migrations/0005_importstructuralvariantbgjob.py index 05675005a..bc8eafb4f 100644 --- a/svs/migrations/0005_importstructuralvariantbgjob.py +++ b/svs/migrations/0005_importstructuralvariantbgjob.py @@ -2,10 +2,12 @@ # Generated by Django 1.11.21 on 2019-07-03 15:59 from __future__ import unicode_literals +import uuid + import django.contrib.postgres.fields from django.db import migrations, models import django.db.models.deletion -import uuid + import variants.models diff --git a/svs/migrations/0006_auto_20190703_1709.py b/svs/migrations/0006_auto_20190703_1709.py index ddb7f7de6..31e59fe4f 100644 --- a/svs/migrations/0006_auto_20190703_1709.py +++ b/svs/migrations/0006_auto_20190703_1709.py @@ -2,8 +2,8 @@ # Generated by Django 1.11.21 on 2019-07-03 17:09 from __future__ import unicode_literals -from django.db import migrations from django.conf import settings +from django.db import migrations class Migration(migrations.Migration): diff --git a/svs/migrations/0007_partition_sv_tables.py b/svs/migrations/0007_partition_sv_tables.py index 4b1c88cc6..d343055f2 100644 --- a/svs/migrations/0007_partition_sv_tables.py +++ b/svs/migrations/0007_partition_sv_tables.py @@ -2,11 +2,11 @@ """Setup ``svs`` tables as partitioned. """ -from django.db import migrations, models -from django.conf import settings -import django.contrib.postgres.fields import uuid +from django.conf import settings +import django.contrib.postgres.fields +from django.db import migrations, models operations = [ migrations.DeleteModel("StructuralVariant"), diff --git a/svs/migrations/0008_set_unlogged_table.py b/svs/migrations/0008_set_unlogged_table.py index b249dab7c..13e1c6838 100644 --- a/svs/migrations/0008_set_unlogged_table.py +++ b/svs/migrations/0008_set_unlogged_table.py @@ -2,8 +2,8 @@ """SET svs tables as UNLOGGED to improve insertion performance.""" from __future__ import unicode_literals -from django.db import migrations from django.conf import settings +from django.db import migrations if not settings.IS_TESTING: operations = ( diff --git a/svs/migrations/0010_set_logged_table.py b/svs/migrations/0010_set_logged_table.py index 9636646df..00ed24f37 100644 --- a/svs/migrations/0010_set_logged_table.py +++ b/svs/migrations/0010_set_logged_table.py @@ -2,8 +2,8 @@ """SET svs tables as LOGGED against data loss.""" from __future__ import unicode_literals -from django.db import migrations from django.conf import settings +from django.db import migrations if not settings.IS_TESTING: operations = ( diff --git a/svs/migrations/0011_rename_partitions.py b/svs/migrations/0011_rename_partitions.py index a75484c92..168ff8d2e 100644 --- a/svs/migrations/0011_rename_partitions.py +++ b/svs/migrations/0011_rename_partitions.py @@ -2,8 +2,8 @@ """Rename tables to make consistent again.""" from __future__ import unicode_literals -from django.db import migrations from django.conf import settings +from django.db import migrations if not settings.IS_TESTING: operations = [ diff --git a/svs/migrations/0014_svs_stucturalvariants_alter_unique_constraint.py b/svs/migrations/0014_svs_stucturalvariants_alter_unique_constraint.py index df7633140..fa979b92b 100644 --- a/svs/migrations/0014_svs_stucturalvariants_alter_unique_constraint.py +++ b/svs/migrations/0014_svs_stucturalvariants_alter_unique_constraint.py @@ -2,8 +2,8 @@ """Setup ``svs`` tables as partitioned. """ -from django.db import migrations from django.conf import settings +from django.db import migrations operations = [] diff --git a/svs/migrations/0015_set_logged_table.py b/svs/migrations/0015_set_logged_table.py index d4d9f9872..5b013a890 100644 --- a/svs/migrations/0015_set_logged_table.py +++ b/svs/migrations/0015_set_logged_table.py @@ -2,8 +2,8 @@ """SET svs tables as LOGGED against data loss.""" from __future__ import unicode_literals -from django.db import migrations from django.conf import settings +from django.db import migrations if not settings.IS_TESTING: operations = ( diff --git a/svs/migrations/0017_backgroundsv_backgroundsvset_buildbackgroundsvsetjob_cleanupbackgroundsvsetjob.py b/svs/migrations/0017_backgroundsv_backgroundsvset_buildbackgroundsvsetjob_cleanupbackgroundsvsetjob.py index 303057276..dc0a981f8 100644 --- a/svs/migrations/0017_backgroundsv_backgroundsvset_buildbackgroundsvsetjob_cleanupbackgroundsvsetjob.py +++ b/svs/migrations/0017_backgroundsv_backgroundsvset_buildbackgroundsvsetjob_cleanupbackgroundsvsetjob.py @@ -1,9 +1,10 @@ # Generated by Django 3.2.12 on 2022-06-27 13:40 +import uuid + import bgjobs.models from django.db import migrations, models import django.db.models.deletion -import uuid class Migration(migrations.Migration): @@ -91,7 +92,10 @@ class Migration(migrations.Migration): ), ), ], - options={"ordering": ("-date_created",), "abstract": False,}, + options={ + "ordering": ("-date_created",), + "abstract": False, + }, bases=(bgjobs.models.JobModelMessageMixin, models.Model), ), migrations.CreateModel( @@ -122,7 +126,10 @@ class Migration(migrations.Migration): ), ), ], - options={"ordering": ("-date_created",), "abstract": False,}, + options={ + "ordering": ("-date_created",), + "abstract": False, + }, bases=(bgjobs.models.JobModelMessageMixin, models.Model), ), migrations.CreateModel( diff --git a/svs/migrations/0018_extend_structuralvariant.py b/svs/migrations/0018_extend_structuralvariant.py index 2141629ea..6fa3268ef 100644 --- a/svs/migrations/0018_extend_structuralvariant.py +++ b/svs/migrations/0018_extend_structuralvariant.py @@ -19,9 +19,8 @@ - num_hemi_ref """ -from django.db import migrations, models from django.conf import settings - +from django.db import migrations, models if settings.IS_TESTING: operations = [ diff --git a/svs/models.py b/svs/models.py index deafbb29a..61d47cdec 100644 --- a/svs/models.py +++ b/svs/models.py @@ -1,26 +1,23 @@ -import uuid as uuid_object from datetime import datetime, timedelta +import uuid as uuid_object -from django.utils.timezone import localtime - -from varfish.utils import JSONField -from variants.helpers import get_engine from bgjobs.models import BackgroundJob, JobModelMessageMixin -from postgres_copy import CopyManager - from django.conf import settings from django.contrib.postgres.fields import ArrayField -from django.urls import reverse from django.db import models from django.db.models.signals import pre_delete from django.dispatch import receiver +from django.urls import reverse from django.utils import timezone +from django.utils.timezone import localtime +from postgres_copy import CopyManager from projectroles.models import Project from projectroles.plugins import get_backend_api from sqlalchemy import and_ -from variants.models import Case, VARIANT_RATING_CHOICES, VariantImporterBase, SiteBgJobBase -from variants.helpers import get_meta +from varfish.utils import JSONField +from variants.helpers import get_engine, get_meta +from variants.models import VARIANT_RATING_CHOICES, Case, SiteBgJobBase, VariantImporterBase #: Django user model. AUTH_USER_MODEL = getattr(settings, "AUTH_USER_MODEL", "auth.User") @@ -535,8 +532,7 @@ def get_absolute_url(self): class SvAnnotationReleaseInfo(models.Model): - """Model to track the database releases used during annotation of a case. - """ + """Model to track the database releases used during annotation of a case.""" #: Release of genomebuild genomebuild = models.CharField(max_length=32, default="GRCh37") @@ -547,9 +543,15 @@ class SvAnnotationReleaseInfo(models.Model): #: Data release release = models.CharField(max_length=512) #: Link to case - case = models.ForeignKey(Case, on_delete=models.CASCADE,) + case = models.ForeignKey( + Case, + on_delete=models.CASCADE, + ) #: Link to variant set - variant_set = models.ForeignKey(StructuralVariantSet, on_delete=models.CASCADE,) + variant_set = models.ForeignKey( + StructuralVariantSet, + on_delete=models.CASCADE, + ) class Meta: unique_together = ("genomebuild", "table", "variant_set") @@ -730,7 +732,10 @@ def get_human_readable_type(self): return "Build background SV set" def get_absolute_url(self): - return reverse("svs:build-bg-sv-set-job-detail", kwargs={"job": self.sodar_uuid},) + return reverse( + "svs:build-bg-sv-set-job-detail", + kwargs={"job": self.sodar_uuid}, + ) class CleanupBackgroundSvSetJob(SiteBgJobBase): @@ -746,4 +751,7 @@ def get_human_readable_type(self): return "Cleanup building background SV set" def get_absolute_url(self): - return reverse("svs:cleanup-bg-sv-set-job-detail", kwargs={"job": self.sodar_uuid},) + return reverse( + "svs:cleanup-bg-sv-set-job-detail", + kwargs={"job": self.sodar_uuid}, + ) diff --git a/svs/plugins.py b/svs/plugins.py index 9836e89a2..d7ef1f5fd 100644 --- a/svs/plugins.py +++ b/svs/plugins.py @@ -1,13 +1,13 @@ -from projectroles.plugins import ProjectAppPluginPoint from bgjobs.plugins import BackgroundJobsPluginPoint +from projectroles.plugins import ProjectAppPluginPoint from .models import ( + BuildBackgroundSvSetJob, Case, + CleanupBackgroundSvSetJob, + ImportStructuralVariantBgJob, StructuralVariantComment, StructuralVariantFlags, - ImportStructuralVariantBgJob, - BuildBackgroundSvSetJob, - CleanupBackgroundSvSetJob, ) from .urls import urlpatterns diff --git a/svs/queries.py b/svs/queries.py index a05fd5f4c..edb29e6d8 100644 --- a/svs/queries.py +++ b/svs/queries.py @@ -1,47 +1,43 @@ """Building queries for structural variants based on the ``QueryParts`` infrastructure from ``variants``.""" import enum +from itertools import chain import logging import operator import sys -from itertools import chain from django.conf import settings from django.db.models import Q -import sqlparse - -from sqlalchemy import column, VARCHAR, ARRAY, any_, text -from sqlalchemy.sql import select, func, and_, or_, not_, true, cast, case, desc +from sqlalchemy import ARRAY, VARCHAR, any_, column, text +from sqlalchemy.sql import and_, case, cast, desc, func, not_, or_, select, true from sqlalchemy.sql.functions import coalesce -from sqlalchemy.types import Integer, Float +from sqlalchemy.types import Float, Integer +import sqlparse -from regmaps.models import RegMapCollection, RegElement, RegInteraction -from .models import ( - StructuralVariant, - StructuralVariantGeneAnnotation, - StructuralVariantComment, - StructuralVariantFlags, - StructuralVariantSet, - BackgroundSv, - BackgroundSvSet, -) +from geneinfo.models import Hgnc from genomicfeatures.models import ( EnsemblRegulatoryFeature, GeneInterval, - TadSet, - TadInterval, TadBoundaryInterval, + TadInterval, + TadSet, VistaEnhancer, ) -from geneinfo.models import Hgnc -from svdbs.models import ThousandGenomesSv, DbVarSv, ExacCnv, DgvSvs, DgvGoldStandardSvs, GnomAdSv -from variants.queries import ( - QueryParts, - QueryPartsBuilder, - ExtendQueryPartsBase, - ExtendQueryPartsGenotypeDefaultBase, - ExtendQueryPartsCaseJoinAndFilter as _ExtendQueryPartsCaseJoinAndFilter, - ExtendQueryPartsDiseaseGeneJoin as _ExtendQueryPartsDiseaseGeneJoin, +from regmaps.models import RegElement, RegInteraction, RegMapCollection +from svdbs.models import DbVarSv, DgvGoldStandardSvs, DgvSvs, ExacCnv, GnomAdSv, ThousandGenomesSv +from variants.queries import ExtendQueryPartsBase +from variants.queries import ExtendQueryPartsCaseJoinAndFilter as _ExtendQueryPartsCaseJoinAndFilter +from variants.queries import ExtendQueryPartsDiseaseGeneJoin as _ExtendQueryPartsDiseaseGeneJoin +from variants.queries import ExtendQueryPartsGenotypeDefaultBase, QueryParts, QueryPartsBuilder + +from .models import ( + BackgroundSv, + BackgroundSvSet, + StructuralVariant, + StructuralVariantComment, + StructuralVariantFlags, + StructuralVariantGeneAnnotation, + StructuralVariantSet, ) #: Logger to use in this module. @@ -516,9 +512,9 @@ def extend_conditions(self, _query_parts): yield result # Whether to include coding/non-coding transcripts. if not self.kwargs["transcripts_coding"]: - yield self._transcript_coding_field() == False # equality from SQL Alchemy + yield self._transcript_coding_field() == False # noqa: E712 if not self.kwargs["transcripts_noncoding"]: - yield self._transcript_coding_field() == True # equality from SQL Alchemy + yield self._transcript_coding_field() == True # noqa: E712 def _effect_field(self): """Return the effects field of ``StructuralVariantGeneAnnotation`` to use based on the selected database.""" diff --git a/svs/rules.py b/svs/rules.py index 4a2a5c5b4..aef12cd6f 100644 --- a/svs/rules.py +++ b/svs/rules.py @@ -1,6 +1,5 @@ -import rules from projectroles import rules as pr_rules - +import rules rules.add_perm( "svs.view_data", diff --git a/svs/tasks.py b/svs/tasks.py index 7a103cde1..73e4367e5 100644 --- a/svs/tasks.py +++ b/svs/tasks.py @@ -1,12 +1,11 @@ +from bgjobs.models import BackgroundJob +from celery.schedules import crontab from django.conf import settings from django.contrib.auth import get_user_model from django.db import transaction from config.celery import app -from celery.schedules import crontab - -from bgjobs.models import BackgroundJob -from svs import models, bg_db +from svs import bg_db, models #: The User model to use. from svs.models import BuildBackgroundSvSetJob, CleanupBackgroundSvSetJob diff --git a/svs/tests/factories.py b/svs/tests/factories.py index 8a7651446..5cba07517 100644 --- a/svs/tests/factories.py +++ b/svs/tests/factories.py @@ -1,8 +1,11 @@ """Factory Boy factory classes for ``svs``.""" +import typing +import uuid + +import attr import binning import factory -import uuid from svs.forms import ( FILTER_FORM_TRANSLATE_EFFECTS, @@ -11,17 +14,16 @@ ) from variants.models import Case from variants.tests.factories import CaseFactory + from ..models import ( + BackgroundSv, + BackgroundSvSet, StructuralVariant, - StructuralVariantGeneAnnotation, - StructuralVariantFlags, StructuralVariantComment, + StructuralVariantFlags, + StructuralVariantGeneAnnotation, StructuralVariantSet, - BackgroundSv, - BackgroundSvSet, ) -import typing -import attr def default_genotypes(): @@ -304,7 +306,9 @@ class BackgroundSvFactory(factory.django.DjangoModelFactory): class Meta: model = BackgroundSv - bg_sv_set = factory.SubFactory(BackgroundSvSetFactory,) + bg_sv_set = factory.SubFactory( + BackgroundSvSetFactory, + ) release = "GRCh37" chromosome = factory.Iterator(list(map(str, range(1, 23))) + ["X", "Y"]) diff --git a/svs/tests/helpers.py b/svs/tests/helpers.py index edacccbaa..bb3bebba0 100644 --- a/svs/tests/helpers.py +++ b/svs/tests/helpers.py @@ -1,10 +1,11 @@ """Common helper code for tests""" -from variants.helpers import get_engine from django.test import TestCase from genomicfeatures.tests.factories import TadSetFactory +from variants.helpers import get_engine from variants.models import Case, CaseAwareProject + from .factories import FormDataFactory diff --git a/svs/tests/test_bg_db.py b/svs/tests/test_bg_db.py index 57a292b34..4d22aaddc 100644 --- a/svs/tests/test_bg_db.py +++ b/svs/tests/test_bg_db.py @@ -3,13 +3,13 @@ import attrs from bgjobs.models import BackgroundJob +from django.contrib.auth import get_user_model from test_plus.test import TestCase from svs import bg_db -from django.contrib.auth import get_user_model #: The User model to use. -from svs.models import BuildBackgroundSvSetJob, BackgroundSvSet, BackgroundSv +from svs.models import BackgroundSv, BackgroundSvSet, BuildBackgroundSvSetJob User = get_user_model() @@ -278,7 +278,10 @@ def testWithSingleChromSingleVar(self): expected = { "chr1": [ bg_db.SvCluster( - params=algo.params, rng=algo.rng, mean=record_1, records=[record_1], + params=algo.params, + rng=algo.rng, + mean=record_1, + records=[record_1], ) ] } @@ -312,7 +315,12 @@ def testWithSingleChromMultipleVariants(self): end=2250, orientation=bg_db.PairedEndOrientation.THREE_TO_FIVE, ) - records = {"chr1": [record_1, record_2,]} + records = { + "chr1": [ + record_1, + record_2, + ] + } algo, clusters = self._run_clustering(records) expected = { "chr1": [ @@ -343,8 +351,12 @@ def testWithTwoChromsSingleVar(self): orientation=bg_db.PairedEndOrientation.THREE_TO_FIVE, ) records = { - "chr1": [record_1,], - "chr2": [record_2,], + "chr1": [ + record_1, + ], + "chr2": [ + record_2, + ], } algo, clusters = self._run_clustering(records) expected = { @@ -413,8 +425,14 @@ def testWithTwoChromsMultipleVariants(self): orientation=bg_db.PairedEndOrientation.THREE_TO_FIVE, ) records = { - "chr1": [record_1, record_2,], - "chr2": [record_3, record_4,], + "chr1": [ + record_1, + record_2, + ], + "chr2": [ + record_3, + record_4, + ], } algo, clusters = self._run_clustering(records) expected = { @@ -442,7 +460,9 @@ def testWithBreakend(self): orientation=bg_db.PairedEndOrientation.THREE_TO_FIVE, ) record_2 = attrs.evolve( - record_1, pos=1001, orientation=bg_db.PairedEndOrientation.THREE_TO_THREE, + record_1, + pos=1001, + orientation=bg_db.PairedEndOrientation.THREE_TO_THREE, ) record_3 = attrs.evolve(record_1, end=record_1.end + 1) mean_1 = attrs.evolve(record_1, end=record_1.end + 1) @@ -462,7 +482,12 @@ def testWithBreakend(self): class TestModelToAttrs(TestCase): def testWithDel(self): - sv_model = StructuralVariantFactory(chromosome="3", chromosome_no=3, start=1000, end=2000,) + sv_model = StructuralVariantFactory( + chromosome="3", + chromosome_no=3, + start=1000, + end=2000, + ) sv_record = bg_db.sv_model_to_attrs(sv_model) self.maxDiff = None expected = bg_db.SvRecord( @@ -528,7 +553,9 @@ def setUp(self): job_type="variants.export_file_bg_job", user=self.root_user, ) - self.build_sv_set_bg_job = BuildBackgroundSvSetJob.objects.create(bg_job=self.bg_job,) + self.build_sv_set_bg_job = BuildBackgroundSvSetJob.objects.create( + bg_job=self.bg_job, + ) def testWithNoRecords(self): self.assertEqual(BackgroundSvSet.objects.count(), 0) diff --git a/svs/tests/test_import.py b/svs/tests/test_import.py index 9a7dde6bd..4706ca43e 100644 --- a/svs/tests/test_import.py +++ b/svs/tests/test_import.py @@ -9,9 +9,9 @@ from svs.models import ( ImportStructuralVariantBgJob, - StructuralVariantSet, StructuralVariant, StructuralVariantGeneAnnotation, + StructuralVariantSet, ) from svs.tasks import run_import_structural_variants_bg_job from svs.tests.factories import StructuralVariantGeneAnnotationFactory diff --git a/svs/tests/test_models.py b/svs/tests/test_models.py index c28cbb12a..66aafb5ef 100644 --- a/svs/tests/test_models.py +++ b/svs/tests/test_models.py @@ -5,23 +5,24 @@ from django.utils import timezone from svs.models import ( + BackgroundSv, + BackgroundSvSet, StructuralVariant, - StructuralVariantGeneAnnotation, StructuralVariantComment, StructuralVariantFlags, + StructuralVariantGeneAnnotation, StructuralVariantSet, cleanup_variant_sets, - BackgroundSv, - BackgroundSvSet, ) + from .factories import ( + BackgroundSvFactory, + BackgroundSvSetFactory, + StructuralVariantCommentFactory, StructuralVariantFactory, - StructuralVariantGeneAnnotationFactory, StructuralVariantFlagsFactory, - StructuralVariantCommentFactory, + StructuralVariantGeneAnnotationFactory, StructuralVariantSetFactory, - BackgroundSvFactory, - BackgroundSvSetFactory, ) diff --git a/svs/tests/test_queries.py b/svs/tests/test_queries.py index 1069e7bef..9c017b517 100644 --- a/svs/tests/test_queries.py +++ b/svs/tests/test_queries.py @@ -6,22 +6,23 @@ from genomicfeatures.tests.factories import ( EnsemblRegulatoryFeatureFactory, GeneIntervalFactory, - TadSetFactory, - TadIntervalFactory, TadBoundaryIntervalFactory, + TadIntervalFactory, + TadSetFactory, VistaEnhancerFactory, ) -from regmaps.tests.factories import RegElementFactory, RegMapFactory, RegElementTypeFactory +from regmaps.tests.factories import RegElementFactory, RegElementTypeFactory, RegMapFactory from svdbs.tests.factories import GnomAdSvFactory + +from ..models import SV_SUB_TYPE_CHOICES, SV_TYPE_CHOICES, StructuralVariant +from ..queries import SingleCaseFilterQuery from .factories import ( + BackgroundSvFactory, StructuralVariantFactory, StructuralVariantGeneAnnotationFactory, StructuralVariantSetFactory, - BackgroundSvFactory, ) from .helpers import QueryTestBase -from ..models import SV_TYPE_CHOICES, SV_SUB_TYPE_CHOICES, StructuralVariant -from ..queries import SingleCaseFilterQuery class SvsInCaseWithDeNovoGenotypeFilterQueryTest(QueryTestBase): @@ -948,7 +949,9 @@ def setUp(self): self.case = self.variant_set.case self.sv = StructuralVariantFactory(variant_set=self.variant_set) StructuralVariantGeneAnnotationFactory( - sv=self.sv, refseq_transcript_coding=True, ensembl_transcript_coding=True, + sv=self.sv, + refseq_transcript_coding=True, + ensembl_transcript_coding=True, ) def testIncludeTranscriptCoding(self): @@ -1330,7 +1333,10 @@ def testOverlapFails(self): def testOverlapAnyValidationPasses(self): result = self.run_query( SingleCaseFilterQuery, - {self.regmap_key_element: ["__any__"], self.regmap_key_map: ["__any__"],}, + { + self.regmap_key_element: ["__any__"], + self.regmap_key_map: ["__any__"], + }, 1, ) self.assertUUIDEquals(self.sv.sv_uuid, result[0]["sv_uuid"]) @@ -1358,7 +1364,12 @@ def testNoOverlapPasses(self): self.element.end = self.element.start + 20 self.element.save() result = self.run_query( - SingleCaseFilterQuery, {self.regmap_key_element: [], self.regmap_key_map: [],}, 1 + SingleCaseFilterQuery, + { + self.regmap_key_element: [], + self.regmap_key_map: [], + }, + 1, ) self.assertUUIDEquals(self.sv.sv_uuid, result[0]["sv_uuid"]) self.assertEqual( @@ -1419,7 +1430,10 @@ def testNoOverlapFails(self): self.element.save() self.run_query( SingleCaseFilterQuery, - {self.regmap_key_element: ["__any__"], self.regmap_key_map: ["__any__"],}, + { + self.regmap_key_element: ["__any__"], + self.regmap_key_map: ["__any__"], + }, 0, ) diff --git a/svs/tests/test_views.py b/svs/tests/test_views.py index acef6d981..7221e833b 100644 --- a/svs/tests/test_views.py +++ b/svs/tests/test_views.py @@ -14,7 +14,8 @@ def setUp(self): super().setUp() self.case, _, self.variant_set = CaseWithVariantSetFactory.get(project=self.project) self.sv_gene_annotation = StructuralVariantGeneAnnotationFactory( - sv__variant_set__case=self.case, sv__variant_set=self.variant_set, + sv__variant_set__case=self.case, + sv__variant_set=self.variant_set, ) self.sv = StructuralVariant.objects.get(sv_uuid=self.sv_gene_annotation.sv_uuid) self.sv_gene_annotation2 = StructuralVariantGeneAnnotationFactory( diff --git a/svs/urls.py b/svs/urls.py index 6d072225a..19fa30f9f 100644 --- a/svs/urls.py +++ b/svs/urls.py @@ -1,6 +1,6 @@ from django.conf.urls import url -from . import views +from . import views app_name = "svs" diff --git a/svs/views.py b/svs/views.py index 6aab7636d..000bcfbc1 100644 --- a/svs/views.py +++ b/svs/views.py @@ -7,42 +7,42 @@ from django.apps import apps from django.core.exceptions import ImproperlyConfigured -from django.urls import resolve -from projectroles.models import Project - -from variants.helpers import get_engine -from projectroles.views import LoginRequiredMixin from django.db import transaction from django.forms import model_to_dict -from django.http import HttpResponse, Http404, JsonResponse -from django.shortcuts import render, redirect, get_object_or_404, reverse +from django.http import Http404, HttpResponse, JsonResponse +from django.shortcuts import get_object_or_404, render +from django.urls import resolve from django.utils import timezone from django.views import View -from django.views.generic import FormView, DetailView, TemplateView +from django.views.generic import DetailView, FormView, TemplateView from django.views.generic.detail import SingleObjectMixin, SingleObjectTemplateResponseMixin +from projectroles.models import Project from projectroles.plugins import get_backend_api -from projectroles.views import LoggedInPermissionMixin, ProjectContextMixin, ProjectPermissionMixin +from projectroles.views import ( + LoggedInPermissionMixin, + LoginRequiredMixin, + ProjectContextMixin, + ProjectPermissionMixin, +) from geneinfo.views import get_gene_infos from regmaps.models import RegElement, RegInteraction -from .forms import ( - FilterForm, - StructuralVariantCommentForm, - StructuralVariantFlagsForm, -) +from variants.helpers import get_engine +from variants.models import Case +from variants.views import UUIDEncoder + +from .forms import FilterForm, StructuralVariantCommentForm, StructuralVariantFlagsForm from .models import ( - StructuralVariantFlags, - StructuralVariantComment, + BuildBackgroundSvSetJob, + CleanupBackgroundSvSetJob, + ImportStructuralVariantBgJob, StructuralVariant, + StructuralVariantComment, + StructuralVariantFlags, StructuralVariantGeneAnnotation, - ImportStructuralVariantBgJob, StructuralVariantSet, - BuildBackgroundSvSetJob, - CleanupBackgroundSvSetJob, ) from .queries import SingleCaseFilterQuery, best_matching_flags -from variants.models import Case -from variants.views import UUIDEncoder class CaseFilterView( @@ -370,7 +370,11 @@ def post(self, *_args, **kwargs): class MultiStructuralVariantFlagsAndCommentApiView( - LoginRequiredMixin, LoggedInPermissionMixin, ProjectPermissionMixin, ProjectContextMixin, View, + LoginRequiredMixin, + LoggedInPermissionMixin, + ProjectPermissionMixin, + ProjectContextMixin, + View, ): """A view that returns JSON for the ``SmallVariantFlags`` for a variant of a case and allows updates.""" @@ -529,8 +533,7 @@ class ImportStructuralVariantsJobDetailView( ProjectContextMixin, DetailView, ): - """Display status and further details of import case background jobs. - """ + """Display status and further details of import case background jobs.""" permission_required = "variants.view_data" template_name = "svs/import_job_detail.html" @@ -643,16 +646,6 @@ def get_form_kwargs(self): result["case"] = self.get_case_object() return result - def get_context_data(self, **kwargs): - """Put the ``Case`` object into the context.""" - context = super().get_context_data(**kwargs) - context["object"] = self.get_case_object() - context["pedigree"] = self.get_case_object().get_filtered_pedigree_with_samples() - context["variant_set_exists"] = StructuralVariantSet.objects.filter( - case_id=context["object"].id, state="active" - ).exists() - return context - def get_context_data(self, *args, **kwargs): context_data = super().get_context_data(*args, **kwargs) @@ -760,10 +753,12 @@ def _augment_context_data(self, context_data, form_data, results): class BuildBackgroundSvSetJobDetailView( - LoginRequiredMixin, LoggedInPermissionMixin, ProjectContextMixin, DetailView, + LoginRequiredMixin, + LoggedInPermissionMixin, + ProjectContextMixin, + DetailView, ): - """Display status and further details of build sv set background jobs. - """ + """Display status and further details of build sv set background jobs.""" permission_required = "variants.view_data" template_name = "svs/build_bg_job_detail.html" @@ -773,10 +768,12 @@ class BuildBackgroundSvSetJobDetailView( class CleanupBackgroundSvSetJobDetailView( - LoginRequiredMixin, LoggedInPermissionMixin, ProjectContextMixin, DetailView, + LoginRequiredMixin, + LoggedInPermissionMixin, + ProjectContextMixin, + DetailView, ): - """Display status and further details of cleanup sv set background jobs. - """ + """Display status and further details of cleanup sv set background jobs.""" permission_required = "variants.view_data" template_name = "svs/cleanup_bg_job_detail.html" diff --git a/var_stats_qc/migrations/0001_initial.py b/var_stats_qc/migrations/0001_initial.py index 176b00b32..278f057af 100644 --- a/var_stats_qc/migrations/0001_initial.py +++ b/var_stats_qc/migrations/0001_initial.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals from django.db import migrations, models + import var_stats_qc.models diff --git a/var_stats_qc/qc.py b/var_stats_qc/qc.py index 45ddf09f0..19058c0f3 100644 --- a/var_stats_qc/qc.py +++ b/var_stats_qc/qc.py @@ -4,7 +4,7 @@ import numpy as np from sqlalchemy import or_ -from sqlalchemy.sql import select, and_, not_, func +from sqlalchemy.sql import and_, func, not_, select from .models import ReferenceSite diff --git a/varfish/users/adapters.py b/varfish/users/adapters.py index bc5b290f3..f60d2a9dc 100644 --- a/varfish/users/adapters.py +++ b/varfish/users/adapters.py @@ -1,6 +1,6 @@ -from django.conf import settings from allauth.account.adapter import DefaultAccountAdapter from allauth.socialaccount.adapter import DefaultSocialAccountAdapter +from django.conf import settings class AccountAdapter(DefaultAccountAdapter): diff --git a/varfish/users/admin.py b/varfish/users/admin.py index 3dc6b4961..aa0fa4f46 100644 --- a/varfish/users/admin.py +++ b/varfish/users/admin.py @@ -2,6 +2,7 @@ from django.contrib import admin from django.contrib.auth.admin import UserAdmin as AuthUserAdmin from django.contrib.auth.forms import UserChangeForm, UserCreationForm + from .models import User diff --git a/varfish/users/apps.py b/varfish/users/apps.py index 4118215e0..fd1bbbe06 100644 --- a/varfish/users/apps.py +++ b/varfish/users/apps.py @@ -7,7 +7,7 @@ class UsersConfig(AppConfig): def ready(self): """Override this to put in: - Users system checks - Users signal registration + Users system checks + Users signal registration """ pass diff --git a/varfish/users/migrations/0002_auto_20181015_1316.py b/varfish/users/migrations/0002_auto_20181015_1316.py index 443e56412..5b39f540c 100644 --- a/varfish/users/migrations/0002_auto_20181015_1316.py +++ b/varfish/users/migrations/0002_auto_20181015_1316.py @@ -2,9 +2,10 @@ # Generated by Django 1.11.10 on 2018-10-15 13:16 from __future__ import unicode_literals -from django.db import migrations, models import uuid +from django.db import migrations, models + class Migration(migrations.Migration): diff --git a/varfish/users/migrations/0003_auto_20200505_0724.py b/varfish/users/migrations/0003_auto_20200505_0724.py index ea841d1cf..9520aac92 100644 --- a/varfish/users/migrations/0003_auto_20200505_0724.py +++ b/varfish/users/migrations/0003_auto_20200505_0724.py @@ -12,5 +12,8 @@ class Migration(migrations.Migration): ] operations = [ - migrations.AlterModelOptions(name="user", options={"ordering": ["name", "username"]},), + migrations.AlterModelOptions( + name="user", + options={"ordering": ["name", "username"]}, + ), ] diff --git a/varfish/users/models.py b/varfish/users/models.py index c7b6d6fa8..2c0c4ecd0 100644 --- a/varfish/users/models.py +++ b/varfish/users/models.py @@ -1,5 +1,5 @@ -from projectroles.models import SODARUser from django.conf import settings +from projectroles.models import SODARUser class User(SODARUser): diff --git a/varfish/users/tasks.py b/varfish/users/tasks.py index 0d862f3cb..3059d1ca3 100644 --- a/varfish/users/tasks.py +++ b/varfish/users/tasks.py @@ -3,7 +3,7 @@ from celery.schedules import crontab from django.conf import settings from django.core.management import call_command -from projectroles.models import RemoteSite, SODAR_CONSTANTS +from projectroles.models import SODAR_CONSTANTS, RemoteSite from config.celery import app diff --git a/varfish/users/views.py b/varfish/users/views.py index 1b1b0f321..fcafecd6d 100644 --- a/varfish/users/views.py +++ b/varfish/users/views.py @@ -1,6 +1,5 @@ from django.urls import reverse from django.views.generic import DetailView, ListView, RedirectView, UpdateView - from projectroles.views import LoginRequiredMixin from .models import User diff --git a/varfish/utils.py b/varfish/utils.py index 40c3c20d8..723b9fccd 100644 --- a/varfish/utils.py +++ b/varfish/utils.py @@ -3,6 +3,7 @@ import json import django.db.models.fields.json + from varfish.users.models import User @@ -22,7 +23,7 @@ def get_subclasses(classes, level=0): def receiver_subclasses(signal, sender, dispatch_uid_prefix, **kwargs): - """ A decorator for connecting receivers and all receiver's subclasses to signals. + """A decorator for connecting receivers and all receiver's subclasses to signals. Used by passing in the signal and keyword arguments to connect:: @@ -43,8 +44,7 @@ def _decorator(func): class JSONField(django.db.models.fields.json.JSONField): - """Helper JSONField class that works when SQLAlchemy (via aldjemy) is already parsing JSON values into Python types. - """ + """Helper JSONField class that works when SQLAlchemy (via aldjemy) is already parsing JSON values into Python types.""" def from_db_value(self, value, expression, connection): if value is None: diff --git a/variants/admin.py b/variants/admin.py index e0e17e9d8..2941f563e 100644 --- a/variants/admin.py +++ b/variants/admin.py @@ -1,23 +1,23 @@ from django.contrib import admin from .models import ( - Case, CaddSubmissionBgJob, + Case, + CasePhenotypeTerms, DistillerSubmissionBgJob, - SmallVariant, ExportFileBgJob, ExportFileJobResult, - SmallVariantFlags, - SmallVariantQuery, - SmallVariantComment, FilterBgJob, - ProjectCasesFilterBgJob, - SyncCaseListBgJob, - SyncCaseResultMessage, ImportVariantsBgJob, + ProjectCasesFilterBgJob, + SmallVariant, + SmallVariantComment, + SmallVariantFlags, + SmallVariantQuery, SmallVariantSet, SpanrSubmissionBgJob, - CasePhenotypeTerms, + SyncCaseListBgJob, + SyncCaseResultMessage, ) # Register your models here. diff --git a/variants/file_export.py b/variants/file_export.py index 02bebad6b..d0a1d5c0b 100644 --- a/variants/file_export.py +++ b/variants/file_export.py @@ -1,43 +1,44 @@ """This module contains the code for file export""" -import datetime -import math from collections import OrderedDict +import contextlib +import datetime from datetime import timedelta +import math from tempfile import NamedTemporaryFile -import contextlib -from extra_annos.models import ExtraAnnoField -from variants.helpers import get_engine -from django.utils import timezone from django.conf import settings +from django.utils import timezone +from projectroles.plugins import get_backend_api import vcfpy import wrapt import xlsxwriter from cohorts.models import Cohort +from extra_annos.models import ExtraAnnoField +from variants.helpers import get_engine + from .models import ( Case, CaseAwareProject, ExportFileJobResult, ExportProjectCasesFileBgJobResult, SmallVariantComment, - annotate_with_phenotype_scores, - annotate_with_pathogenicity_scores, + VariantScoresFactory, annotate_with_joint_scores, + annotate_with_pathogenicity_scores, + annotate_with_phenotype_scores, annotate_with_transcripts, - unroll_extra_annos_result, prioritize_genes, - VariantScoresFactory, + unroll_extra_annos_result, ) -from .templatetags.variants_tags import flag_class -from projectroles.plugins import get_backend_api from .queries import ( CaseExportTableQuery, CaseExportVcfQuery, ProjectExportTableQuery, ProjectExportVcfQuery, ) +from .templatetags.variants_tags import flag_class #: Color to use for variants flagged as positive. BG_COLOR_POSITIVE = "#dc3848" @@ -245,8 +246,7 @@ def _is_jannovar_enabled(): class CaseExporterBase: - """Base class for export of (filtered) case data from single case or all cases of a project. - """ + """Base class for export of (filtered) case data from single case or all cases of a project.""" #: The query class to use for building single-case queries. query_class_single_case = None @@ -509,8 +509,7 @@ def _write_leading(self): """ def _write_variants(self): - """Write out the actual data, override called functions rather than this one. - """ + """Write out the actual data, override called functions rather than this one.""" self._begin_write_variants() self._write_variants_header() self._write_variants_data() diff --git a/variants/forms.py b/variants/forms.py index f4c559e78..fb1248228 100644 --- a/variants/forms.py +++ b/variants/forms.py @@ -1,25 +1,23 @@ -import shutil from functools import lru_cache -from itertools import chain import io +from itertools import chain import os +import re +import shutil import tempfile -import vcfpy -from django.conf import settings from django import forms +from django.conf import settings from django.core.files.storage import FileSystemStorage -from django.utils.text import get_valid_filename - -from cohorts.models import Cohort -from .models import SmallVariantComment, SmallVariantFlags, AcmgCriteriaRating, Case, CaseComments -from .templatetags.variants_tags import only_source_name, get_term_description -from geneinfo.models import Hgnc, HpoName, Hpo from django.db.models import Q +from django.utils.text import get_valid_filename from projectroles.app_settings import AppSettingAPI +import vcfpy -import re +from geneinfo.models import Hgnc, Hpo, HpoName +from .models import AcmgCriteriaRating, Case, CaseComments, SmallVariantComment, SmallVariantFlags +from .templatetags.variants_tags import get_term_description, only_source_name app_settings = AppSettingAPI() @@ -107,7 +105,6 @@ def save(self, commit=True): if idx == -1: self.instance.pedigree[i][key] = "0" else: - x = self.instance.pedigree parent_name = self.instance.pedigree[idx]["patient"] self.instance.pedigree[i][key] = parent_name @@ -553,7 +550,11 @@ class ExportFileResubmitForm(forms.Form): class ExportProjectCasesFileResubmitForm(forms.Form): file_type = forms.ChoiceField( initial="xlsx", - choices=(("xlsx", "Excel (.xlsx)"), ("tsv", "TSV (.tsv)"), ("vcf", "VCF (.vcf.gz)"),), + choices=( + ("xlsx", "Excel (.xlsx)"), + ("tsv", "TSV (.tsv)"), + ("vcf", "VCF (.vcf.gz)"), + ), widget=forms.Select(attrs={"class": "form-control"}), ) @@ -813,7 +814,10 @@ def __init__(self, *args, **kwargs): min_value=0, required=False, widget=forms.TextInput( - attrs={"placeholder": "Maximal frequency in mtDB", "class": "numberDecimal",} + attrs={ + "placeholder": "Maximal frequency in mtDB", + "class": "numberDecimal", + } ), ) self.fields["mtdb_count"] = forms.IntegerField( @@ -821,7 +825,10 @@ def __init__(self, *args, **kwargs): initial=10, required=False, widget=forms.TextInput( - attrs={"placeholder": "Maximal count in mtDB", "class": "numberInteger",} + attrs={ + "placeholder": "Maximal count in mtDB", + "class": "numberInteger", + } ), ) self.fields["helixmtdb_enabled"] = forms.BooleanField( @@ -834,7 +841,10 @@ def __init__(self, *args, **kwargs): min_value=0, required=False, widget=forms.TextInput( - attrs={"placeholder": "Maximal frequency in HelixMTdb", "class": "numberDecimal",} + attrs={ + "placeholder": "Maximal frequency in HelixMTdb", + "class": "numberDecimal", + } ), ) self.fields["helixmtdb_hom_count"] = forms.IntegerField( @@ -842,7 +852,10 @@ def __init__(self, *args, **kwargs): initial=10, required=False, widget=forms.TextInput( - attrs={"placeholder": "Maximal hom. count in HelixMTdb", "class": "numberInteger",} + attrs={ + "placeholder": "Maximal hom. count in HelixMTdb", + "class": "numberInteger", + } ), ) self.fields["helixmtdb_het_count"] = forms.IntegerField( @@ -850,7 +863,10 @@ def __init__(self, *args, **kwargs): initial=10, required=False, widget=forms.TextInput( - attrs={"placeholder": "Maximal het. count in HelixMTdb", "class": "numberInteger",} + attrs={ + "placeholder": "Maximal het. count in HelixMTdb", + "class": "numberInteger", + } ), ) self.fields["mitomap_enabled"] = forms.BooleanField(label="", required=False, initial=True) @@ -861,7 +877,10 @@ def __init__(self, *args, **kwargs): min_value=0, required=False, widget=forms.TextInput( - attrs={"placeholder": "Maximal frequency in MITOMAP", "class": "numberDecimal",} + attrs={ + "placeholder": "Maximal frequency in MITOMAP", + "class": "numberDecimal", + } ), ) self.fields["mitomap_count"] = forms.IntegerField( @@ -869,7 +888,10 @@ def __init__(self, *args, **kwargs): initial=10, required=False, widget=forms.TextInput( - attrs={"placeholder": "Maximal count in MITOMAP", "class": "numberInteger",} + attrs={ + "placeholder": "Maximal count in MITOMAP", + "class": "numberInteger", + } ), ) diff --git a/variants/migrations/0001_initial.py b/variants/migrations/0001_initial.py index d4ce1b1a1..126dee519 100644 --- a/variants/migrations/0001_initial.py +++ b/variants/migrations/0001_initial.py @@ -2,10 +2,11 @@ # Generated by Django 1.11.16 on 2018-10-19 10:45 from __future__ import unicode_literals +import uuid + import django.contrib.postgres.fields from django.db import migrations, models import django.db.models.deletion -import uuid class Migration(migrations.Migration): diff --git a/variants/migrations/0002_exportfilebgjob_exportfilejobresult.py b/variants/migrations/0002_exportfilebgjob_exportfilejobresult.py index 88cc417a2..325d07cfb 100644 --- a/variants/migrations/0002_exportfilebgjob_exportfilejobresult.py +++ b/variants/migrations/0002_exportfilebgjob_exportfilejobresult.py @@ -2,9 +2,10 @@ # Generated by Django 1.11.16 on 2018-10-26 12:27 from __future__ import unicode_literals +import uuid + from django.db import migrations, models import django.db.models.deletion -import uuid class Migration(migrations.Migration): @@ -29,7 +30,10 @@ class Migration(migrations.Migration): "sodar_uuid", models.UUIDField(default=uuid.uuid4, help_text="Case SODAR UUID", unique=True), ), - ("query_args", models.JSONField(help_text="(Validated) query parameters"),), + ( + "query_args", + models.JSONField(help_text="(Validated) query parameters"), + ), ( "file_type", models.CharField( diff --git a/variants/migrations/0005_load_btree_gin_extension.py b/variants/migrations/0005_load_btree_gin_extension.py index 0481e4701..d42eeffc0 100644 --- a/variants/migrations/0005_load_btree_gin_extension.py +++ b/variants/migrations/0005_load_btree_gin_extension.py @@ -2,9 +2,8 @@ # Generated by Django 1.11.16 on 2018-11-01 07:52 from __future__ import unicode_literals -from django.db import migrations - from django.contrib.postgres.operations import BtreeGinExtension +from django.db import migrations class Migration(migrations.Migration): diff --git a/variants/migrations/0008_auto_20181113_1912.py b/variants/migrations/0008_auto_20181113_1912.py index a59a7690c..33cc229f0 100644 --- a/variants/migrations/0008_auto_20181113_1912.py +++ b/variants/migrations/0008_auto_20181113_1912.py @@ -2,9 +2,10 @@ # Generated by Django 1.11.16 on 2018-11-13 19:12 from __future__ import unicode_literals +import uuid + from django.db import migrations, models import django.db.models.deletion -import uuid class Migration(migrations.Migration): diff --git a/variants/migrations/0009_smallvariantcomment.py b/variants/migrations/0009_smallvariantcomment.py index d115eb80f..c186dbba0 100644 --- a/variants/migrations/0009_smallvariantcomment.py +++ b/variants/migrations/0009_smallvariantcomment.py @@ -2,10 +2,11 @@ # Generated by Django 1.11.16 on 2018-11-13 20:12 from __future__ import unicode_literals +import uuid + from django.conf import settings from django.db import migrations, models import django.db.models.deletion -import uuid class Migration(migrations.Migration): diff --git a/variants/migrations/0016_smallvariantquery.py b/variants/migrations/0016_smallvariantquery.py index e32180baa..64aeef24b 100644 --- a/variants/migrations/0016_smallvariantquery.py +++ b/variants/migrations/0016_smallvariantquery.py @@ -2,10 +2,11 @@ # Generated by Django 1.11.16 on 2018-11-19 11:10 from __future__ import unicode_literals +import uuid + from django.conf import settings from django.db import migrations, models import django.db.models.deletion -import uuid class Migration(migrations.Migration): @@ -37,7 +38,10 @@ class Migration(migrations.Migration): ), ("form_id", models.CharField(help_text="Identifier of the form", max_length=100)), ("form_version", models.IntegerField(help_text="Version of form when saving")), - ("query_settings", models.JSONField(help_text="The query settings"),), + ( + "query_settings", + models.JSONField(help_text="The query settings"), + ), ( "name", models.CharField( diff --git a/variants/migrations/0017_distillersubmissionbgjob.py b/variants/migrations/0017_distillersubmissionbgjob.py index 3792e24d1..280fcdf1b 100644 --- a/variants/migrations/0017_distillersubmissionbgjob.py +++ b/variants/migrations/0017_distillersubmissionbgjob.py @@ -2,9 +2,10 @@ # Generated by Django 1.11.16 on 2018-11-22 11:10 from __future__ import unicode_literals +import uuid + from django.db import migrations, models import django.db.models.deletion -import uuid class Migration(migrations.Migration): @@ -29,7 +30,10 @@ class Migration(migrations.Migration): "sodar_uuid", models.UUIDField(default=uuid.uuid4, help_text="Case SODAR UUID", unique=True), ), - ("query_args", models.JSONField(help_text="(Validated) query parameters"),), + ( + "query_args", + models.JSONField(help_text="(Validated) query parameters"), + ), ( "distiller_project_id", models.CharField( diff --git a/variants/migrations/0020_auto_20181126_1930.py b/variants/migrations/0020_auto_20181126_1930.py index 0296809b7..b04af992b 100644 --- a/variants/migrations/0020_auto_20181126_1930.py +++ b/variants/migrations/0020_auto_20181126_1930.py @@ -2,10 +2,11 @@ # Generated by Django 1.11.16 on 2018-11-26 19:30 from __future__ import unicode_literals +import uuid + import bgjobs.models from django.db import migrations, models import django.db.models.deletion -import uuid class Migration(migrations.Migration): diff --git a/variants/migrations/0021_auto_20181127_0911.py b/variants/migrations/0021_auto_20181127_0911.py index 9826db198..24808def6 100644 --- a/variants/migrations/0021_auto_20181127_0911.py +++ b/variants/migrations/0021_auto_20181127_0911.py @@ -2,10 +2,11 @@ # Generated by Django 1.11.16 on 2018-11-27 09:11 from __future__ import unicode_literals +import uuid + from django.conf import settings from django.db import migrations, models import django.db.models.deletion -import uuid class Migration(migrations.Migration): @@ -37,7 +38,10 @@ class Migration(migrations.Migration): ), ("form_id", models.CharField(help_text="Identifier of the form", max_length=100)), ("form_version", models.IntegerField(help_text="Version of form when saving")), - ("query_settings", models.JSONField(help_text="The query settings"),), + ( + "query_settings", + models.JSONField(help_text="The query settings"), + ), ( "name", models.CharField( diff --git a/variants/migrations/0022_auto_20181127_1110.py b/variants/migrations/0022_auto_20181127_1110.py index 64b96e247..87b443a47 100644 --- a/variants/migrations/0022_auto_20181127_1110.py +++ b/variants/migrations/0022_auto_20181127_1110.py @@ -2,11 +2,12 @@ # Generated by Django 1.11.16 on 2018-11-27 11:10 from __future__ import unicode_literals +import uuid + import bgjobs.models from django.db import migrations, models import django.db.models.deletion import django.utils.timezone -import uuid class Migration(migrations.Migration): @@ -35,7 +36,10 @@ class Migration(migrations.Migration): "sodar_uuid", models.UUIDField(default=uuid.uuid4, help_text="Case SODAR UUID", unique=True), ), - ("query_args", models.JSONField(help_text="(Validated) query parameters"),), + ( + "query_args", + models.JSONField(help_text="(Validated) query parameters"), + ), ( "file_type", models.CharField( diff --git a/variants/migrations/0026_auto_20190128_1210.py b/variants/migrations/0026_auto_20190128_1210.py index b984b17f4..694425d2f 100644 --- a/variants/migrations/0026_auto_20190128_1210.py +++ b/variants/migrations/0026_auto_20190128_1210.py @@ -2,10 +2,11 @@ # Generated by Django 1.11.16 on 2019-01-28 12:10 from __future__ import unicode_literals +import uuid + import bgjobs.models from django.db import migrations, models import django.db.models.deletion -import uuid class Migration(migrations.Migration): diff --git a/variants/migrations/0030_auto_20190307_0723.py b/variants/migrations/0030_auto_20190307_0723.py index 717498731..ec153801b 100644 --- a/variants/migrations/0030_auto_20190307_0723.py +++ b/variants/migrations/0030_auto_20190307_0723.py @@ -2,11 +2,12 @@ # Generated by Django 1.11.20 on 2019-03-07 07:23 from __future__ import unicode_literals +import uuid + import bgjobs.models from django.conf import settings from django.db import migrations, models import django.db.models.deletion -import uuid class Migration(migrations.Migration): @@ -73,7 +74,10 @@ class Migration(migrations.Migration): ), ("form_id", models.CharField(help_text="Identifier of the form", max_length=100)), ("form_version", models.IntegerField(help_text="Version of form when saving")), - ("query_settings", models.JSONField(help_text="The query settings"),), + ( + "query_settings", + models.JSONField(help_text="The query settings"), + ), ( "name", models.CharField( diff --git a/variants/migrations/0039_acmgcriteriarating.py b/variants/migrations/0039_acmgcriteriarating.py index 16602a70a..991b4b91a 100644 --- a/variants/migrations/0039_acmgcriteriarating.py +++ b/variants/migrations/0039_acmgcriteriarating.py @@ -2,10 +2,11 @@ # Generated by Django 1.11.20 on 2019-05-17 11:26 from __future__ import unicode_literals +import uuid + from django.conf import settings from django.db import migrations, models import django.db.models.deletion -import uuid class Migration(migrations.Migration): diff --git a/variants/migrations/0041_array_cat_agg.py b/variants/migrations/0041_array_cat_agg.py index 59f68b9d4..adc0e0c4c 100644 --- a/variants/migrations/0041_array_cat_agg.py +++ b/variants/migrations/0041_array_cat_agg.py @@ -5,7 +5,6 @@ from django.db import connection, migrations - POSTGRES_VERSION = connection.cursor().connection.server_version ARRAY_TYPE = "anyarray" if POSTGRES_VERSION < 140000 else "anycompatiblearray" diff --git a/variants/migrations/0043_synccaselistbgjob_synccaseresultmessage.py b/variants/migrations/0043_synccaselistbgjob_synccaseresultmessage.py index 998f596cc..f687f2206 100644 --- a/variants/migrations/0043_synccaselistbgjob_synccaseresultmessage.py +++ b/variants/migrations/0043_synccaselistbgjob_synccaseresultmessage.py @@ -2,10 +2,11 @@ # Generated by Django 1.11.20 on 2019-06-21 09:24 from __future__ import unicode_literals +import uuid + import bgjobs.models from django.db import migrations, models import django.db.models.deletion -import uuid class Migration(migrations.Migration): diff --git a/variants/migrations/0046_partition_smallvariants_table.py b/variants/migrations/0046_partition_smallvariants_table.py index 1f41f1d19..d2e9443e2 100644 --- a/variants/migrations/0046_partition_smallvariants_table.py +++ b/variants/migrations/0046_partition_smallvariants_table.py @@ -2,10 +2,9 @@ """Setup ``variants_smallvariants`` table as partitioned. """ -from django.db import migrations, models -from django.contrib.postgres.fields import ArrayField from django.conf import settings - +from django.contrib.postgres.fields import ArrayField +from django.db import migrations, models operations = [ migrations.RunSQL( diff --git a/variants/migrations/0048_importvariantsbgjob.py b/variants/migrations/0048_importvariantsbgjob.py index ddbc16b87..dd26e1dca 100644 --- a/variants/migrations/0048_importvariantsbgjob.py +++ b/variants/migrations/0048_importvariantsbgjob.py @@ -2,11 +2,12 @@ # Generated by Django 1.11.21 on 2019-07-03 11:59 from __future__ import unicode_literals +import uuid + import bgjobs.models import django.contrib.postgres.fields from django.db import migrations, models import django.db.models.deletion -import uuid class Migration(migrations.Migration): diff --git a/variants/migrations/0053_auto_20190809_1636.py b/variants/migrations/0053_auto_20190809_1636.py index 0ca969c4d..9c3b78bad 100644 --- a/variants/migrations/0053_auto_20190809_1636.py +++ b/variants/migrations/0053_auto_20190809_1636.py @@ -2,10 +2,11 @@ # Generated by Django 1.11.20 on 2019-08-09 16:36 from __future__ import unicode_literals +import uuid + from django.conf import settings from django.db import migrations, models import django.db.models.deletion -import uuid class Migration(migrations.Migration): diff --git a/variants/migrations/0054_set_unlogged_table.py b/variants/migrations/0054_set_unlogged_table.py index bcfd1dd41..039012ab5 100644 --- a/variants/migrations/0054_set_unlogged_table.py +++ b/variants/migrations/0054_set_unlogged_table.py @@ -2,8 +2,8 @@ """SET small variant table as UNLOGGED to improve insertion performance.""" from __future__ import unicode_literals -from django.db import migrations from django.conf import settings +from django.db import migrations if not settings.IS_TESTING: operations = [migrations.RunSQL("ALTER TABLE variants_smallvariant SET UNLOGGED;")] + [ diff --git a/variants/migrations/0056_add_info_and_distance.py b/variants/migrations/0056_add_info_and_distance.py index 473805fd8..941078f54 100644 --- a/variants/migrations/0056_add_info_and_distance.py +++ b/variants/migrations/0056_add_info_and_distance.py @@ -2,7 +2,6 @@ from django.conf import settings from django.db import migrations, models - if not settings.IS_TESTING: # Operations using raw SQL. operations = [ @@ -14,7 +13,9 @@ # Operations using the Django ORM. operations = [ migrations.AddField( - model_name="smallvariant", name="info", field=models.JSONField(null=True), + model_name="smallvariant", + name="info", + field=models.JSONField(null=True), ), migrations.AddField( model_name="smallvariant", name="refseq_exon_dist", field=models.IntegerField(null=True) diff --git a/variants/migrations/0068_kioskannotatebgjob.py b/variants/migrations/0068_kioskannotatebgjob.py index 08d3e152a..cc550bdf5 100644 --- a/variants/migrations/0068_kioskannotatebgjob.py +++ b/variants/migrations/0068_kioskannotatebgjob.py @@ -2,9 +2,11 @@ # Generated by Django 1.11.25 on 2019-12-12 16:56 from __future__ import unicode_literals +import uuid + from django.db import migrations, models import django.db.models.deletion -import uuid + import variants.models diff --git a/variants/migrations/0069_set_logged_table.py b/variants/migrations/0069_set_logged_table.py index 183696c14..9780bdafc 100644 --- a/variants/migrations/0069_set_logged_table.py +++ b/variants/migrations/0069_set_logged_table.py @@ -2,8 +2,8 @@ """SET small variant table as LOGGED to improve stability.""" from __future__ import unicode_literals -from django.db import migrations from django.conf import settings +from django.db import migrations if not settings.IS_TESTING: operations = [migrations.RunSQL("ALTER TABLE variants_smallvariant SET LOGGED;")] + [ diff --git a/variants/migrations/0070_smallvariant_create_index_chromosome_no.py b/variants/migrations/0070_smallvariant_create_index_chromosome_no.py index f7cd79157..9583999db 100644 --- a/variants/migrations/0070_smallvariant_create_index_chromosome_no.py +++ b/variants/migrations/0070_smallvariant_create_index_chromosome_no.py @@ -2,9 +2,8 @@ """Setup ``variants_smallvariants`` table as partitioned. """ -from django.db import migrations from django.conf import settings - +from django.db import migrations operations = [] diff --git a/variants/migrations/0071_smallvariantflags_flag_molecular.py b/variants/migrations/0071_smallvariantflags_flag_molecular.py index 2247531e4..bacd5818c 100644 --- a/variants/migrations/0071_smallvariantflags_flag_molecular.py +++ b/variants/migrations/0071_smallvariantflags_flag_molecular.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals from django.db import migrations, models + from variants.models import SmallVariantComment, SmallVariantFlags diff --git a/variants/migrations/0072_deletecasebgjob.py b/variants/migrations/0072_deletecasebgjob.py index 08bbea003..babde9bb1 100644 --- a/variants/migrations/0072_deletecasebgjob.py +++ b/variants/migrations/0072_deletecasebgjob.py @@ -2,10 +2,11 @@ # Generated by Django 1.11.25 on 2020-02-18 10:35 from __future__ import unicode_literals +import uuid + import bgjobs.models from django.db import migrations, models import django.db.models.deletion -import uuid class Migration(migrations.Migration): diff --git a/variants/migrations/0073_clearexpiredexportedfilesbgjob_clearinactivevariantsetsbgjob_clearoldkioskcasesbgjob_refreshsmallvar.py b/variants/migrations/0073_clearexpiredexportedfilesbgjob_clearinactivevariantsetsbgjob_clearoldkioskcasesbgjob_refreshsmallvar.py index 91953a299..37fbf8a3f 100644 --- a/variants/migrations/0073_clearexpiredexportedfilesbgjob_clearinactivevariantsetsbgjob_clearoldkioskcasesbgjob_refreshsmallvar.py +++ b/variants/migrations/0073_clearexpiredexportedfilesbgjob_clearinactivevariantsetsbgjob_clearoldkioskcasesbgjob_refreshsmallvar.py @@ -2,10 +2,11 @@ # Generated by Django 1.11.29 on 2020-05-26 17:28 from __future__ import unicode_literals +import uuid + import bgjobs.models from django.db import migrations, models import django.db.models.deletion -import uuid class Migration(migrations.Migration): @@ -43,7 +44,10 @@ class Migration(migrations.Migration): ), ), ], - options={"ordering": ("-date_created",), "abstract": False,}, + options={ + "ordering": ("-date_created",), + "abstract": False, + }, bases=(bgjobs.models.JobModelMessageMixin, models.Model), ), migrations.CreateModel( @@ -73,7 +77,10 @@ class Migration(migrations.Migration): ), ), ], - options={"ordering": ("-date_created",), "abstract": False,}, + options={ + "ordering": ("-date_created",), + "abstract": False, + }, bases=(bgjobs.models.JobModelMessageMixin, models.Model), ), migrations.CreateModel( @@ -103,7 +110,10 @@ class Migration(migrations.Migration): ), ), ], - options={"ordering": ("-date_created",), "abstract": False,}, + options={ + "ordering": ("-date_created",), + "abstract": False, + }, bases=(bgjobs.models.JobModelMessageMixin, models.Model), ), migrations.CreateModel( @@ -133,7 +143,10 @@ class Migration(migrations.Migration): ), ), ], - options={"ordering": ("-date_created",), "abstract": False,}, + options={ + "ordering": ("-date_created",), + "abstract": False, + }, bases=(bgjobs.models.JobModelMessageMixin, models.Model), ), ] diff --git a/variants/migrations/0074_caddsubmissionbgjob.py b/variants/migrations/0074_caddsubmissionbgjob.py index c4ca31027..5ad82d032 100644 --- a/variants/migrations/0074_caddsubmissionbgjob.py +++ b/variants/migrations/0074_caddsubmissionbgjob.py @@ -2,10 +2,11 @@ # Generated by Django 1.11.29 on 2020-05-29 14:04 from __future__ import unicode_literals +import uuid + import bgjobs.models from django.db import migrations, models import django.db.models.deletion -import uuid class Migration(migrations.Migration): @@ -33,7 +34,10 @@ class Migration(migrations.Migration): "sodar_uuid", models.UUIDField(default=uuid.uuid4, help_text="Case SODAR UUID", unique=True), ), - ("query_args", models.JSONField(help_text="(Validated) query parameters"),), + ( + "query_args", + models.JSONField(help_text="(Validated) query parameters"), + ), ( "cadd_version", models.CharField( diff --git a/variants/migrations/0079_auto_20210204_1006.py b/variants/migrations/0079_auto_20210204_1006.py index 917a23f86..4dacea15a 100644 --- a/variants/migrations/0079_auto_20210204_1006.py +++ b/variants/migrations/0079_auto_20210204_1006.py @@ -34,9 +34,14 @@ class Migration(migrations.Migration): ), ), ], - options={"ordering": ("individual",),}, + options={ + "ordering": ("individual",), + }, + ), + migrations.AlterModelOptions( + name="case", + options={"ordering": ("-date_modified",)}, ), - migrations.AlterModelOptions(name="case", options={"ordering": ("-date_modified",)},), migrations.AddField( model_name="casephenotypeterms", name="case", @@ -48,6 +53,7 @@ class Migration(migrations.Migration): ), ), migrations.AlterUniqueTogether( - name="casephenotypeterms", unique_together=set([("case", "individual")]), + name="casephenotypeterms", + unique_together=set([("case", "individual")]), ), ] diff --git a/variants/migrations/0080_spanrsubmissionbgjob.py b/variants/migrations/0080_spanrsubmissionbgjob.py index 5f38a025d..0278a34fb 100644 --- a/variants/migrations/0080_spanrsubmissionbgjob.py +++ b/variants/migrations/0080_spanrsubmissionbgjob.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Generated by Django 1.11.29 on 2021-04-01 08:45 from __future__ import unicode_literals + import uuid import bgjobs.models @@ -30,7 +31,10 @@ class Migration(migrations.Migration): "sodar_uuid", models.UUIDField(default=uuid.uuid4, help_text="Case SODAR UUID", unique=True), ), - ("query_args", models.JSONField(help_text="(Validated) query parameters"),), + ( + "query_args", + models.JSONField(help_text="(Validated) query parameters"), + ), ( "spanr_job_url", models.CharField(help_text="The SPANR job URL", max_length=100, null=True), diff --git a/variants/migrations/0081_set_logged_table_again.py b/variants/migrations/0081_set_logged_table_again.py index 726dcac2d..ca4744030 100644 --- a/variants/migrations/0081_set_logged_table_again.py +++ b/variants/migrations/0081_set_logged_table_again.py @@ -2,8 +2,8 @@ """SET small variant table as LOGGED to improve stability (#2).""" from __future__ import unicode_literals -from django.db import migrations from django.conf import settings +from django.db import migrations if not settings.IS_TESTING: operations = [migrations.RunSQL("ALTER TABLE variants_smallvariant SET LOGGED;")] + [ diff --git a/variants/migrations/0082_auto_20210617_1409.py b/variants/migrations/0082_auto_20210617_1409.py index 47aac82bf..1af5b0ffa 100644 --- a/variants/migrations/0082_auto_20210617_1409.py +++ b/variants/migrations/0082_auto_20210617_1409.py @@ -12,7 +12,9 @@ class Migration(migrations.Migration): operations = [ migrations.AlterField( - model_name="casealignmentstats", name="bam_stats", field=models.JSONField(default=dict), + model_name="casealignmentstats", + name="bam_stats", + field=models.JSONField(default=dict), ), migrations.AlterField( model_name="importvariantsbgjob", diff --git a/variants/migrations/0083_auto_20211129_1443.py b/variants/migrations/0083_auto_20211129_1443.py index 4cf646901..877b2c86f 100644 --- a/variants/migrations/0083_auto_20211129_1443.py +++ b/variants/migrations/0083_auto_20211129_1443.py @@ -1,6 +1,7 @@ # Generated by Django 3.2.9 on 2021-11-29 14:43 from django.db import migrations + import varfish.utils @@ -12,14 +13,20 @@ class Migration(migrations.Migration): operations = [ migrations.AlterField( - model_name="caddpathogenicityscorecache", name="info", field=varfish.utils.JSONField(), + model_name="caddpathogenicityscorecache", + name="info", + field=varfish.utils.JSONField(), ), migrations.AlterField( model_name="caddsubmissionbgjob", name="query_args", field=varfish.utils.JSONField(help_text="(Validated) query parameters"), ), - migrations.AlterField(model_name="case", name="pedigree", field=varfish.utils.JSONField(),), + migrations.AlterField( + model_name="case", + name="pedigree", + field=varfish.utils.JSONField(), + ), migrations.AlterField( model_name="casealignmentstats", name="bam_stats", diff --git a/variants/migrations/0085_add_variant_index.py b/variants/migrations/0085_add_variant_index.py index d8a423563..b43af37e0 100644 --- a/variants/migrations/0085_add_variant_index.py +++ b/variants/migrations/0085_add_variant_index.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.12 on 2022-03-19 08:42 -from django.db import migrations from django.conf import settings +from django.db import migrations operations = [] diff --git a/variants/migrations/0086_smallvariantsummary_excludefrominhousedb.py b/variants/migrations/0086_smallvariantsummary_excludefrominhousedb.py index d5c817365..b9badf905 100644 --- a/variants/migrations/0086_smallvariantsummary_excludefrominhousedb.py +++ b/variants/migrations/0086_smallvariantsummary_excludefrominhousedb.py @@ -5,7 +5,7 @@ """ from django.conf import settings -from django.db import migrations, models +from django.db import migrations SQL_OUTER = r""" DROP MATERIALIZED VIEW IF EXISTS variants_smallvariantsummary; @@ -104,7 +104,12 @@ if settings.IS_TESTING: operations = [] else: - operations = [migrations.RunSQL(SQL_OUTER % SQL_INNER_FORWARD, SQL_OUTER % SQL_INNER_REVERSE,)] + operations = [ + migrations.RunSQL( + SQL_OUTER % SQL_INNER_FORWARD, + SQL_OUTER % SQL_INNER_REVERSE, + ) + ] class Migration(migrations.Migration): diff --git a/variants/models.py b/variants/models.py index f2697829a..76c2af635 100644 --- a/variants/models.py +++ b/variants/models.py @@ -1,64 +1,57 @@ +from collections import defaultdict import contextlib +from datetime import datetime, timedelta import itertools +from itertools import chain +import json +import math import os +import re import shlex import shutil import subprocess import tempfile import time -from datetime import datetime, timedelta -import json -from collections import defaultdict - -import binning -import wrapt -from itertools import chain -import math -import re -import requests -from django.utils.timezone import localtime +import uuid as uuid_object -from varfish.utils import JSONField -from variants.helpers import get_engine +from bgjobs.models import ( + LOG_LEVEL_CHOICES, + LOG_LEVEL_ERROR, + LOG_LEVEL_INFO, + BackgroundJob, + JobModelMessageMixin, +) from bgjobs.plugins import BackgroundJobsPluginPoint +import binning +from django.conf import settings from django.contrib.auth import get_user_model -from django.forms import model_to_dict -from django.utils.html import strip_tags -from sqlalchemy import select, func, and_, delete -import uuid as uuid_object - -from postgres_copy import CopyManager - -from django.db import models, transaction, connection, utils -from django.db.models import Q from django.contrib.postgres.fields import ArrayField from django.contrib.postgres.indexes import GinIndex from django.core.exceptions import ValidationError -from django.urls import reverse -from django.dispatch import receiver -from django.conf import settings +from django.db import connection, models, transaction, utils +from django.db.models import Q from django.db.models.signals import pre_delete +from django.dispatch import receiver +from django.forms import model_to_dict +from django.urls import reverse from django.utils import timezone - +from django.utils.html import strip_tags +from django.utils.timezone import localtime +from postgres_copy import CopyManager +from projectroles.app_settings import AppSettingAPI from projectroles.models import Project -from bgjobs.models import ( - BackgroundJob, - JobModelMessageMixin, - LOG_LEVEL_CHOICES, - LOG_LEVEL_ERROR, - LOG_LEVEL_INFO, -) from projectroles.plugins import get_backend_api +import requests +from sqlalchemy import and_, delete, func, select +import wrapt -from geneinfo.models import Hgnc, EnsemblToGeneSymbol - +from geneinfo.models import EnsemblToGeneSymbol, Hgnc from genomicfeatures.models import GeneInterval #: The SQL Alchemy engine to use from importer.management.helpers import open_file, tsv_reader - -from variants.helpers import get_meta -from projectroles.app_settings import AppSettingAPI +from varfish.utils import JSONField +from variants.helpers import get_engine, get_meta app_settings = AppSettingAPI() @@ -262,7 +255,7 @@ def sample_variant_stats(self): class SmallVariant(models.Model): - """"Information of a single variant, knows its case.""" + """ "Information of a single variant, knows its case.""" #: Genome build release = models.CharField(max_length=32) @@ -711,8 +704,7 @@ def get_members_with_samples(self): return sorted([x["patient"] for x in self.get_filtered_pedigree_with_samples()]) def get_trio_roles(self): - """Returns a dict with keys mapping ``index``, ``mother``, ``father`` to pedigree member names if present. - """ + """Returns a dict with keys mapping ``index``, ``mother``, ``father`` to pedigree member names if present.""" result = {"index": self.index} for member in self.pedigree: if member["patient"] == self.index: @@ -1046,8 +1038,7 @@ def cleanup_variant_sets(min_age_hours=12): class AnnotationReleaseInfo(models.Model): - """Model to track the database releases used during annotation of a case. - """ + """Model to track the database releases used during annotation of a case.""" #: Release of genomebuild genomebuild = models.CharField(max_length=32, default="GRCh37") @@ -1058,9 +1049,15 @@ class AnnotationReleaseInfo(models.Model): #: Data release release = models.CharField(max_length=512) #: Link to case - case = models.ForeignKey(Case, on_delete=models.CASCADE,) + case = models.ForeignKey( + Case, + on_delete=models.CASCADE, + ) #: Link to variant set - variant_set = models.ForeignKey(SmallVariantSet, on_delete=models.CASCADE,) + variant_set = models.ForeignKey( + SmallVariantSet, + on_delete=models.CASCADE, + ) class Meta: unique_together = ("genomebuild", "table", "variant_set") @@ -1305,7 +1302,9 @@ class CaddSubmissionBgJob(JobModelMessageMixin, models.Model): ) cadd_job_id = models.CharField( - max_length=100, null=True, help_text="The project ID that CADD assigned on submission", + max_length=100, + null=True, + help_text="The project ID that CADD assigned on submission", ) def get_human_readable_type(self): @@ -1745,8 +1744,7 @@ def query_type(self): class SmallVariantQuery(SmallVariantQueryBase): - """Allow saving of single-case queries to the ``SmallVariant`` model. - """ + """Allow saving of single-case queries to the ``SmallVariant`` model.""" # TODO: rename to reflect single-case @@ -1767,9 +1765,7 @@ def get_project(self): class ProjectCasesSmallVariantQuery(SmallVariantQueryBase): - """Allow saving of whole-project queries to the ``SmallVariant`` model. - - """ + """Allow saving of whole-project queries to the ``SmallVariant`` model.""" #: The related case. project = models.ForeignKey( @@ -2333,9 +2329,7 @@ def annotate_with_phenotype_scores(rows, gene_scores): def annotate_with_transcripts(rows, database): - """Annotate the results in ``rows`` with transcripts (RefSeq or Ensembl) - - """ + """Annotate the results in ``rows`` with transcripts (RefSeq or Ensembl)""" rows = [RowWithTranscripts(row, database) for row in rows] for row in rows: transcripts = load_molecular_impact(row) @@ -2505,9 +2499,10 @@ def annotate_with_joint_scores(rows): # Get list of rows and assign joint scores. rows = [RowWithJointScore(row) for row in rows] for row in rows: - key = "-".join( - map(str, [row["chromosome"], row["start"], row["reference"], row["alternative"]]) - ) + # TODO: cleanup + # key = "-".join( + # map(str, [row["chromosome"], row["start"], row["reference"], row["alternative"]]) + # ) row._self_joint_score = (row.phenotype_score or 0) * (row.pathogenicity_score or 0) # Get highest score for each gene. gene_scores = {} @@ -2542,8 +2537,7 @@ def gene_score(row): def unroll_extra_annos_result(rows, fields): - """unroll the extra annotation results in columns in such a way that all writer can operate on extra annotations. - """ + """unroll the extra annotation results in columns in such a way that all writer can operate on extra annotations.""" # Get list of rows with extra annotations rows_ = [RowWithExtraAnno(row, fields, getattr(row, "extra_annos")) for row in rows] @@ -3634,7 +3628,10 @@ def run(self): case, case_created = Case.objects.get_or_create( name=self.import_job.case_name, project=self.import_job.project, - defaults={"index": self.import_job.index_name, "pedigree": pedigree,}, + defaults={ + "index": self.import_job.index_name, + "pedigree": pedigree, + }, ) # Create new variant set for case. variant_set = getattr(case, self.variant_set_attribute).create(state="importing") @@ -4067,8 +4064,8 @@ def update_variant_counts(case, kind=None, logger=lambda _: None): This is done without changing the ``date_modified`` field. """ - from svs import models as sv_models # noqa from importer.models import CaseVariantType # noqa + from svs import models as sv_models # noqa if not kind or kind == CaseVariantType.SMALL.name: logger("Updating variant counts for small variants ...") @@ -4155,7 +4152,10 @@ class ClearExpiredExportedFilesBgJob(SiteBgJobBase): spec_name = "variants.clear_expired_exported_files_bg_job" def get_absolute_url(self): - return reverse("variants:clear-expired-job-detail", kwargs={"job": self.sodar_uuid},) + return reverse( + "variants:clear-expired-job-detail", + kwargs={"job": self.sodar_uuid}, + ) class ClearInactiveVariantSetsBgJob(SiteBgJobBase): @@ -4168,7 +4168,10 @@ class ClearInactiveVariantSetsBgJob(SiteBgJobBase): spec_name = "variants.clear_inactive_variant_sets_bg_job" def get_absolute_url(self): - return reverse("variants:clear-inactive-variant-set-job", kwargs={"job": self.sodar_uuid},) + return reverse( + "variants:clear-inactive-variant-set-job", + kwargs={"job": self.sodar_uuid}, + ) class ClearOldKioskCasesBgJob(SiteBgJobBase): @@ -4182,7 +4185,8 @@ class ClearOldKioskCasesBgJob(SiteBgJobBase): def get_absolute_url(self): return reverse( - "variants:clear-old-kiosk-cases-job-detail", kwargs={"job": self.sodar_uuid}, + "variants:clear-old-kiosk-cases-job-detail", + kwargs={"job": self.sodar_uuid}, ) @@ -4197,5 +4201,6 @@ class RefreshSmallVariantSummaryBgJob(SiteBgJobBase): def get_absolute_url(self): return reverse( - "variants:refresh-small-variant-summaries-job-detail", kwargs={"job": self.sodar_uuid}, + "variants:refresh-small-variant-summaries-job-detail", + kwargs={"job": self.sodar_uuid}, ) diff --git a/variants/plugins.py b/variants/plugins.py index 0d294b71e..c66c74a97 100644 --- a/variants/plugins.py +++ b/variants/plugins.py @@ -1,32 +1,31 @@ +from bgjobs.plugins import BackgroundJobsPluginPoint from django.urls import reverse - from projectroles.constants import get_sodar_constants from projectroles.plugins import ProjectAppPluginPoint -from bgjobs.plugins import BackgroundJobsPluginPoint from .models import ( - Case, CASE_STATUS_CHOICES, - SmallVariantComment, - SmallVariantFlags, - ExportFileBgJob, - ExportProjectCasesFileBgJob, CaddSubmissionBgJob, - SpanrSubmissionBgJob, - DistillerSubmissionBgJob, - ComputeProjectVariantsStatsBgJob, - FilterBgJob, - ProjectCasesFilterBgJob, - SyncCaseListBgJob, - ImportVariantsBgJob, + Case, CaseComments, ClearExpiredExportedFilesBgJob, ClearInactiveVariantSetsBgJob, ClearOldKioskCasesBgJob, + ComputeProjectVariantsStatsBgJob, + DistillerSubmissionBgJob, + ExportFileBgJob, + ExportProjectCasesFileBgJob, + FilterBgJob, + ImportVariantsBgJob, + ProjectCasesFilterBgJob, RefreshSmallVariantSummaryBgJob, + SmallVariantComment, + SmallVariantFlags, + SpanrSubmissionBgJob, + SyncCaseListBgJob, ) -from .urls import urlpatterns from .templatetags.variants_tags import case_status_to_color +from .urls import urlpatterns # Global SODAR constants SODAR_CONSTANTS = get_sodar_constants() diff --git a/variants/queries.py b/variants/queries.py index 2551da54c..e4f1cf8b7 100644 --- a/variants/queries.py +++ b/variants/queries.py @@ -1,17 +1,18 @@ -import contextlib +# Note that we are using a lot of ``# noqa: E711`` here as for SQLAlchemy queries we need to test for NULL +# with ``COLUMN == None`` and for True-ness with ``COLUMN == True`` etc. + from itertools import chain import typing import attr - -from sqlalchemy.dialects.postgresql.array import OVERLAP -from sqlalchemy.sql.functions import GenericFunction, ReturnTypeFromArgs from django.conf import settings from django.core.exceptions import ImproperlyConfigured -from sqlalchemy import Table, true, column, union, literal_column, delete, tuple_ +from sqlalchemy import Table, column, delete, literal_column, true, tuple_, union from sqlalchemy.dialects.postgresql import ARRAY -from sqlalchemy.sql import select, func, and_, not_, or_, cast -from sqlalchemy.types import VARCHAR, Integer, Float, String +from sqlalchemy.dialects.postgresql.array import OVERLAP +from sqlalchemy.sql import and_, cast, func, not_, or_, select +from sqlalchemy.sql.functions import GenericFunction, ReturnTypeFromArgs +from sqlalchemy.types import VARCHAR, Float, Integer, String import sqlparse from clinvar.models import Clinvar @@ -19,33 +20,33 @@ from conservation.models import KnowngeneAA from dbsnp.models import Dbsnp from extra_annos.models import ExtraAnno -from frequencies.models import MtDb, HelixMtDb, Mitomap +from frequencies.models import HelixMtDb, Mitomap, MtDb from geneinfo.models import ( - Hgnc, - RefseqToHgnc, Acmg, + EnsemblToGeneSymbol, + ExacConstraints, + GeneIdInHpo, GeneIdToInheritance, GnomadConstraints, - ExacConstraints, + Hgnc, MgiMapping, - RefseqToGeneSymbol, RefseqToEnsembl, - EnsemblToGeneSymbol, - GeneIdInHpo, + RefseqToGeneSymbol, + RefseqToHgnc, ) from hgmd.models import HgmdPublicLocus from svs.models import StructuralVariant, StructuralVariantGeneAnnotation +from variants.forms import FILTER_FORM_TRANSLATE_INHERITANCE +from variants.helpers import get_meta from variants.models import ( + AcmgCriteriaRating, Case, SmallVariant, - SmallVariantSummary, - SmallVariantFlags, SmallVariantComment, - AcmgCriteriaRating, + SmallVariantFlags, SmallVariantSet, + SmallVariantSummary, ) -from variants.forms import FILTER_FORM_TRANSLATE_INHERITANCE -from variants.helpers import get_meta class _ArrayCatAgg(ReturnTypeFromArgs): @@ -196,7 +197,7 @@ class ExtendQueryPartsDbsnpJoinAndFilter(ExtendQueryPartsDbsnpJoin): def extend_conditions(self, _query_parts): # Do not enable option if clinvar filter is activated as all clinvar variants have a dbsnp entry. if self.kwargs["remove_if_in_dbsnp"] and not self.kwargs["require_in_clinvar"]: - return [column("rsid") == None] + return [column("rsid") == None] # noqa: E711 return [] @@ -409,11 +410,16 @@ def __init__(self, *args, **kwargs): ) def extend_conditions(self, _query_parts): - return [and_(self._build_membership_term(), self._build_significance_term(),)] + return [ + and_( + self._build_membership_term(), + self._build_significance_term(), + ) + ] def _build_membership_term(self): if self.kwargs["require_in_clinvar"]: - return SmallVariant.sa.in_clinvar == True + return SmallVariant.sa.in_clinvar == True # noqa: E712 else: return True @@ -979,9 +985,9 @@ def extend_conditions(self, _query_parts): field = getattr(SmallVariant.sa, "%s_transcript_coding" % self.kwargs["database_select"]) terms = [] if not self.kwargs["transcripts_coding"]: - terms.append(field == False) # equality from SQL Alchemy + terms.append(field == False) # noqa: E712 if not self.kwargs["transcripts_noncoding"]: - terms.append(field == True) # equality from SQL Alchemy + terms.append(field == True) # noqa: E712 return [and_(*terms)] @@ -1014,7 +1020,7 @@ def _build_gene_sub_query(self, hgnc_field, gene_list): Hgnc.sa.entrez_id.in_(gene_list), Hgnc.sa.symbol.in_(gene_list), ), - getattr(Hgnc.sa, hgnc_field) != None, # SQL Alchemy forces us to use ``!=`` + getattr(Hgnc.sa, hgnc_field) != None, # noqa: E711 ) ) .distinct() @@ -1830,10 +1836,6 @@ class CaseExportVcfQuery(CasePrefetchQuery): builder = CaseExportVcfQueryPartsBuilder -class CaseLoadPrefetchedQuery(CasePrefetchQuery): - builder = CaseLoadPrefetchedQueryPartsBuilder - - class CaseSecondHitsQuery(CasePrefetchQuery): builder = QueryPartsBuilder diff --git a/variants/query_presets.py b/variants/query_presets.py index 2d38ef05d..bfa581a38 100644 --- a/variants/query_presets.py +++ b/variants/query_presets.py @@ -8,7 +8,7 @@ ``CATEGORY_PRESETS`` constant that defines the presets """ -from enum import unique, Enum +from enum import Enum, unique import itertools import typing @@ -117,83 +117,9 @@ def to_settings( ) if self._is_recessive(): - # Get index, parents, and others (not index, not parents) individuals - recessive_index = index_candidates[0] - parents = [ - s for s in samples if s.name in (recessive_index.father, recessive_index.mother) - ] - parent_names = {p.name for p in parents} - others = {s for s in affected_samples if s.name != index and s.name not in parent_names} - # Fill ``genotype`` for the index - if self == Inheritance.HOMOZYGOUS_RECESSIVE: - genotype = {recessive_index.name: GenotypeChoice.HOM.value} - mode = {"recessive_mode": None} - elif self == Inheritance.COMPOUND_HETEROZYGOUS: - genotype = {recessive_index.name: None} - mode = {"recessive_mode": "compound-recessive"} - elif self == Inheritance.RECESSIVE: - genotype = {recessive_index.name: None} - mode = {"recessive_mode": "recessive"} - else: - raise RuntimeError(f"Unexpected recessive mode of inheritance: {self}") - # Fill ``genotype`` for parents and others - if self == Inheritance.HOMOZYGOUS_RECESSIVE: - affected_parents = len([p for p in parents if p.is_affected()]) - for parent in parents: - if parent.is_affected(): - genotype[parent.name] = GenotypeChoice.HOM.value - else: - if affected_parents > 0: - genotype[parent.name] = GenotypeChoice.REF.value - else: - genotype[parent.name] = GenotypeChoice.HET.value - else: - for parent in parents: - genotype[parent.name] = None - for other in others: - genotype[other.name] = GenotypeChoice.ANY.value - # Compose the dict with recessive index, recessive mode, and genotype - return { - "recessive_index": recessive_index.name, - "genotype": {k: e for k, e in genotype.items()}, - **mode, - } + return self._to_settings_recessive(affected_samples, index, index_candidates, samples) elif self == Inheritance.X_RECESSIVE: - male_index_candidates = list( - itertools.chain( - [s for s in index_candidates if s.sex == Sex.MALE], - [s for s in index_candidates if s.sex == Sex.UNKNOWN], - index_candidates, - samples, - ) - ) - recessive_index = male_index_candidates[0] - index_father = samples_by_name.get(recessive_index.father, None) - index_mother = samples_by_name.get(recessive_index.mother, None) - others = [ - s - for s in samples - if s.name - not in (recessive_index.name, recessive_index.father, recessive_index.mother) - ] - genotype = {recessive_index.name: GenotypeChoice.HOM} - if index_father and index_father.is_affected(): - genotype[recessive_index.father] = GenotypeChoice.HOM - if index_mother: - genotype[recessive_index.mother] = GenotypeChoice.REF - elif index_father and not index_father.is_affected(): - genotype[recessive_index.father] = GenotypeChoice.REF - if index_mother: - genotype[recessive_index.mother] = GenotypeChoice.HET - elif index_mother: # and not index_father - genotype[recessive_index.mother] = GenotypeChoice.ANY - for other in others: - genotype[other.name] = GenotypeChoice.ANY - return { - "recessive_index": recessive_index.name, - "recessive_mode": None, - "genotype": {k: e.value for k, e in genotype.items()}, - } + return self._to_settings_x_recessive(index_candidates, samples, samples_by_name) elif self == Inheritance.AFFECTED_CARRIERS: return { "recessive_index": None, @@ -236,6 +162,85 @@ def to_settings( else: raise ValueError(f"Cannot generate settings for inheritance {self}") + def _to_settings_x_recessive(self, index_candidates, samples, samples_by_name): + male_index_candidates = list( + itertools.chain( + [s for s in index_candidates if s.sex == Sex.MALE], + [s for s in index_candidates if s.sex == Sex.UNKNOWN], + index_candidates, + samples, + ) + ) + recessive_index = male_index_candidates[0] + index_father = samples_by_name.get(recessive_index.father, None) + index_mother = samples_by_name.get(recessive_index.mother, None) + others = [ + s + for s in samples + if s.name not in (recessive_index.name, recessive_index.father, recessive_index.mother) + ] + genotype = {recessive_index.name: GenotypeChoice.HOM} + if index_father and index_father.is_affected(): + genotype[recessive_index.father] = GenotypeChoice.HOM + if index_mother: + genotype[recessive_index.mother] = GenotypeChoice.REF + elif index_father and not index_father.is_affected(): + genotype[recessive_index.father] = GenotypeChoice.REF + if index_mother: + genotype[recessive_index.mother] = GenotypeChoice.HET + elif index_mother: # and not index_father + genotype[recessive_index.mother] = GenotypeChoice.ANY + for other in others: + genotype[other.name] = GenotypeChoice.ANY + result = { + "recessive_index": recessive_index.name, + "recessive_mode": None, + "genotype": {k: e.value for k, e in genotype.items()}, + } + return result + + def _to_settings_recessive(self, affected_samples, index, index_candidates, samples): + # Get index, parents, and others (not index, not parents) individuals + recessive_index = index_candidates[0] + parents = [s for s in samples if s.name in (recessive_index.father, recessive_index.mother)] + parent_names = {p.name for p in parents} + others = {s for s in affected_samples if s.name != index and s.name not in parent_names} + # Fill ``genotype`` for the index + if self == Inheritance.HOMOZYGOUS_RECESSIVE: + genotype = {recessive_index.name: GenotypeChoice.HOM.value} + mode = {"recessive_mode": None} + elif self == Inheritance.COMPOUND_HETEROZYGOUS: + genotype = {recessive_index.name: None} + mode = {"recessive_mode": "compound-recessive"} + elif self == Inheritance.RECESSIVE: + genotype = {recessive_index.name: None} + mode = {"recessive_mode": "recessive"} + else: + raise RuntimeError(f"Unexpected recessive mode of inheritance: {self}") + # Fill ``genotype`` for parents and others + if self == Inheritance.HOMOZYGOUS_RECESSIVE: + affected_parents = len([p for p in parents if p.is_affected()]) + for parent in parents: + if parent.is_affected(): + genotype[parent.name] = GenotypeChoice.HOM.value + else: + if affected_parents > 0: + genotype[parent.name] = GenotypeChoice.REF.value + else: + genotype[parent.name] = GenotypeChoice.HET.value + else: + for parent in parents: + genotype[parent.name] = None + for other in others: + genotype[other.name] = GenotypeChoice.ANY.value + # Compose the dict with recessive index, recessive mode, and genotype + result = { + "recessive_index": recessive_index.name, + "genotype": {k: e for k, e in genotype.items()}, + **mode, + } + return result + @attrs.frozen class _FrequencyPresets: @@ -795,19 +800,25 @@ class _ChromosomePresets: } #: Presets for the "X-chromosome" chromosome/region/gene settings x_chromosome: typing.Dict[str, typing.Any] = { - "genomic_region": ["X",], + "genomic_region": [ + "X", + ], "gene_allowlist": [], "gene_blocklist": [], } #: Presets for the "Y-chromosomes" chromosome/region/gene settings y_chromosome: typing.Dict[str, typing.Any] = { - "genomic_region": ["Y",], + "genomic_region": [ + "Y", + ], "gene_allowlist": [], "gene_blocklist": [], } #: Presets for the "mitochondrial" chromosome/region/gene settings mt_chromosome: typing.Dict[str, typing.Any] = { - "genomic_region": ["MT",], + "genomic_region": [ + "MT", + ], "gene_allowlist": [], "gene_blocklist": [], } diff --git a/variants/query_schemas.py b/variants/query_schemas.py index 56ba181eb..542d72c2a 100644 --- a/variants/query_schemas.py +++ b/variants/query_schemas.py @@ -1,9 +1,9 @@ """Utility code for query schemas.""" -from enum import unique, Enum import copy -import os.path +from enum import Enum, unique import json +import os.path import re import typing @@ -12,8 +12,8 @@ import cattr from jsonschema import Draft7Validator, validators -from variants.models import Case from variants.forms import FILTER_FORM_TRANSLATE_EFFECTS +from variants.models import Case def extend_with_default(validator_class): @@ -25,10 +25,18 @@ def set_defaults(validator, properties, instance, schema): if "default" in sub_schema: instance.setdefault(key, sub_schema["default"]) - for error in validate_properties(validator, properties, instance, schema,): + for error in validate_properties( + validator, + properties, + instance, + schema, + ): yield error - return validators.extend(validator_class, {"properties": set_defaults},) + return validators.extend( + validator_class, + {"properties": set_defaults}, + ) def load_json(path): diff --git a/variants/rules.py b/variants/rules.py index 2610fdeda..c3dd17a30 100644 --- a/variants/rules.py +++ b/variants/rules.py @@ -1,6 +1,5 @@ -import rules from projectroles import rules as pr_rules - +import rules rules.add_perm( "variants.view_data", diff --git a/variants/serializers.py b/variants/serializers.py index 5a6ede25a..e2f576193 100644 --- a/variants/serializers.py +++ b/variants/serializers.py @@ -8,18 +8,19 @@ from bgjobs.models import BackgroundJob from django.db import transaction from django.db.models import Q -from projectroles.serializers import SODARProjectModelSerializer, SODARModelSerializer +from projectroles.serializers import SODARModelSerializer, SODARProjectModelSerializer from rest_framework import serializers -from geneinfo.models import HpoName, Hgnc +from geneinfo.models import Hgnc, HpoName +from variants.tasks import single_case_filter_task + from .forms import FilterForm -from .models import Case, SmallVariantQuery, FilterBgJob, SmallVariant +from .models import Case, FilterBgJob, SmallVariant, SmallVariantQuery from .query_schemas import ( SCHEMA_QUERY_V1, DefaultValidatingDraft7Validator, convert_query_json_to_small_variant_filter_form_v1, ) -from variants.tasks import single_case_filter_task def create_only_validator(value, serializer_field): diff --git a/variants/submit_external.py b/variants/submit_external.py index b095f6925..f69e331c1 100644 --- a/variants/submit_external.py +++ b/variants/submit_external.py @@ -6,10 +6,8 @@ from projectroles.plugins import get_backend_api import requests - from .file_export import CaseExporterVcf - #: URL to MutationDistiller submission form DISTILLER_POST_URL = "https://www.mutationdistiller.org/QE/MT/MTQE_start.cgi" diff --git a/variants/submit_filter.py b/variants/submit_filter.py index 9f7d694c6..f62301eb2 100644 --- a/variants/submit_filter.py +++ b/variants/submit_filter.py @@ -2,19 +2,17 @@ from django.conf import settings from django.db import transaction - from projectroles.plugins import get_backend_api -from cohorts.models import Cohort -from variants.helpers import get_engine from variants.forms import PATHO_SCORES_MAPPING -from variants.models import prioritize_genes, VariantScoresFactory +from variants.helpers import get_engine +from variants.models import VariantScoresFactory, prioritize_genes + from .queries import CasePrefetchQuery, ProjectPrefetchQuery class FilterBase: - """Base class for filtering and storing case query results. - """ + """Base class for filtering and storing case query results.""" def __init__(self, job, variant_query): """Constructor""" @@ -128,8 +126,7 @@ def get_var(row): class CaseFilter(FilterBase): - """Class for storing query results for a single case. - """ + """Class for storing query results for a single case.""" def _get_genomebuild(self): return self.variant_query.case.release @@ -140,8 +137,7 @@ def _get_assembled_query(self): class ProjectCasesFilter(FilterBase): - """Class for storing query results for cases of a project. - """ + """Class for storing query results for cases of a project.""" def _get_genomebuild(self): if self.job.cohort: diff --git a/variants/sync_upstream.py b/variants/sync_upstream.py index c9e38ef9f..162f63c70 100644 --- a/variants/sync_upstream.py +++ b/variants/sync_upstream.py @@ -4,12 +4,12 @@ import re import typing -import attr from altamisa.isatab import StudyReader -from django.db import transaction +import attr from bgjobs.models import LOG_LEVEL_ERROR, LOG_LEVEL_WARNING -from projectroles.plugins import get_backend_api +from django.db import transaction from projectroles.models import RemoteSite +from projectroles.plugins import get_backend_api import requests from variants.models import CaseAwareProject, SyncCaseResultMessage diff --git a/variants/tasks.py b/variants/tasks.py index b22bd2af3..46b975716 100644 --- a/variants/tasks.py +++ b/variants/tasks.py @@ -1,12 +1,8 @@ -from config.celery import app from celery.schedules import crontab -from . import file_export -from . import models -from . import submit_external -from . import variant_stats -from . import submit_filter -from . import sync_upstream +from config.celery import app + +from . import file_export, models, submit_external, submit_filter, sync_upstream, variant_stats from .helpers import get_engine diff --git a/variants/templatetags/variants_tags.py b/variants/templatetags/variants_tags.py index 1fe20f636..7129eeb84 100644 --- a/variants/templatetags/variants_tags.py +++ b/variants/templatetags/variants_tags.py @@ -1,19 +1,17 @@ import sys from django import template +from django.conf import settings from django.utils import formats from django.utils.html import avoid_wrapping import nltk - -from django.conf import settings -from ..models import ( - Case, - only_source_name as _models_only_source_name, - _variant_scores_mutationtaster_rank_model, -) from projectroles.app_settings import AppSettingAPI + from geneinfo.models import GeneIdToInheritance, Hpo, HpoName +from ..models import Case, _variant_scores_mutationtaster_rank_model +from ..models import only_source_name as _models_only_source_name + modes_of_inheritance = dict(GeneIdToInheritance.MODES_OF_INHERITANCE) register = template.Library() @@ -464,8 +462,8 @@ def mode_of_inheritance_description(mode_of_inheritance): @register.filter -def listsort(l): - return sorted(l) +def listsort(the_list): + return sorted(the_list) @register.filter diff --git a/variants/tests/_fixtures.py b/variants/tests/_fixtures.py index 97649568d..ab7b44725 100644 --- a/variants/tests/_fixtures.py +++ b/variants/tests/_fixtures.py @@ -3,12 +3,11 @@ import aldjemy.core import binning - from projectroles.models import Project + +from geneinfo.models import Hgnc, RefseqToHgnc from variants.models import SmallVariant, SmallVariantSummary from variants.variant_stats import rebuild_case_variant_stats -from geneinfo.models import Hgnc, RefseqToHgnc - #: Shared data for ``Project`` to use for all test cases. PROJECT_DICT = { diff --git a/variants/tests/factories.py b/variants/tests/factories.py index 4bde1f3a8..9f18fe8e3 100644 --- a/variants/tests/factories.py +++ b/variants/tests/factories.py @@ -1,44 +1,43 @@ """Factory Boy factory classes for ``variants``.""" -import uuid from datetime import datetime +import typing +import uuid +import attr +from bgjobs.tests.factories import BackgroundJobFactory import binning -import factory from django.utils import timezone - +import factory from projectroles.models import SODAR_CONSTANTS, RemoteSite -from bgjobs.tests.factories import BackgroundJobFactory - from config.settings.base import VARFISH_CADD_SUBMISSION_VERSION + from ..models import ( + AcmgCriteriaRating, + CaddSubmissionBgJob, Case, - SmallVariant, - SmallVariantQuery, - ProjectCasesSmallVariantQuery, - SmallVariantSummary, - FilterBgJob, - ProjectCasesFilterBgJob, CaseAwareProject, + CaseComments, + CasePhenotypeTerms, + CaseVariantStats, + DeleteCaseBgJob, DistillerSubmissionBgJob, ExportFileBgJob, ExportFileJobResult, ExportProjectCasesFileBgJob, ExportProjectCasesFileBgJobResult, - SmallVariantFlags, + FilterBgJob, + ProjectCasesFilterBgJob, + ProjectCasesSmallVariantQuery, + SampleVariantStatistics, + SmallVariant, SmallVariantComment, + SmallVariantFlags, + SmallVariantQuery, SmallVariantSet, - CaseVariantStats, - SampleVariantStatistics, - CaseComments, - DeleteCaseBgJob, - CaddSubmissionBgJob, - CasePhenotypeTerms, + SmallVariantSummary, SyncCaseListBgJob, - AcmgCriteriaRating, ) -import typing -import attr @attr.s(auto_attribs=True) diff --git a/variants/tests/helpers.py b/variants/tests/helpers.py index 6e4499dee..f3dcfa6e2 100644 --- a/variants/tests/helpers.py +++ b/variants/tests/helpers.py @@ -1,19 +1,18 @@ """Common helper code for tests""" +from django.conf import settings from django.test import RequestFactory -from test_plus.test import TestCase from projectroles.tests.test_permissions import TestProjectPermissionBase from projectroles.tests.test_permissions_api import TestProjectAPIPermissionBase -from django.conf import settings +from test_plus.test import TestCase from beaconsite.models import Site from beaconsite.tests.factories import ConsortiumFactory, SiteFactory - from cohorts.models import Cohort -from .factories import ProcessedFormDataFactory, ProjectFactory -from ..models import Case, CaseAwareProject from variants.helpers import get_engine +from ..models import Case, CaseAwareProject +from .factories import ProcessedFormDataFactory, ProjectFactory #: A known invalid MIME type. VARFISH_INVALID_MIMETYPE = "application/vnd.bihealth.invalid+json" diff --git a/variants/tests/test_file_export.py b/variants/tests/test_file_export.py index aaf68b68c..093080e64 100644 --- a/variants/tests/test_file_export.py +++ b/variants/tests/test_file_export.py @@ -1,32 +1,33 @@ """Tests for the ``file_export`` module.""" +from datetime import timedelta import gzip import io -from datetime import timedelta import tempfile from unittest.mock import patch +from bgjobs.models import BackgroundJob import django from django.utils import timezone import openpyxl +from projectroles.models import Project from requests_mock import Mocker from test_plus.test import TestCase from timeline.models import ProjectEvent from clinvar.tests.factories import ClinvarFactory from cohorts.tests.factories import TestCohortBase -from extra_annos.tests.factories import ExtraAnnoFieldFactory, ExtraAnnoFactory +from extra_annos.tests.factories import ExtraAnnoFactory, ExtraAnnoFieldFactory from geneinfo.tests.factories import GnomadConstraintsFactory, RefseqToEnsemblFactory from variants.tests.factories import ( - SmallVariantFactory, - ResubmitFormDataFactory, - ExportProjectCasesFileBgJobFactory, CaseWithVariantSetFactory, + ExportProjectCasesFileBgJobFactory, + ResubmitFormDataFactory, + SmallVariantFactory, ) + from .. import file_export -from ..models import ExportFileBgJob, ExportProjectCasesFileBgJob, Case, CaseAwareProject -from bgjobs.models import BackgroundJob -from projectroles.models import Project +from ..models import Case, CaseAwareProject, ExportFileBgJob, ExportProjectCasesFileBgJob class ExportTestBase(TestCase): @@ -384,7 +385,12 @@ def _test_tabular(self, arrs, has_trailing): ): member = list(small_var.genotype.keys())[0] self.assertSequenceEqual( - arrs[i + 1][:3], [member, "chr" + small_var.chromosome, str(small_var.start),], + arrs[i + 1][:3], + [ + member, + "chr" + small_var.chromosome, + str(small_var.start), + ], ) self.assertSequenceEqual( arrs[i + 1][-5:], @@ -546,7 +552,10 @@ def test_export_tsv_as_contributor(self): result = str(exporter.generate(), "utf-8") arrs = [line.split("\t") for line in result.split("\n")] self._test_tabular( - arrs, 13, True, self.project2_case1_smallvars + self.project2_case2_smallvars, + arrs, + 13, + True, + self.project2_case1_smallvars + self.project2_case2_smallvars, ) def test_export_tsv_as_superuser_for_cohort_by_contributor(self): @@ -558,7 +567,10 @@ def test_export_tsv_as_superuser_for_cohort_by_contributor(self): result = str(exporter.generate(), "utf-8") arrs = [line.split("\t") for line in result.split("\n")] self._test_tabular( - arrs, 13, True, self.project2_case1_smallvars + self.project2_case2_smallvars, + arrs, + 13, + True, + self.project2_case1_smallvars + self.project2_case2_smallvars, ) def test_export_tsv_as_contributor_for_cohort_by_superuser(self): @@ -570,7 +582,10 @@ def test_export_tsv_as_contributor_for_cohort_by_superuser(self): result = str(exporter.generate(), "utf-8") arrs = [line.split("\t") for line in result.split("\n")] self._test_tabular( - arrs, 13, True, self.project2_case1_smallvars + self.project2_case2_smallvars, + arrs, + 13, + True, + self.project2_case1_smallvars + self.project2_case2_smallvars, ) def _test_tabular(self, arrs, ref, has_trailing, smallvars): @@ -582,7 +597,12 @@ def _test_tabular(self, arrs, ref, has_trailing, smallvars): for i, small_var in enumerate(sorted(smallvars, key=lambda x: (x.chromosome_no, x.start))): member = Case.objects.get(id=small_var.case_id).get_members()[0] self.assertSequenceEqual( - arrs[i + 1][:3], [member, "chr" + small_var.chromosome, str(small_var.start),], + arrs[i + 1][:3], + [ + member, + "chr" + small_var.chromosome, + str(small_var.start), + ], ) self.assertSequenceEqual( arrs[i + 1][-5:], @@ -672,7 +692,11 @@ def test_export_vcf_as_contributor(self): with file_export.CaseExporterVcf(bgjob, cohort) as exporter: result = exporter.generate() self._test_vcf( - result, 13, self.project2_case1_smallvars + self.project2_case2_smallvars, cohort, user, + result, + 13, + self.project2_case1_smallvars + self.project2_case2_smallvars, + cohort, + user, ) def test_export_vcf_as_superuser_for_cohort_by_contributor(self): @@ -683,7 +707,11 @@ def test_export_vcf_as_superuser_for_cohort_by_contributor(self): with file_export.CaseExporterVcf(bgjob, cohort) as exporter: result = exporter.generate() self._test_vcf( - result, 13, self.project2_case1_smallvars + self.project2_case2_smallvars, cohort, user, + result, + 13, + self.project2_case1_smallvars + self.project2_case2_smallvars, + cohort, + user, ) def test_export_vcf_as_contributor_for_cohort_by_superuser(self): @@ -694,7 +722,11 @@ def test_export_vcf_as_contributor_for_cohort_by_superuser(self): with file_export.CaseExporterVcf(bgjob, cohort) as exporter: result = exporter.generate() self._test_vcf( - result, 13, self.project2_case1_smallvars + self.project2_case2_smallvars, cohort, user, + result, + 13, + self.project2_case1_smallvars + self.project2_case2_smallvars, + cohort, + user, ) def test_export_xlsx_as_superuser(self): @@ -738,7 +770,10 @@ def test_export_xlsx_as_contributor(self): variants_sheet = workbook["Variants"] arrs = [[cell.value for cell in row] for row in variants_sheet.rows] self._test_tabular( - arrs, 13, False, self.project2_case1_smallvars + self.project2_case2_smallvars, + arrs, + 13, + False, + self.project2_case1_smallvars + self.project2_case2_smallvars, ) def test_export_xlsx_as_superuser_for_cohort_by_contributor(self): @@ -757,7 +792,10 @@ def test_export_xlsx_as_superuser_for_cohort_by_contributor(self): variants_sheet = workbook["Variants"] arrs = [[cell.value for cell in row] for row in variants_sheet.rows] self._test_tabular( - arrs, 13, False, self.project2_case1_smallvars + self.project2_case2_smallvars, + arrs, + 13, + False, + self.project2_case1_smallvars + self.project2_case2_smallvars, ) def test_export_xlsx_as_contributor_for_cohort_by_superuser(self): @@ -776,7 +814,10 @@ def test_export_xlsx_as_contributor_for_cohort_by_superuser(self): variants_sheet = workbook["Variants"] arrs = [[cell.value for cell in row] for row in variants_sheet.rows] self._test_tabular( - arrs, 13, False, self.project2_case1_smallvars + self.project2_case2_smallvars, + arrs, + 13, + False, + self.project2_case1_smallvars + self.project2_case2_smallvars, ) diff --git a/variants/tests/test_forms.py b/variants/tests/test_forms.py index c387f8bee..0db0a1dc5 100644 --- a/variants/tests/test_forms.py +++ b/variants/tests/test_forms.py @@ -4,11 +4,12 @@ from geneinfo.tests.factories import HpoFactory, HpoNameFactory from variants.tests.factories import ( - FormDataFactory, - CaseWithVariantSetFactory, CasePhenotypeTermsFactory, + CaseWithVariantSetFactory, + FormDataFactory, ) -from ..forms import FilterForm, CaseTermsForm + +from ..forms import CaseTermsForm, FilterForm class TestFormBase(TestCase): @@ -27,10 +28,26 @@ def setUp(self): hpo_id="HP:0000003", name="Disease 1;;Alternative Description", ), - HpoFactory(database_id="DECIPHER:1", hpo_id="HP:0000003", name="Disease 2",), - HpoFactory(database_id="DECIPHER:1", hpo_id="HP:0000004", name="Disease 2",), - HpoFactory(database_id="ORPHA:1", hpo_id="HP:0000004", name="Disease 3",), - HpoFactory(database_id="ORPHA:1", hpo_id="HP:0000005", name="Disease 3",), + HpoFactory( + database_id="DECIPHER:1", + hpo_id="HP:0000003", + name="Disease 2", + ), + HpoFactory( + database_id="DECIPHER:1", + hpo_id="HP:0000004", + name="Disease 2", + ), + HpoFactory( + database_id="ORPHA:1", + hpo_id="HP:0000004", + name="Disease 3", + ), + HpoFactory( + database_id="ORPHA:1", + hpo_id="HP:0000005", + name="Disease 3", + ), ] self.maxDiff = None @@ -71,10 +88,12 @@ def test_hpo_terms_plain(self): form = FilterForm(form_data, case=self.variant_set.case, user=0) self.assertTrue(form.is_valid()) self.assertListEqual( - form.cleaned_data["prio_hpo_terms"], sorted([self.hponame.hpo_id]), + form.cleaned_data["prio_hpo_terms"], + sorted([self.hponame.hpo_id]), ) self.assertListEqual( - form.cleaned_data["prio_hpo_terms_curated"], sorted([self.hponame.hpo_id]), + form.cleaned_data["prio_hpo_terms_curated"], + sorted([self.hponame.hpo_id]), ) def test_hpo_terms_omim_to_hpo_conversion(self): @@ -87,7 +106,8 @@ def test_hpo_terms_omim_to_hpo_conversion(self): form = FilterForm(form_data, case=self.variant_set.case, user=0) self.assertTrue(form.is_valid()) self.assertListEqual( - form.cleaned_data["prio_hpo_terms"], sorted([self.hpos[0].database_id]), + form.cleaned_data["prio_hpo_terms"], + sorted([self.hpos[0].database_id]), ) self.assertListEqual( form.cleaned_data["prio_hpo_terms_curated"], @@ -104,7 +124,8 @@ def test_hpo_terms_decipher_to_hpo_conversion(self): form = FilterForm(form_data, case=self.variant_set.case, user=0) self.assertTrue(form.is_valid()) self.assertListEqual( - form.cleaned_data["prio_hpo_terms"], sorted([self.hpos[2].database_id]), + form.cleaned_data["prio_hpo_terms"], + sorted([self.hpos[2].database_id]), ) self.assertListEqual( form.cleaned_data["prio_hpo_terms_curated"], @@ -121,7 +142,8 @@ def test_hpo_terms_orpha_to_hpo_conversion(self): form = FilterForm(form_data, case=self.variant_set.case, user=0) self.assertTrue(form.is_valid()) self.assertListEqual( - form.cleaned_data["prio_hpo_terms"], sorted([self.hpos[4].database_id]), + form.cleaned_data["prio_hpo_terms"], + sorted([self.hpos[4].database_id]), ) self.assertListEqual( form.cleaned_data["prio_hpo_terms_curated"], diff --git a/variants/tests/test_import.py b/variants/tests/test_import.py index 45e4bbb91..bba8d5991 100644 --- a/variants/tests/test_import.py +++ b/variants/tests/test_import.py @@ -7,9 +7,9 @@ from bgjobs.models import BackgroundJob from test_plus.test import TestCase -from variants.models import ImportVariantsBgJob, Case, SmallVariantSet, SmallVariant +from variants.models import Case, ImportVariantsBgJob, SmallVariant, SmallVariantSet from variants.tasks import run_import_variants_bg_job -from variants.tests.factories import SmallVariantFactory, CaseFactory +from variants.tests.factories import CaseFactory, SmallVariantFactory def write_test_files(directory, name, members): diff --git a/variants/tests/test_models.py b/variants/tests/test_models.py index f6005526e..c2eee4db9 100644 --- a/variants/tests/test_models.py +++ b/variants/tests/test_models.py @@ -1,25 +1,25 @@ """Tests for ``variants.models``.""" -import uuid from datetime import datetime, timedelta from unittest.mock import patch +import uuid from django.conf import settings -from projectroles.models import Project, SODAR_CONSTANTS +from projectroles.models import SODAR_CONSTANTS, Project +from test_plus.test import TestCase from variants.tests.factories import ( - SmallVariantFlagsFactory, + CaseWithVariantSetFactory, ProjectFactory, SmallVariantFactory, - CaseWithVariantSetFactory, + SmallVariantFlagsFactory, ) -from test_plus.test import TestCase from ..models import ( + Case, + SmallVariant, SmallVariantFlags, SmallVariantSet, cleanup_variant_sets, - SmallVariant, - Case, clear_old_kiosk_cases, ) diff --git a/variants/tests/test_queries.py b/variants/tests/test_queries.py index 3f0ec4bf7..e08e3a2e7 100644 --- a/variants/tests/test_queries.py +++ b/variants/tests/test_queries.py @@ -5,52 +5,51 @@ - VCF export is only tested for case one at the moment, as it shares a major part of the implementation with the render and tabular file export query. """ -from variants.helpers import get_engine - from clinvar.tests.factories import ClinvarFactory from cohorts.tests.factories import TestCohortBase from conservation.tests.factories import KnownGeneAAFactory +from dbsnp.tests.factories import DbsnpFactory from extra_annos.tests.factories import ExtraAnnoFactory -from frequencies.tests.factories import MitomapFactory, HelixMtDbFactory, MtDbFactory +from frequencies.tests.factories import HelixMtDbFactory, MitomapFactory, MtDbFactory +from geneinfo.tests.factories import ( + AcmgFactory, + EnsemblToGeneSymbolFactory, + ExacConstraintsFactory, + GeneIdInHpoFactory, + GeneIdToInheritanceFactory, + GnomadConstraintsFactory, + HgncFactory, + MgiMappingFactory, + RefseqToEnsemblFactory, + RefseqToGeneSymbolFactory, +) from hgmd.tests.factories import HgmdPublicLocusFactory -from variants.models import Case, SmallVariantSet, SmallVariantFlags +from variants.helpers import get_engine +from variants.models import Case, SmallVariantFlags, SmallVariantSet from variants.queries import ( - CasePrefetchQuery, CaseExportTableQuery, CaseExportVcfQuery, CaseLoadPrefetchedQuery, - ProjectPrefetchQuery, - ProjectLoadPrefetchedQuery, + CasePrefetchQuery, KnownGeneAAQuery, + ProjectLoadPrefetchedQuery, + ProjectPrefetchQuery, SmallVariantUserAnnotationQuery, normalize_chrom, ) -from geneinfo.tests.factories import ( - HgncFactory, - AcmgFactory, - GeneIdToInheritanceFactory, - GeneIdInHpoFactory, - GnomadConstraintsFactory, - ExacConstraintsFactory, - MgiMappingFactory, - RefseqToGeneSymbolFactory, - EnsemblToGeneSymbolFactory, - RefseqToEnsemblFactory, -) -from dbsnp.tests.factories import DbsnpFactory + from .factories import ( - SmallVariantFactory, - SmallVariantSummaryFactory, - ProjectFactory, - ProjectCasesSmallVariantQueryFactory, - SmallVariantQueryFactory, + AcmgCriteriaRatingFactory, CaseWithVariantSetFactory, - SmallVariantFlagsFactory, + ProjectCasesSmallVariantQueryFactory, + ProjectFactory, SmallVariantCommentFactory, - AcmgCriteriaRatingFactory, + SmallVariantFactory, + SmallVariantFlagsFactory, + SmallVariantQueryFactory, + SmallVariantSummaryFactory, ) -from .helpers import TestBase, SupportQueryTestBase - +from .helpers import SupportQueryTestBase, TestBase # TODO: select correct cases from multiple ones # TODO: prefetch from multiple cases @@ -94,7 +93,8 @@ def setUp(self): ] # Prepare disease gene GeneIdInHpoFactory( - ensembl_gene_id=small_vars[0].ensembl_gene_id, entrez_id=small_vars[0].refseq_gene_id, + ensembl_gene_id=small_vars[0].ensembl_gene_id, + entrez_id=small_vars[0].refseq_gene_id, ), # Prepare constraints self.gnomad_constraints = GnomadConstraintsFactory( @@ -187,15 +187,23 @@ def setUp(self): ] # Prepare constraints self.gnomad_constraints = [ - GnomadConstraintsFactory(ensembl_gene_id=small_vars[0].ensembl_gene_id, pLI=0.8,), - GnomadConstraintsFactory(ensembl_gene_id=small_vars[1].ensembl_gene_id, pLI=0.4,), + GnomadConstraintsFactory( + ensembl_gene_id=small_vars[0].ensembl_gene_id, + pLI=0.8, + ), + GnomadConstraintsFactory( + ensembl_gene_id=small_vars[1].ensembl_gene_id, + pLI=0.4, + ), ] self.exac_constraints = [ ExacConstraintsFactory( - ensembl_transcript_id=small_vars[0].ensembl_transcript_id, pLI=1.0, + ensembl_transcript_id=small_vars[0].ensembl_transcript_id, + pLI=1.0, ), ExacConstraintsFactory( - ensembl_transcript_id=small_vars[1].ensembl_transcript_id, pLI=0.5, + ensembl_transcript_id=small_vars[1].ensembl_transcript_id, + pLI=0.5, ), ] # Prepare smallvariant query results @@ -600,7 +608,7 @@ def setUp(self): def _setup_additional_variant_with_missing_inhouse_record(self): # Setup an additional variant and make sure it passes the default filter settings. - freq = 1 / 10 ** 3 # 0.001 + freq = 1 / 10**3 # 0.001 count = 1 SmallVariantFactory( release=self.case.release, @@ -1638,21 +1646,33 @@ def test_frequency_mitomap_enabled_vcf(self): def test_frequency_mtdb_limits_filter(self): self.run_query( CasePrefetchQuery, - {"mtdb_enabled": True, "mtdb_frequency": 0.01, "mtdb_count": None,}, + { + "mtdb_enabled": True, + "mtdb_frequency": 0.01, + "mtdb_count": None, + }, 3, ) def test_frequency_mtdb_limits_export(self): self.run_query( CaseExportTableQuery, - {"mtdb_enabled": True, "mtdb_frequency": 0.01, "mtdb_count": None,}, + { + "mtdb_enabled": True, + "mtdb_frequency": 0.01, + "mtdb_count": None, + }, 3, ) def test_frequency_mtdb_limits_vcf(self): self.run_query( CaseExportVcfQuery, - {"mtdb_enabled": True, "mtdb_frequency": 0.01, "mtdb_count": None,}, + { + "mtdb_enabled": True, + "mtdb_frequency": 0.01, + "mtdb_count": None, + }, 3, ) @@ -1695,39 +1715,67 @@ def test_frequency_helixmtdb_limits_vcf(self): def test_frequency_mitomap_limits_filter(self): self.run_query( CasePrefetchQuery, - {"mitomap_enabled": True, "mitomap_frequency": 0.01, "mitomap_count": None,}, + { + "mitomap_enabled": True, + "mitomap_frequency": 0.01, + "mitomap_count": None, + }, 3, ) def test_frequency_mitomap_limits_export(self): self.run_query( CaseExportTableQuery, - {"mitomap_enabled": True, "mitomap_frequency": 0.01, "mitomap_count": None,}, + { + "mitomap_enabled": True, + "mitomap_frequency": 0.01, + "mitomap_count": None, + }, 3, ) def test_frequency_mitomap_limits_vcf(self): self.run_query( CaseExportVcfQuery, - {"mitomap_enabled": True, "mitomap_frequency": 0.01, "mitomap_count": None,}, + { + "mitomap_enabled": True, + "mitomap_frequency": 0.01, + "mitomap_count": None, + }, 3, ) def test_count_mtdb_limits_filter(self): self.run_query( - CasePrefetchQuery, {"mtdb_enabled": True, "mtdb_frequency": None, "mtdb_count": 2,}, 3, + CasePrefetchQuery, + { + "mtdb_enabled": True, + "mtdb_frequency": None, + "mtdb_count": 2, + }, + 3, ) def test_count_mtdb_limits_export(self): self.run_query( CaseExportTableQuery, - {"mtdb_enabled": True, "mtdb_frequency": None, "mtdb_count": 2,}, + { + "mtdb_enabled": True, + "mtdb_frequency": None, + "mtdb_count": 2, + }, 3, ) def test_count_mtdb_limits_vcf(self): self.run_query( - CaseExportVcfQuery, {"mtdb_enabled": True, "mtdb_frequency": None, "mtdb_count": 2,}, 3, + CaseExportVcfQuery, + { + "mtdb_enabled": True, + "mtdb_frequency": None, + "mtdb_count": 2, + }, + 3, ) def test_count_helixmtdb_het_limits_filter(self): @@ -1805,21 +1853,33 @@ def test_count_helixmtdb_hom_limits_vcf(self): def test_count_mitomap_limits_filter(self): self.run_query( CasePrefetchQuery, - {"mitomap_enabled": True, "mitomap_frequency": None, "mitomap_count": 2,}, + { + "mitomap_enabled": True, + "mitomap_frequency": None, + "mitomap_count": 2, + }, 3, ) def test_count_mitomap_limits_export(self): self.run_query( CaseExportTableQuery, - {"mitomap_enabled": True, "mitomap_frequency": None, "mitomap_count": 2,}, + { + "mitomap_enabled": True, + "mitomap_frequency": None, + "mitomap_count": 2, + }, 3, ) def test_count_mitomap_limits_vcf(self): self.run_query( CaseExportVcfQuery, - {"mitomap_enabled": True, "mitomap_frequency": None, "mitomap_count": 2,}, + { + "mitomap_enabled": True, + "mitomap_frequency": None, + "mitomap_count": 2, + }, 3, ) @@ -2752,7 +2812,7 @@ def test_genomic_region_two_regions_vcf(self): self.run_query(CaseExportVcfQuery, {"genomic_region": [("1", 1, 199), ("1", 300, 399)]}, 4) def test_genomic_region_only_chromosome(self): - self.run_query(CasePrefetchQuery, {"genomic_region": [("1", 0, 2 ** 31 - 1)]}, 6) + self.run_query(CasePrefetchQuery, {"genomic_region": [("1", 0, 2**31 - 1)]}, 6) # --------------------------------------------------------------------------- @@ -4646,11 +4706,19 @@ def setUp(self): ) def test_display_hgmd_membership_query(self): - res = self.run_query(CasePrefetchQuery, {"require_in_hgmd_public": False}, 2,) + res = self.run_query( + CasePrefetchQuery, + {"require_in_hgmd_public": False}, + 2, + ) self.assertEqual(res[1].hgmd_accession, self.hgmd.variation_name) def test_require_in_hgmd_and_display_membership_query(self): - res = self.run_query(CasePrefetchQuery, {"require_in_hgmd_public": True}, 1,) + res = self.run_query( + CasePrefetchQuery, + {"require_in_hgmd_public": True}, + 1, + ) self.assertEqual(res[0].hgmd_accession, self.hgmd.variation_name) diff --git a/variants/tests/test_query_presets.py b/variants/tests/test_query_presets.py index fab00bc7f..cb5aaac27 100644 --- a/variants/tests/test_query_presets.py +++ b/variants/tests/test_query_presets.py @@ -486,7 +486,10 @@ def testToSettingsCompoundHeterozygous(self): { "recessive_index": "index", "recessive_mode": "compound-recessive", - "genotype": {"index": None, "father": None,}, + "genotype": { + "index": None, + "father": None, + }, }, ) # child with mother @@ -498,7 +501,10 @@ def testToSettingsCompoundHeterozygous(self): { "recessive_index": "index", "recessive_mode": "compound-recessive", - "genotype": {"index": None, "mother": None,}, + "genotype": { + "index": None, + "mother": None, + }, }, ) # trio denovo @@ -510,7 +516,11 @@ def testToSettingsCompoundHeterozygous(self): { "recessive_index": "index", "recessive_mode": "compound-recessive", - "genotype": {"index": None, "father": None, "mother": None,}, + "genotype": { + "index": None, + "father": None, + "mother": None, + }, }, ) # trio dominant inherited @@ -522,7 +532,11 @@ def testToSettingsCompoundHeterozygous(self): { "recessive_index": "index", "recessive_mode": "compound-recessive", - "genotype": {"index": None, "father": None, "mother": None,}, + "genotype": { + "index": None, + "father": None, + "mother": None, + }, }, ) @@ -548,7 +562,10 @@ def testToSettingsRecessive(self): { "recessive_index": "index", "recessive_mode": "recessive", - "genotype": {"index": None, "father": None,}, + "genotype": { + "index": None, + "father": None, + }, }, ) # child with mother @@ -560,7 +577,10 @@ def testToSettingsRecessive(self): { "recessive_index": "index", "recessive_mode": "recessive", - "genotype": {"index": None, "mother": None,}, + "genotype": { + "index": None, + "mother": None, + }, }, ) # trio denovo @@ -572,7 +592,11 @@ def testToSettingsRecessive(self): { "recessive_index": "index", "recessive_mode": "recessive", - "genotype": {"index": None, "father": None, "mother": None,}, + "genotype": { + "index": None, + "father": None, + "mother": None, + }, }, ) # trio dominant inherited @@ -584,7 +608,11 @@ def testToSettingsRecessive(self): { "recessive_index": "index", "recessive_mode": "recessive", - "genotype": {"index": None, "father": None, "mother": None,}, + "genotype": { + "index": None, + "father": None, + "mother": None, + }, }, ) @@ -1377,7 +1405,11 @@ def testValues(self): def testToSettingsWholeGenome(self): self.assertEqual( query_presets.Chromosomes.WHOLE_GENOME.to_settings(), - {"genomic_region": [], "gene_allowlist": [], "gene_blocklist": [],}, + { + "genomic_region": [], + "gene_allowlist": [], + "gene_blocklist": [], + }, ) def testToSettingsAutosomes(self): @@ -1393,19 +1425,37 @@ def testToSettingsAutosomes(self): def testToSettingsXChromosome(self): self.assertEqual( query_presets.Chromosomes.X_CHROMOSOME.to_settings(), - {"genomic_region": ["X",], "gene_allowlist": [], "gene_blocklist": [],}, + { + "genomic_region": [ + "X", + ], + "gene_allowlist": [], + "gene_blocklist": [], + }, ) def testToSettingsYChromosome(self): self.assertEqual( query_presets.Chromosomes.Y_CHROMOSOME.to_settings(), - {"genomic_region": ["Y",], "gene_allowlist": [], "gene_blocklist": [],}, + { + "genomic_region": [ + "Y", + ], + "gene_allowlist": [], + "gene_blocklist": [], + }, ) def testToSettingsMTChromosome(self): self.assertEqual( query_presets.Chromosomes.MT_CHROMOSOME.to_settings(), - {"genomic_region": ["MT",], "gene_allowlist": [], "gene_blocklist": [],}, + { + "genomic_region": [ + "MT", + ], + "gene_allowlist": [], + "gene_blocklist": [], + }, ) diff --git a/variants/tests/test_schemas.py b/variants/tests/test_schemas.py index 360ba4a54..91cb44f3f 100644 --- a/variants/tests/test_schemas.py +++ b/variants/tests/test_schemas.py @@ -8,9 +8,9 @@ from variants.forms import FilterForm from variants.query_schemas import ( SCHEMA_QUERY_V1, - load_json, DefaultValidatingDraft7Validator, convert_query_json_to_small_variant_filter_form_v1, + load_json, ) from variants.tests.factories import CaseWithVariantSetFactory diff --git a/variants/tests/test_submit_filter.py b/variants/tests/test_submit_filter.py index 1cac13f5b..d9f2fa4e7 100644 --- a/variants/tests/test_submit_filter.py +++ b/variants/tests/test_submit_filter.py @@ -3,28 +3,27 @@ import json from unittest.mock import patch +from django.conf import settings from requests_mock import Mocker - from test_plus.test import TestCase +from variants.models import SmallVariantQueryGeneScores, SmallVariantQueryVariantScores from variants.tests.factories import ( - SmallVariantFactory, + CaseWithVariantSetFactory, FilterBgJobFactory, - ProjectCasesFilterBgJobFactory, FormDataFactory, - CaseWithVariantSetFactory, + ProjectCasesFilterBgJobFactory, + SmallVariantFactory, ) + from ..models import ( - SmallVariantQuery, - ProjectCasesSmallVariantQuery, CaddPathogenicityScoreCache, MutationTasterPathogenicityScoreCache, + ProjectCasesSmallVariantQuery, + SmallVariantQuery, UmdPathogenicityScoreCache, ) from ..submit_filter import CaseFilter, ProjectCasesFilter -from variants.models import SmallVariantQueryGeneScores -from variants.models import SmallVariantQueryVariantScores -from django.conf import settings class CaseFilterTest(TestCase): diff --git a/variants/tests/test_sync_upstream.py b/variants/tests/test_sync_upstream.py index 4c44c2e30..fe944b52d 100644 --- a/variants/tests/test_sync_upstream.py +++ b/variants/tests/test_sync_upstream.py @@ -5,15 +5,15 @@ import os from unittest.mock import patch -import requests_mock from projectroles.constants import SODAR_CONSTANTS +import requests_mock from test_plus.test import TestCase from variants import sync_upstream from variants.tests.factories import ( CaseWithVariantSetFactory, - SyncCaseListBgJobFactory, RemoteSiteFactory, + SyncCaseListBgJobFactory, ) @@ -100,7 +100,11 @@ def setUp(self): def testRun(self, r_mock): r_mock.get( "%s/samplesheets/api/remote/get/%s/%s?isa=1" - % (self.remote_site.url, self.project.sodar_uuid, self.remote_site.secret,), + % ( + self.remote_site.url, + self.project.sodar_uuid, + self.remote_site.secret, + ), status_code=200, text=self.isa_tab_json, ) diff --git a/variants/tests/test_tasks.py b/variants/tests/test_tasks.py index da2eff450..87faa9edc 100644 --- a/variants/tests/test_tasks.py +++ b/variants/tests/test_tasks.py @@ -4,16 +4,16 @@ separately but that's fine. """ from unittest import mock -from unittest.mock import Mock, call -from unittest.mock import patch +from unittest.mock import Mock, call, patch +from bgjobs.models import BackgroundJob +from projectroles.models import Project from test_plus.test import TestCase from variants.tests.factories import CaseWithVariantSetFactory + from .. import tasks from ..models import ExportFileBgJob -from bgjobs.models import BackgroundJob -from projectroles.models import Project class ExportFileTaskTest(TestCase): @@ -73,5 +73,8 @@ def test_calls_correct_function(self): sender = Mock() tasks.setup_periodic_tasks(sender) sender.add_periodic_task.assert_has_calls( - [call(schedule=mock.ANY, sig=mock.ANY), call(schedule=mock.ANY, sig=mock.ANY),] + [ + call(schedule=mock.ANY, sig=mock.ANY), + call(schedule=mock.ANY, sig=mock.ANY), + ] ) diff --git a/variants/tests/test_templatetags.py b/variants/tests/test_templatetags.py index 022f75b80..c0f4f0f37 100644 --- a/variants/tests/test_templatetags.py +++ b/variants/tests/test_templatetags.py @@ -1,8 +1,8 @@ """Tests for ``variants.templatetags``.""" -from geneinfo.tests.factories import HpoFactory, HpoNameFactory from test_plus.test import TestCase +from geneinfo.tests.factories import HpoFactory, HpoNameFactory from variants.templatetags import variants_tags diff --git a/variants/tests/test_ui.py b/variants/tests/test_ui.py index 65bade878..5088b4a5f 100644 --- a/variants/tests/test_ui.py +++ b/variants/tests/test_ui.py @@ -1,35 +1,33 @@ """UI tests for the projectroles app""" +import json import os import socket -import json import time from unittest import skipIf from django.contrib import auth from django.test import LiveServerTestCase from django.urls import reverse - +from projectroles.models import SODAR_CONSTANTS, Role +from projectroles.tests.test_models import ProjectMixin, RoleAssignmentMixin +import projectroles.tests.test_ui from selenium import webdriver from selenium.common.exceptions import NoSuchElementException, StaleElementReferenceException from selenium.webdriver.common.by import By -from selenium.webdriver.support.ui import WebDriverWait -from selenium.webdriver.support import expected_conditions as ec from selenium.webdriver.common.desired_capabilities import DesiredCapabilities - -from projectroles.models import Role, SODAR_CONSTANTS -from projectroles.tests.test_models import ProjectMixin, RoleAssignmentMixin -import projectroles.tests.test_ui +from selenium.webdriver.support import expected_conditions as ec +from selenium.webdriver.support.ui import WebDriverWait from extra_annos.tests.factories import ExtraAnnoFactory, ExtraAnnoFieldFactory from variants.tests.factories import ( + CaseWithVariantSetFactory, + ProjectFactory, SampleVariantStatisticsFactory, SmallVariantFactory, - ProjectFactory, - CaseWithVariantSetFactory, ) -from ..models import update_variant_counts +from ..models import update_variant_counts # SODAR constants PROJECT_ROLE_OWNER = SODAR_CONSTANTS["PROJECT_ROLE_OWNER"] @@ -935,7 +933,7 @@ def test_preset_de_novo_on_quality_settings(self): self.selenium.find_element_by_id("id_%s_dp_het" % self.case.index).get_attribute( "value" ), - "8", + "10", ) @skipIf(SKIP_SELENIUM, SKIP_SELENIUM_MESSAGE) diff --git a/variants/tests/test_views.py b/variants/tests/test_views.py index f905d0e62..4b5d5332f 100644 --- a/variants/tests/test_views.py +++ b/variants/tests/test_views.py @@ -2,99 +2,98 @@ import contextlib import itertools import json +from unittest.mock import patch -import requests_mock +from django.conf import settings from django.urls import reverse +from projectroles.app_settings import AppSettingAPI from projectroles.constants import SODAR_CONSTANTS +from projectroles.models import Project, Role from projectroles.templatetags.projectroles_common_tags import site_version from projectroles.tests.test_models import PROJECT_ROLE_OWNER - +import requests_mock from requests_mock import Mocker -from unittest.mock import patch -from django.conf import settings -from variants.helpers import get_engine -from projectroles.models import Project, Role -from projectroles.app_settings import AppSettingAPI from clinvar.tests.factories import ClinvarFactory -from cohorts.tests.factories import TestCohortBase, CohortFactory +from cohorts.tests.factories import CohortFactory, TestCohortBase from conservation.tests.factories import KnownGeneAAFactory from frequencies.tests.factories import ( - ThousandGenomesFactory, + ExacFactory, GnomadExomesFactory, GnomadGenomesFactory, - ExacFactory, - MitomapFactory, HelixMtDbFactory, + MitomapFactory, MtDbFactory, + ThousandGenomesFactory, ) +from geneinfo.models import Hgnc, HpoName, RefseqToHgnc from geneinfo.tests.factories import ( - HpoFactory, + EnsemblToGeneSymbolFactory, + EnsemblToRefseqFactory, + ExacConstraintsFactory, + GnomadConstraintsFactory, HgncFactory, - Mim2geneMedgenFactory, + HpoFactory, HpoNameFactory, - GnomadConstraintsFactory, - ExacConstraintsFactory, - EnsemblToRefseqFactory, + Mim2geneMedgenFactory, RefseqToEnsemblFactory, - EnsemblToGeneSymbolFactory, RefseqToGeneSymbolFactory, ) from svs.models import StructuralVariant from svs.tests.factories import StructuralVariantFactory +from variants.helpers import get_engine from variants.models import ( + AcmgCriteriaRating, + CaddPathogenicityScoreCache, + CaddSubmissionBgJob, Case, + ComputeProjectVariantsStatsBgJob, + DistillerSubmissionBgJob, ExportFileBgJob, + ExportProjectCasesFileBgJob, FilterBgJob, + MutationTasterPathogenicityScoreCache, ProjectCasesFilterBgJob, - ExportProjectCasesFileBgJob, - DistillerSubmissionBgJob, - ComputeProjectVariantsStatsBgJob, + SmallVariant, + SmallVariantComment, SmallVariantFlags, - AcmgCriteriaRating, SmallVariantSet, - SmallVariant, - update_variant_counts, - MutationTasterPathogenicityScoreCache, - UmdPathogenicityScoreCache, - CaddPathogenicityScoreCache, - CaddSubmissionBgJob, SpanrSubmissionBgJob, - SmallVariantComment, + UmdPathogenicityScoreCache, + update_variant_counts, ) from variants.queries import DeleteSmallVariantsQuery, DeleteStructuralVariantsQuery from variants.templatetags.variants_tags import smallvar_description from variants.tests.factories import ( + AcmgCriteriaRatingFormDataFactory, + CaseCommentsFactory, + CaseCommentsFormFactory, CaseFactory, - SmallVariantFactory, - FormDataFactory, - FilterBgJobFactory, - ProjectCasesFilterBgJobFactory, - ProjectFactory, + CaseNotesStatusFormFactory, + CaseWithVariantSetFactory, + DeleteCaseBgJobFactory, DistillerSubmissionBgJobFactory, ExportFileBgJobFactory, ExportFileJobResultFactory, ExportProjectCasesFileBgJobFactory, - SmallVariantFlagsFormDataFactory, - SmallVariantCommentFormDataFactory, ExportProjectCasesFileBgJobResultFactory, - AcmgCriteriaRatingFormDataFactory, - SmallVariantFlagsFactory, - SmallVariantCommentFactory, - CaseNotesStatusFormFactory, - CaseCommentsFormFactory, - CaseCommentsFactory, + FilterBgJobFactory, + FormDataFactory, + MultiSmallVariantFlagsAndCommentFormDataFactory, + ProjectCasesFilterBgJobFactory, + ProjectFactory, + RemoteSiteFactory, SampleVariantStatisticsFactory, - DeleteCaseBgJobFactory, - CaseWithVariantSetFactory, + SmallVariantCommentFactory, + SmallVariantCommentFormDataFactory, + SmallVariantFactory, + SmallVariantFlagsFactory, + SmallVariantFlagsFormDataFactory, SmallVariantQueryFactory, - RemoteSiteFactory, - MultiSmallVariantFlagsAndCommentFormDataFactory, ) from variants.tests.helpers import ViewTestBase from variants.tests.test_sync_upstream import load_isa_tab from variants.variant_stats import rebuild_case_variant_stats, rebuild_project_variant_stats -from geneinfo.models import HpoName, Hgnc, RefseqToHgnc # TODO: This base class is still used by geneinfo view tests. @@ -274,7 +273,9 @@ def test_render_empty_tsv(self): response = self.client.get( reverse( "variants:project-download-annotations", - kwargs={"project": self.case.project.sodar_uuid,}, + kwargs={ + "project": self.case.project.sodar_uuid, + }, ) ) self.assertEqual(response.status_code, 200) @@ -485,8 +486,7 @@ def test_delete_as_user_and_fail(self): class TestCaseDeleteJobDetailView(ViewTestBase): - """Test ProjectStatsJobDetailView. - """ + """Test ProjectStatsJobDetailView.""" def setUp(self): super().setUp() @@ -932,8 +932,7 @@ def test_post_spanr(self, mock): class TestCasePrefetchFilterView(ViewTestBase): - """Tests for CasePrefetchFilterView. - """ + """Tests for CasePrefetchFilterView.""" def setUp(self): super().setUp() @@ -987,8 +986,7 @@ def test_invalid_form(self): class TestCaseFilterJobView(ViewTestBase): - """Tests for CaseFilterJobView. - """ + """Tests for CaseFilterJobView.""" def setUp(self): super().setUp() @@ -1051,9 +1049,15 @@ def setUp(self): ), ] self.decipher_term = HpoFactory( - database_id="DECIPHER:1", hpo_id="HP:0000004", name="Disease 2", + database_id="DECIPHER:1", + hpo_id="HP:0000004", + name="Disease 2", + ) + self.orpha_term = HpoFactory( + database_id="ORPHA:1", + hpo_id="HP:0000005", + name="Disease 3", ) - self.orpha_term = HpoFactory(database_id="ORPHA:1", hpo_id="HP:0000005", name="Disease 3",) self.bgjob.smallvariantquery.query_results.add(self.small_vars[0], self.small_vars[2]) self.bgjob.smallvariantquery.query_settings["prio_hpo_terms"] = [ self.hpo_term.hpo_id, @@ -1467,8 +1471,7 @@ def test_ranking_umd_and_pheno_results(self, mock): class TestFilterJobDetailView(ViewTestBase): - """Tests for FilterJobDetailView. - """ + """Tests for FilterJobDetailView.""" def setUp(self): super().setUp() @@ -1496,8 +1499,7 @@ def test_correct_case_name(self): class TestFilterJobResubmitView(ViewTestBase): - """Tests for FilterJobResubmitView. - """ + """Tests for FilterJobResubmitView.""" def setUp(self): super().setUp() @@ -1526,8 +1528,7 @@ def test_redirect(self): class TestFilterJobGetStatus(ViewTestBase): - """Tests for FilterJobGetStatusView. - """ + """Tests for FilterJobGetStatusView.""" def setUp(self): super().setUp() @@ -1568,8 +1569,7 @@ def test_getting_status_missing_uuid(self): class TestFilterJobGetPrevious(ViewTestBase): - """Tests for FilterJobGetPreviousView. - """ + """Tests for FilterJobGetPreviousView.""" def setUp(self): super().setUp() @@ -1605,8 +1605,7 @@ def test_getting_previous_job_non_existing(self): class TestProjectCasesFilterJobGetStatus(ViewTestBase): - """Tests for ProjectCasesFilterJobGetStatusView. - """ + """Tests for ProjectCasesFilterJobGetStatusView.""" def setUp(self): super().setUp() @@ -1650,8 +1649,7 @@ def test_getting_status_missing_uuid(self): class TestProjectCasesFilterJobGetPrevious(ViewTestBase): - """Tests for ProjectCasesFilterJobGetPreviousView. - """ + """Tests for ProjectCasesFilterJobGetPreviousView.""" def setUp(self): super().setUp() @@ -1689,8 +1687,7 @@ def test_getting_previous_job_non_existing(self): class TestProjectCasesFilterView(ViewTestBase): - """Tests for FilterProjectCasesFilterView. - """ + """Tests for FilterProjectCasesFilterView.""" def setUp(self): super().setUp() @@ -1877,7 +1874,10 @@ def test_download_filter_cohort_as_superuser(self): response, reverse( "variants:project-cases-export-job-detail", - kwargs={"project": project.sodar_uuid, "job": created_job.sodar_uuid,}, + kwargs={ + "project": project.sodar_uuid, + "job": created_job.sodar_uuid, + }, ), ) @@ -1905,7 +1905,10 @@ def test_download_filter_cohort_as_contributor(self): response, reverse( "variants:project-cases-export-job-detail", - kwargs={"project": project.sodar_uuid, "job": created_job.sodar_uuid,}, + kwargs={ + "project": project.sodar_uuid, + "job": created_job.sodar_uuid, + }, ), ) @@ -1933,7 +1936,10 @@ def test_download_filter_cohort_as_superuser_for_cohort_by_contributor(self): response, reverse( "variants:project-cases-export-job-detail", - kwargs={"project": project.sodar_uuid, "job": created_job.sodar_uuid,}, + kwargs={ + "project": project.sodar_uuid, + "job": created_job.sodar_uuid, + }, ), ) @@ -1961,7 +1967,10 @@ def test_download_filter_cohort_as_contributor_for_cohort_by_superuser(self): response, reverse( "variants:project-cases-export-job-detail", - kwargs={"project": project.sodar_uuid, "job": created_job.sodar_uuid,}, + kwargs={ + "project": project.sodar_uuid, + "job": created_job.sodar_uuid, + }, ), ) @@ -2075,8 +2084,7 @@ def test_get_previous_job_existing_as_contributor_for_cohort_by_superuser(self): class TestProjectCasesPrefetchFilterView(ViewTestBase): - """Tests for FilterProjectCasesPrefetchFilterView. - """ + """Tests for FilterProjectCasesPrefetchFilterView.""" def setUp(self): super().setUp() @@ -2132,8 +2140,7 @@ def test_variant_set_missing(self): class TestProjectCasesFilterJobDetailView(ViewTestBase): - """Tests for ProjectCasesFilterJobDetailView. - """ + """Tests for ProjectCasesFilterJobDetailView.""" def setUp(self): super().setUp() @@ -2161,8 +2168,7 @@ def test_correct_project_name(self): class TestProjectCasesLoadPrefetchedFilterView(ViewTestBase): - """Tests for ProjectCasesLoadPrefetchedFilterView. - """ + """Tests for ProjectCasesLoadPrefetchedFilterView.""" def setUp(self): super().setUp() @@ -2590,8 +2596,7 @@ def test_ranking_umd_results(self, mock): class TestProjectCasesFilterJobResubmitView(ViewTestBase): - """Tests for ProjectCasesFilterJobResubmitView. - """ + """Tests for ProjectCasesFilterJobResubmitView.""" def setUp(self): super().setUp() @@ -2622,8 +2627,7 @@ def test_redirect(self): class TestDistillerSubmissionJobDetailView(ViewTestBase): - """Tests for DistillerSubmissionJobDetailView. - """ + """Tests for DistillerSubmissionJobDetailView.""" def setUp(self): super().setUp() @@ -2651,8 +2655,7 @@ def test_correct_case_name(self): class TestDistillerSubmissionJobResubmitView(ViewTestBase): - """Tests for DistillerSubmissionJobResubmitView. - """ + """Tests for DistillerSubmissionJobResubmitView.""" def setUp(self): super().setUp() @@ -2921,7 +2924,8 @@ def _base_test_content(self, db): response.context["pop_freqs"]["1000GP"]["Total"]["af"], self.thousand_genomes.af ) self.assertEqual( - response.context["clinvar"][0].pathogenicity, self.clinvar.pathogenicity, + response.context["clinvar"][0].pathogenicity, + self.clinvar.pathogenicity, ) self.assertEqual( response.context["knowngeneaa"][0]["alignment"], self.knowngeneaa.alignment @@ -3372,8 +3376,7 @@ def test_download(self): class TestExportProjectCasesFileJobResubmitView(ViewTestBase): - """Test ExportProjectCasesFileJobResubmitView. - """ + """Test ExportProjectCasesFileJobResubmitView.""" def setUp(self): super().setUp() @@ -3419,8 +3422,7 @@ def test_render_detail_view(self): class TestExportProjectCasesFileJobDownloadView(ViewTestBase): - """Test ExportProjectCasesFileJobDownloadView. - """ + """Test ExportProjectCasesFileJobDownloadView.""" def setUp(self): super().setUp() @@ -3465,8 +3467,7 @@ def test_download(self): class TestProjectStatsJobCreateView(ViewTestBase): - """Test ProjectStatsJobCreateView. - """ + """Test ProjectStatsJobCreateView.""" def setUp(self): super().setUp() @@ -3495,8 +3496,7 @@ def test_project_stat_job_creation(self): class TestProjectStatsJobDetailView(ViewTestBase): - """Test ProjectStatsJobDetailView. - """ + """Test ProjectStatsJobDetailView.""" def setUp(self): super().setUp() @@ -3526,8 +3526,7 @@ def test_render(self): class TestSmallVariantFlagsApiView(ViewTestBase): - """Test SmallVariantFlagsApiView. - """ + """Test SmallVariantFlagsApiView.""" def setUp(self): super().setUp() @@ -3726,8 +3725,7 @@ def test_post_provoke_form_error(self): class TestSmallVariantCommentSubmitApiView(ViewTestBase): - """Test SmallVariantCommentApiView. - """ + """Test SmallVariantCommentApiView.""" def setUp(self): super().setUp() @@ -3735,7 +3733,9 @@ def setUp(self): self.randomuser.save() case, self.variant_set, _ = CaseWithVariantSetFactory.get("small") self._make_assignment( - case.project, self.randomuser, Role.objects.get_or_create(name=PROJECT_ROLE_OWNER)[0], + case.project, + self.randomuser, + Role.objects.get_or_create(name=PROJECT_ROLE_OWNER)[0], ) self.small_var = SmallVariantFactory(variant_set=self.variant_set) @@ -3849,8 +3849,7 @@ def test_user_cant_edit_admin_case_comment(self): class TestMultiSmallVariantFlagsAndCommentApiView(ViewTestBase): - """Test MultiSmallVariantFlagsAndCommentApiView. - """ + """Test MultiSmallVariantFlagsAndCommentApiView.""" def setUp(self): super().setUp() @@ -3878,7 +3877,11 @@ def test_get_json_response_two_non_existing(self): "variants:multi-small-variant-flags-comment-api", kwargs={"project": self.case.project.sodar_uuid}, ), - {"variant_list": [json.dumps([self.small_vars_post[0], self.small_vars_post[1]])],}, + { + "variant_list": [ + json.dumps([self.small_vars_post[0], self.small_vars_post[1]]) + ], + }, ) self.assertEqual(SmallVariantFlags.objects.count(), 0) self.assertEqual(response.status_code, 200) @@ -3900,7 +3903,10 @@ def test_get_json_response_two_non_existing(self): "flag_summary": None, }, "flags_interfering": [], - "variant_list": [self.small_vars_post[0], self.small_vars_post[1],], + "variant_list": [ + self.small_vars_post[0], + self.small_vars_post[1], + ], }, ) @@ -3915,7 +3921,11 @@ def test_get_json_response_one_existing_one_non_existing(self): ), { **data, - "variant_list": json.dumps([self.small_vars_post[0],]), + "variant_list": json.dumps( + [ + self.small_vars_post[0], + ] + ), "csrfmiddlewaretoken": "xxx", }, ) @@ -3925,7 +3935,11 @@ def test_get_json_response_one_existing_one_non_existing(self): "variants:multi-small-variant-flags-comment-api", kwargs={"project": self.case.project.sodar_uuid}, ), - {"variant_list": [json.dumps([self.small_vars_post[0], self.small_vars_post[1]])],}, + { + "variant_list": [ + json.dumps([self.small_vars_post[0], self.small_vars_post[1]]) + ], + }, ) data.pop("text") self.assertEqual(response.status_code, 200) @@ -3934,7 +3948,10 @@ def test_get_json_response_one_existing_one_non_existing(self): { "flags": data, "flags_interfering": [], - "variant_list": [self.small_vars_post[0], self.small_vars_post[1],], + "variant_list": [ + self.small_vars_post[0], + self.small_vars_post[1], + ], }, ) @@ -3949,7 +3966,12 @@ def test_get_json_response_two_existing_non_conflicting_flags(self): ), { **data, - "variant_list": json.dumps([self.small_vars_post[0], self.small_vars_post[1],]), + "variant_list": json.dumps( + [ + self.small_vars_post[0], + self.small_vars_post[1], + ] + ), "csrfmiddlewaretoken": "xxx", }, ) @@ -3959,7 +3981,11 @@ def test_get_json_response_two_existing_non_conflicting_flags(self): "variants:multi-small-variant-flags-comment-api", kwargs={"project": self.case.project.sodar_uuid}, ), - {"variant_list": [json.dumps([self.small_vars_post[0], self.small_vars_post[1]])],}, + { + "variant_list": [ + json.dumps([self.small_vars_post[0], self.small_vars_post[1]]) + ], + }, ) data.pop("text") self.assertEqual(response.status_code, 200) @@ -3968,7 +3994,10 @@ def test_get_json_response_two_existing_non_conflicting_flags(self): { "flags": data, "flags_interfering": [], - "variant_list": [self.small_vars_post[0], self.small_vars_post[1],], + "variant_list": [ + self.small_vars_post[0], + self.small_vars_post[1], + ], }, ) @@ -3978,7 +4007,9 @@ def test_get_json_response_two_existing_conflicting_flags(self): data1 = vars(MultiSmallVariantFlagsAndCommentFormDataFactory(text="")) data2 = vars( MultiSmallVariantFlagsAndCommentFormDataFactory( - flag_candidate=True, flag_visual="positive", text="", + flag_candidate=True, + flag_visual="positive", + text="", ) ) self.client.post( @@ -3988,7 +4019,11 @@ def test_get_json_response_two_existing_conflicting_flags(self): ), { **data1, - "variant_list": json.dumps([self.small_vars_post[0],]), + "variant_list": json.dumps( + [ + self.small_vars_post[0], + ] + ), "csrfmiddlewaretoken": "xxx", }, ) @@ -3999,7 +4034,11 @@ def test_get_json_response_two_existing_conflicting_flags(self): ), { **data2, - "variant_list": json.dumps([self.small_vars_post[1],]), + "variant_list": json.dumps( + [ + self.small_vars_post[1], + ] + ), "csrfmiddlewaretoken": "xxx", }, ) @@ -4009,7 +4048,11 @@ def test_get_json_response_two_existing_conflicting_flags(self): "variants:multi-small-variant-flags-comment-api", kwargs={"project": self.case.project.sodar_uuid}, ), - {"variant_list": [json.dumps([self.small_vars_post[0], self.small_vars_post[1]])],}, + { + "variant_list": [ + json.dumps([self.small_vars_post[0], self.small_vars_post[1]]) + ], + }, ) data1.pop("text") self.assertEqual(response.status_code, 200) @@ -4018,7 +4061,10 @@ def test_get_json_response_two_existing_conflicting_flags(self): { "flags": data1, "flags_interfering": sorted(["flag_visual", "flag_candidate"]), - "variant_list": [self.small_vars_post[0], self.small_vars_post[1],], + "variant_list": [ + self.small_vars_post[0], + self.small_vars_post[1], + ], }, ) @@ -4034,7 +4080,12 @@ def test_post_json_response_two_non_existing(self): ), { **data, - "variant_list": json.dumps([self.small_vars_post[0], self.small_vars_post[1],]), + "variant_list": json.dumps( + [ + self.small_vars_post[0], + self.small_vars_post[1], + ] + ), "csrfmiddlewaretoken": "xxx", }, ) @@ -4071,7 +4122,11 @@ def test_post_json_response_one_existing_one_non_existing(self): ), { **data1, - "variant_list": json.dumps([self.small_vars_post[0],]), + "variant_list": json.dumps( + [ + self.small_vars_post[0], + ] + ), "csrfmiddlewaretoken": "xxx", }, ) @@ -4084,7 +4139,12 @@ def test_post_json_response_one_existing_one_non_existing(self): ), { **data2, - "variant_list": json.dumps([self.small_vars_post[0], self.small_vars_post[1],]), + "variant_list": json.dumps( + [ + self.small_vars_post[0], + self.small_vars_post[1], + ] + ), "csrfmiddlewaretoken": "xxx", }, ) @@ -4120,7 +4180,12 @@ def test_post_json_response_two_existing_non_conflicting_flags(self): ), { **data1, - "variant_list": json.dumps([self.small_vars_post[0], self.small_vars_post[1],]), + "variant_list": json.dumps( + [ + self.small_vars_post[0], + self.small_vars_post[1], + ] + ), "csrfmiddlewaretoken": "xxx", }, ) @@ -4132,7 +4197,11 @@ def test_post_json_response_two_existing_non_conflicting_flags(self): ), { **data2, - "variant_list": json.dumps([self.small_vars_post[0],]), + "variant_list": json.dumps( + [ + self.small_vars_post[0], + ] + ), "csrfmiddlewaretoken": "xxx", }, ) @@ -4157,12 +4226,16 @@ def test_post_json_response_two_existing_conflicting_flags(self): data1 = vars(MultiSmallVariantFlagsAndCommentFormDataFactory(text="")) data2 = vars( MultiSmallVariantFlagsAndCommentFormDataFactory( - flag_candidate=True, flag_visual="positive", text="", + flag_candidate=True, + flag_visual="positive", + text="", ) ) data3 = vars( MultiSmallVariantFlagsAndCommentFormDataFactory( - flag_molecular="negative", flag_final_causative=True, text="", + flag_molecular="negative", + flag_final_causative=True, + text="", ) ) self.client.post( @@ -4172,7 +4245,11 @@ def test_post_json_response_two_existing_conflicting_flags(self): ), { **data1, - "variant_list": json.dumps([self.small_vars_post[0],]), + "variant_list": json.dumps( + [ + self.small_vars_post[0], + ] + ), "csrfmiddlewaretoken": "xxx", }, ) @@ -4183,7 +4260,11 @@ def test_post_json_response_two_existing_conflicting_flags(self): ), { **data2, - "variant_list": json.dumps([self.small_vars_post[1],]), + "variant_list": json.dumps( + [ + self.small_vars_post[1], + ] + ), "csrfmiddlewaretoken": "xxx", }, ) @@ -4226,7 +4307,12 @@ def test_post_json_response_add_comment(self): ), { **data, - "variant_list": json.dumps([self.small_vars_post[0], self.small_vars_post[1],]), + "variant_list": json.dumps( + [ + self.small_vars_post[0], + self.small_vars_post[1], + ] + ), "csrfmiddlewaretoken": "xxx", }, ) @@ -4277,7 +4363,12 @@ def test_post_remove_flags(self): ), { **data1, - "variant_list": json.dumps([self.small_vars_post[0], self.small_vars_post[1],]), + "variant_list": json.dumps( + [ + self.small_vars_post[0], + self.small_vars_post[1], + ] + ), "csrfmiddlewaretoken": "xxx", }, ) @@ -4290,7 +4381,12 @@ def test_post_remove_flags(self): ), { **data2, - "variant_list": json.dumps([self.small_vars_post[0], self.small_vars_post[1],]), + "variant_list": json.dumps( + [ + self.small_vars_post[0], + self.small_vars_post[1], + ] + ), "csrfmiddlewaretoken": "xxx", }, ) @@ -4319,7 +4415,12 @@ def test_post_provoke_form_error(self): ), { **vars(MultiSmallVariantFlagsAndCommentFormDataFactory(flag_bookmarker=100)), - "variant_list": json.dumps([self.small_vars_post[0], self.small_vars_post[1],]), + "variant_list": json.dumps( + [ + self.small_vars_post[0], + self.small_vars_post[1], + ] + ), "csrfmiddlewaretoken": "xxx", }, ) @@ -4334,7 +4435,9 @@ def setUp(self): self.randomuser.save() case, self.variant_set, _ = CaseWithVariantSetFactory.get("small") self._make_assignment( - case.project, self.randomuser, Role.objects.get_or_create(name=PROJECT_ROLE_OWNER)[0], + case.project, + self.randomuser, + Role.objects.get_or_create(name=PROJECT_ROLE_OWNER)[0], ) def test_admin_can_delete_user_comment(self): @@ -4390,8 +4493,7 @@ def test_user_cant_delete_admin_comment(self): class TestBackgroundJobListView(ViewTestBase): - """Test BackgroundJobListView. - """ + """Test BackgroundJobListView.""" def setUp(self): super().setUp() @@ -4735,7 +4837,9 @@ def setUp(self): self.randomuser.save() case, self.variant_set, _ = CaseWithVariantSetFactory.get("small") self._make_assignment( - case.project, self.randomuser, Role.objects.get_or_create(name=PROJECT_ROLE_OWNER)[0], + case.project, + self.randomuser, + Role.objects.get_or_create(name=PROJECT_ROLE_OWNER)[0], ) def test_user_submit_case_comment(self): @@ -4805,7 +4909,9 @@ def setUp(self): self.randomuser.save() case, self.variant_set, _ = CaseWithVariantSetFactory.get("small") self._make_assignment( - case.project, self.randomuser, Role.objects.get_or_create(name=PROJECT_ROLE_OWNER)[0], + case.project, + self.randomuser, + Role.objects.get_or_create(name=PROJECT_ROLE_OWNER)[0], ) def test_admin_can_delete_user_case_comment(self): @@ -4953,7 +5059,11 @@ def test_get_json(self, r_mock): """Test display of case list page.""" r_mock.get( "%s/samplesheets/api/remote/get/%s/%s?isa=1" - % (self.remote_site.url, self.project.sodar_uuid, self.remote_site.secret,), + % ( + self.remote_site.url, + self.project.sodar_uuid, + self.remote_site.secret, + ), status_code=200, text=self.isa_tab_json, ) diff --git a/variants/tests/test_views_api.py b/variants/tests/test_views_api.py index 52c039fda..6832cbe5f 100644 --- a/variants/tests/test_views_api.py +++ b/variants/tests/test_views_api.py @@ -3,22 +3,14 @@ from django.urls import reverse from projectroles.tests.test_views_api import EMPTY_KNOX_TOKEN -from .factories import ( - CaseWithVariantSetFactory, - SmallVariantQueryFactory, - FilterBgJobFactory, -) -from .helpers import ( - ApiViewTestBase, - VARFISH_INVALID_VERSION, - VARFISH_INVALID_MIMETYPE, -) +from ..models import FilterBgJob, SmallVariantQuery +from ..query_schemas import SCHEMA_QUERY_V1, DefaultValidatingDraft7Validator +from ..views_api import JobStatus +from .factories import CaseWithVariantSetFactory, FilterBgJobFactory, SmallVariantQueryFactory +from .helpers import VARFISH_INVALID_MIMETYPE, VARFISH_INVALID_VERSION, ApiViewTestBase # TODO: add tests that include permission testing from .test_views import GenerateSmallVariantResultMixin -from ..models import SmallVariantQuery, FilterBgJob -from ..query_schemas import SCHEMA_QUERY_V1, DefaultValidatingDraft7Validator -from ..views_api import JobStatus def transmogrify_pedigree(pedigree): @@ -55,7 +47,8 @@ def _test_list_with_invalid_x(self, media_type=None, version=None): with self.login(self.superuser): response = self.request_knox( reverse( - "variants:api-case-list", kwargs={"project": self.case.project.sodar_uuid}, + "variants:api-case-list", + kwargs={"project": self.case.project.sodar_uuid}, ), media_type=media_type, version=version, @@ -72,7 +65,8 @@ def test_list(self): with self.login(self.superuser): response = self.request_knox( reverse( - "variants:api-case-list", kwargs={"project": str(self.case.project.sodar_uuid)}, + "variants:api-case-list", + kwargs={"project": str(self.case.project.sodar_uuid)}, ), ) @@ -89,7 +83,10 @@ def test_list(self): def _test_retrieve_with_invalid_x(self, media_type=None, version=None): with self.login(self.superuser): response = self.request_knox( - reverse("variants:api-case-retrieve", kwargs={"case": str(self.case.sodar_uuid)},), + reverse( + "variants:api-case-retrieve", + kwargs={"case": str(self.case.sodar_uuid)}, + ), media_type=media_type, version=version, ) @@ -104,7 +101,10 @@ def test_retrieve_with_invalid_media_type(self): def test_retrieve(self): with self.login(self.superuser): response = self.client.get( - reverse("variants:api-case-retrieve", kwargs={"case": self.case.sodar_uuid},) + reverse( + "variants:api-case-retrieve", + kwargs={"case": self.case.sodar_uuid}, + ) ) expected = self._expected_case_data() @@ -618,7 +618,8 @@ class TestSmallVariantQuerySettingsShortcutApiView( def test_get_success(self): url = reverse( - "variants:api-query-settings-shortcut", kwargs={"case": self.case.sodar_uuid}, + "variants:api-query-settings-shortcut", + kwargs={"case": self.case.sodar_uuid}, ) response = self.request_knox(url) @@ -759,7 +760,8 @@ def test_get_access_allowed(self): ] url = reverse( - "variants:api-query-settings-shortcut", kwargs={"case": self.case.sodar_uuid}, + "variants:api-query-settings-shortcut", + kwargs={"case": self.case.sodar_uuid}, ) for user in good_users: @@ -777,7 +779,8 @@ def test_get_access_forbidden(self): ] url = reverse( - "variants:api-query-settings-shortcut", kwargs={"case": self.case.sodar_uuid}, + "variants:api-query-settings-shortcut", + kwargs={"case": self.case.sodar_uuid}, ) for user in bad_users: diff --git a/variants/urls.py b/variants/urls.py index 6fa63d89d..64ee3c50d 100644 --- a/variants/urls.py +++ b/variants/urls.py @@ -1,6 +1,6 @@ from django.conf.urls import url -from . import views, views_api +from . import views, views_api app_name = "variants" ui_urlpatterns = [ diff --git a/variants/variant_stats.py b/variants/variant_stats.py index 4b2a5fc36..67bf9b74b 100644 --- a/variants/variant_stats.py +++ b/variants/variant_stats.py @@ -2,13 +2,12 @@ from django.db import transaction import numpy as np - from projectroles.plugins import get_backend_api + from var_stats_qc.qc import compute_het_hom_chrx, compute_relatedness, compute_relatedness_many from .forms import FILTER_FORM_TRANSLATE_EFFECTS -from .models import SmallVariant, CaseVariantStats, ProjectVariantStats - +from .models import CaseVariantStats, ProjectVariantStats, SmallVariant #: Effects to ignore when computing stats. IGNORE_EFFECTS = ( @@ -43,7 +42,7 @@ def _get_dp_bin(dp): return 200 -def gather_variant_stats(variant_set): +def gather_variant_stats(variant_set): # noqa: C901 """Iterate over ``SmallVariant`` objects of ``variant_set`` and collect various statistics.""" # TODO: could be refactored into class with multiple smaller, easier-to-read functions samples = variant_set.case.get_members_with_samples() diff --git a/variants/views.py b/variants/views.py index e950c7f04..8288d1f97 100644 --- a/variants/views.py +++ b/variants/views.py @@ -1,167 +1,164 @@ +from base64 import b64encode +from collections import defaultdict +import contextlib +import decimal +from itertools import chain import os import re import tempfile -from itertools import chain -from collections import defaultdict import uuid -import contextlib - -import decimal +from bgjobs.models import BackgroundJob +from bgjobs.views import DEFAULT_PAGINATION as BGJOBS_DEFAULT_PAGINATION import binning -import numpy as np -import requests -from base64 import b64encode - -from variants.helpers import get_engine from django.conf import settings from django.contrib import messages from django.contrib.humanize.templatetags.humanize import naturaltime from django.contrib.postgres.aggregates import ArrayAgg from django.core.exceptions import ObjectDoesNotExist, ValidationError +from django.db import transaction from django.db.models import Q from django.forms.models import model_to_dict -from django.http import HttpResponse, Http404, JsonResponse -from django.db import transaction -from django.shortcuts import render, redirect, get_object_or_404, reverse +from django.http import Http404, HttpResponse, JsonResponse +from django.shortcuts import get_object_or_404, redirect, render, reverse from django.utils import timezone from django.views.generic import ( DetailView, FormView, ListView, - View, RedirectView, - UpdateView, TemplateView, + UpdateView, + View, ) from django.views.generic.detail import SingleObjectMixin, SingleObjectTemplateResponseMixin -import xlsxwriter - -import simplejson as json from django.views.generic.edit import FormMixin +import numpy as np +from projectroles.app_settings import AppSettingAPI from projectroles.constants import SODAR_CONSTANTS from projectroles.models import RemoteSite +from projectroles.plugins import get_active_plugins, get_backend_api from projectroles.templatetags.projectroles_common_tags import site_version +from projectroles.views import ( + LoggedInPermissionMixin, + LoginRequiredMixin, + PluginContextMixin, + ProjectContextMixin, + ProjectPermissionMixin, +) +import requests +import simplejson as json +import xlsxwriter -from bgjobs.models import BackgroundJob -from bgjobs.views import DEFAULT_PAGINATION as BGJOBS_DEFAULT_PAGINATION from clinvar.models import Clinvar from cohorts.models import Cohort -from extra_annos.views import ExtraAnnosMixin, ExtraAnnoField +from extra_annos.views import ExtraAnnoField, ExtraAnnosMixin from frequencies.models import MT_DB_INFO -from geneinfo.views import get_gene_infos +from frequencies.views import FrequencyMixin from geneinfo.models import ( - NcbiGeneInfo, - NcbiGeneRif, - HpoName, - Hpo, EnsemblToGeneSymbol, Hgnc, + Hpo, + HpoName, + NcbiGeneInfo, + NcbiGeneRif, build_entrez_id_to_symbol, ) -from frequencies.views import FrequencyMixin -from projectroles.app_settings import AppSettingAPI -from projectroles.views import ( - LoginRequiredMixin, - LoggedInPermissionMixin, - ProjectContextMixin, - ProjectPermissionMixin, - PluginContextMixin, -) -from projectroles.plugins import get_backend_api, get_active_plugins - +from geneinfo.views import get_gene_infos from genomicfeatures.models import GeneInterval from varfish.users.models import User -from .queries import ( - CaseLoadPrefetchedQuery, - ProjectLoadPrefetchedQuery, - KnownGeneAAQuery, - DeleteStructuralVariantsQuery, - DeleteSmallVariantsQuery, - SmallVariantUserAnnotationQuery, - CaseSecondHitsQuery, +from variants.helpers import get_engine + +from .file_export import RowWithSampleProxy +from .forms import ( + FILTER_FORM_TRANSLATE_CLINVAR_STATUS, + FILTER_FORM_TRANSLATE_EFFECTS, + FILTER_FORM_TRANSLATE_SIGNIFICANCE, + RE_FIND_TERMS, + AcmgCriteriaRatingForm, + CaseCommentsForm, + CaseForm, + CaseNotesStatusForm, + CaseTermsForm, + EmptyForm, + ExportFileResubmitForm, + ExportProjectCasesFileResubmitForm, + FilterForm, + KioskUploadForm, + ProjectCasesFilterForm, + ProjectStatsJobForm, + SmallVariantCommentForm, + SmallVariantFlagsForm, + SyncProjectJobForm, + save_file, ) from .models import ( - only_source_name, + CASE_STATUS_CHOICES, + AcmgCriteriaRating, + CaddSubmissionBgJob, Case, + CaseAwareProject, + CaseComments, + ClearExpiredExportedFilesBgJob, + ClearInactiveVariantSetsBgJob, + ClearOldKioskCasesBgJob, + ComputeProjectVariantsStatsBgJob, + DeleteCaseBgJob, + DistillerSubmissionBgJob, ExportFileBgJob, ExportProjectCasesFileBgJob, - CaddSubmissionBgJob, - DistillerSubmissionBgJob, - SpanrSubmissionBgJob, - ComputeProjectVariantsStatsBgJob, FilterBgJob, + ImportVariantsBgJob, + KioskAnnotateBgJob, Project, - CaseAwareProject, + ProjectCasesFilterBgJob, + ProjectCasesSmallVariantQuery, + RefreshSmallVariantSummaryBgJob, + RowWithAffectedCasesPerGene, SmallVariant, - SmallVariantFlags, SmallVariantComment, + SmallVariantFlags, SmallVariantQuery, - ProjectCasesSmallVariantQuery, - ProjectCasesFilterBgJob, - annotate_with_phenotype_scores, - annotate_with_pathogenicity_scores, - annotate_with_joint_scores, - AcmgCriteriaRating, - SyncCaseListBgJob, SmallVariantSet, - ImportVariantsBgJob, - CaseComments, - CASE_STATUS_CHOICES, - RowWithAffectedCasesPerGene, SmallVariantSummary, - KioskAnnotateBgJob, - update_variant_counts, - DeleteCaseBgJob, - RefreshSmallVariantSummaryBgJob, - ClearOldKioskCasesBgJob, - ClearInactiveVariantSetsBgJob, - ClearExpiredExportedFilesBgJob, + SpanrSubmissionBgJob, + SyncCaseListBgJob, + annotate_with_joint_scores, + annotate_with_pathogenicity_scores, + annotate_with_phenotype_scores, load_molecular_impact, + only_source_name, + update_variant_counts, ) -from .forms import ( - ExportFileResubmitForm, - ExportProjectCasesFileResubmitForm, - FILTER_FORM_TRANSLATE_CLINVAR_STATUS, - FILTER_FORM_TRANSLATE_EFFECTS, - FILTER_FORM_TRANSLATE_SIGNIFICANCE, - FilterForm, - ProjectCasesFilterForm, - EmptyForm, - ProjectStatsJobForm, - SmallVariantCommentForm, - SmallVariantFlagsForm, - AcmgCriteriaRatingForm, - CaseForm, - SyncProjectJobForm, - CaseNotesStatusForm, - CaseCommentsForm, - KioskUploadForm, - save_file, - CaseTermsForm, - RE_FIND_TERMS, +from .queries import ( + CaseLoadPrefetchedQuery, + CaseSecondHitsQuery, + DeleteSmallVariantsQuery, + DeleteStructuralVariantsQuery, + KnownGeneAAQuery, + ProjectLoadPrefetchedQuery, + SmallVariantUserAnnotationQuery, ) from .query_presets import QUICK_PRESETS, PedigreeMember from .query_schemas import ( - DefaultValidatingDraft7Validator, SCHEMA_QUERY_V1, + DefaultValidatingDraft7Validator, convert_query_json_to_small_variant_filter_form_v1, ) from .sync_upstream import fetch_remote_pedigree from .tasks import ( - export_file_task, - export_project_cases_file_task, cadd_submission_task, - distiller_submission_task, compute_project_variants_stats, - sync_project_upstream, - single_case_filter_task, + delete_case_bg_job, + distiller_submission_task, + export_file_task, + export_project_cases_file_task, project_cases_filter_task, run_kiosk_bg_job, - delete_case_bg_job, + single_case_filter_task, spanr_submission_task, + sync_project_upstream, ) -from .file_export import RowWithSampleProxy from .templatetags.variants_tags import get_term_description, smallvar_description @@ -218,7 +215,10 @@ def get_queryset(self): super() .get_queryset() .filter(project__sodar_uuid=self.kwargs["project"]) - .prefetch_related("project", "case_comments",) + .prefetch_related( + "project", + "case_comments", + ) ) def get_context_data(self, *args, **kwargs): @@ -253,7 +253,11 @@ def _compute_progress(self, project): class CaseListGetAnnotationsView( - LoginRequiredMixin, LoggedInPermissionMixin, ProjectPermissionMixin, ProjectContextMixin, View, + LoginRequiredMixin, + LoggedInPermissionMixin, + ProjectPermissionMixin, + ProjectContextMixin, + View, ): template_name = "variants/case_list/annotation.html" permission_required = "variants.view_data" @@ -276,7 +280,11 @@ def get(self, *args, **kwargs): result["project"], result["limit"] ) result["sv_commentsflags"] = self.join_sv_comments_and_flags(result["project"]) - return render(self.request, self.template_name, self.get_context_data(**result),) + return render( + self.request, + self.template_name, + self.get_context_data(**result), + ) def join_small_var_comments_and_flags(self, project, limit): def get_gene_symbol(release, chromosome, start, end): @@ -314,7 +322,7 @@ def get_gene_symbol(release, chromosome, start, end): for record in flags: position_key = (record.release, record.chromosome, record.start, record.end) - if not position_key in gene_symbol_cache: + if position_key not in gene_symbol_cache: gene_symbol_cache[position_key] = get_gene_symbol(*position_key) result[(record.chromosome, record.start, record.reference, record.alternative)][ case @@ -325,7 +333,7 @@ def get_gene_symbol(release, chromosome, start, end): for record in comments: position_key = (record.release, record.chromosome, record.start, record.end) - if not position_key in gene_symbol_cache: + if position_key not in gene_symbol_cache: gene_symbol_cache[position_key] = get_gene_symbol(*position_key) result[(record.chromosome, record.start, record.reference, record.alternative)][ case @@ -343,7 +351,7 @@ def get_gene_symbol(release, chromosome, start, end): for record in acmg_ratings: position_key = (record.release, record.chromosome, record.start, record.end) - if not position_key in gene_symbol_cache: + if position_key not in gene_symbol_cache: gene_symbol_cache[position_key] = get_gene_symbol(*position_key) result[(record.chromosome, record.start, record.reference, record.alternative)][ case @@ -393,14 +401,18 @@ def join_sv_comments_and_flags(self, project): class CaseListGetQCView( - LoginRequiredMixin, LoggedInPermissionMixin, ProjectPermissionMixin, ProjectContextMixin, View, + LoginRequiredMixin, + LoggedInPermissionMixin, + ProjectPermissionMixin, + ProjectContextMixin, + View, ): template_name = "variants/case_list/qc.html" permission_required = "variants.view_data" model = Case ordering = ("-date_modified",) - def get(self, *args, **kwargs): + def get(self, *args, **kwargs): # noqa: 901 result = dict() result["project"] = CaseAwareProject.objects.prefetch_related("variant_stats").get( sodar_uuid=kwargs["project"] @@ -455,7 +467,10 @@ def get(self, *args, **kwargs): result["qcdata_sample_variant_stats"] = self.get_sample_variant_stats_content( result["project"], result["sample_variant_stats"] ) - except (SmallVariantSet.variant_stats.RelatedObjectDoesNotExist, AttributeError) as e: + except ( # noqa: F841 + SmallVariantSet.variant_stats.RelatedObjectDoesNotExist, + AttributeError, + ) as _e: pass # swallow, defaults set above # Prepare effect counts data for QC download @@ -495,7 +510,11 @@ def get(self, *args, **kwargs): "\n".join(qcdata_site_depth_content).encode("utf-8") ) - return render(self.request, self.template_name, self.get_context_data(**result),) + return render( + self.request, + self.template_name, + self.get_context_data(**result), + ) def get_relatedness_content(self, project): result = [ @@ -533,7 +552,18 @@ def get_relatedness_content(self, project): def get_sample_variant_stats_content(self, project, sample_variant_stats): result = [ - "\t".join(["Sample", "Ts", "Tv", "Ts/Tv", "SNVs", "InDels", "MNVs", "X hom./het.",]) + "\t".join( + [ + "Sample", + "Ts", + "Tv", + "Ts/Tv", + "SNVs", + "InDels", + "MNVs", + "X hom./het.", + ] + ) ] for item in sample_variant_stats: result.append( @@ -751,7 +781,7 @@ def post(self, *args, **kwargs): kwargs["user"] = self.request.user try: record = CaseComments.objects.get(**kwargs) - except ObjectDoesNotExist as e: + except ObjectDoesNotExist as _e: # noqa: F841 return HttpResponse( json.dumps( {"result": "Not authorized to update comment or no comment found."} @@ -824,7 +854,7 @@ def post(self, *args, **kwargs): try: comment = CaseComments.objects.get(**kwargs) - except ObjectDoesNotExist as e: + except ObjectDoesNotExist as _e: # noqa: F841 return HttpResponse( json.dumps({"result": "Not authorized to delete comment or no comment found."}), content_type="application/json", @@ -889,7 +919,13 @@ def get_annotations_by_variant(case=None, cases=None, project=None): def init_var(description): result[case_uuid].setdefault( - description, {"variants": [], "flags": None, "comments": [], "acmg_rating": None,}, + description, + { + "variants": [], + "flags": None, + "comments": [], + "acmg_rating": None, + }, ) for small_var in annotated_small_vars.small_variants: @@ -1114,7 +1150,18 @@ def get_relatedness_content(self): def get_sample_variant_stats_content(self): case = self.get_object() result = [ - "\t".join(["Sample", "Ts", "Tv", "Ts/Tv", "SNVs", "InDels", "MNVs", "X hom./het.",]) + "\t".join( + [ + "Sample", + "Ts", + "Tv", + "Ts/Tv", + "SNVs", + "InDels", + "MNVs", + "X hom./het.", + ] + ) ] for item in case.latest_variant_set.variant_stats.sample_variant_stats.all(): result.append( @@ -1427,7 +1474,9 @@ def get(self, *args, **kwargs): user=self.request.user, ) delete_job = DeleteCaseBgJob.objects.create( - project=self.get_project(self.request, self.kwargs), bg_job=bg_job, case=case, + project=self.get_project(self.request, self.kwargs), + bg_job=bg_job, + case=case, ) # Construct background job objects bg_job2 = BackgroundJob.objects.create( @@ -1506,7 +1555,10 @@ def get_impl(self, case=None, cases=None, project=None): identifier = "-".join(cases[:5].name) else: identifier = project.title.replace(" ", "-") - response = HttpResponse(f.read(), content_type=content_type,) + response = HttpResponse( + f.read(), + content_type=content_type, + ) response["Content-Disposition"] = "attachment; filename=case-annotations-%s.%s" % ( identifier, file_ext, @@ -1611,7 +1663,9 @@ def yield_rows(self, *, case, cases, project): anno["acmg_rating"].acmg_class if anno["acmg_rating"] else "N/A", ] + row_flags - + ["|".join(comments),] + + [ + "|".join(comments), + ] ) yield row @@ -2703,7 +2757,6 @@ def get_previous_query(self): if not self._previous_query: project = self.get_project(self.request, self.kwargs) cohort = self.get_cohort() - user = self.request.user if "job" in self.kwargs: filter_job = ProjectCasesFilterBgJob.objects.get(sodar_uuid=self.kwargs["job"]) else: @@ -2975,9 +3028,11 @@ def get(self, *args, **kwargs): cohort = Cohort.objects.get(sodar_uuid=cohort_uuid) if cohort_uuid else None if settings.KIOSK_MODE: user = User.get_kiosk_user() + else: + user = self.request.user filter_job = ( ProjectCasesFilterBgJob.objects.filter( - project=project, bg_job__user=self.request.user, cohort=cohort + project=project, bg_job__user=user, cohort=cohort ) .order_by("-bg_job__date_created") .first() @@ -3305,8 +3360,7 @@ class BackgroundJobListView( ProjectContextMixin, ListView, ): - """Display list of export jobs for case. - """ + """Display list of export jobs for case.""" permission_required = "variants.view_data" template_name = "variants/background_job_list.html" @@ -3326,8 +3380,7 @@ class SyncJobDetailView( ProjectContextMixin, DetailView, ): - """Display status and further details of the sync-project-upstream background job. - """ + """Display status and further details of the sync-project-upstream background job.""" permission_required = "variants.view_data" template_name = "variants/sync_job_detail.html" @@ -3343,8 +3396,7 @@ class ImportVariantsJobDetailView( ProjectContextMixin, DetailView, ): - """Display status and further details of the import case background job. - """ + """Display status and further details of the import case background job.""" permission_required = "variants.view_data" template_name = "variants/import_job_detail.html" @@ -3360,8 +3412,7 @@ class CaseDeleteJobDetailView( ProjectContextMixin, DetailView, ): - """Display status and further details of the import case background job. - """ + """Display status and further details of the import case background job.""" permission_required = "variants.view_data" template_name = "variants/case_delete_job_detail.html" @@ -3388,8 +3439,7 @@ class ExportFileJobDetailView( ProjectContextMixin, DetailView, ): - """Display status and further details of the file export background job. - """ + """Display status and further details of the file export background job.""" permission_required = "variants.view_data" template_name = "variants/export_job_detail.html" @@ -3488,8 +3538,7 @@ class ExportProjectCasesFileJobDetailView( ProjectContextMixin, DetailView, ): - """Display status and further details of the file export background job. - """ + """Display status and further details of the file export background job.""" permission_required = "variants.view_data" template_name = "variants/export_project_cases_job_detail.html" @@ -3850,8 +3899,7 @@ class ProjectStatsJobCreateView( ProjectContextMixin, FormView, ): - """Confirm creating a new project statistics computation job. - """ + """Confirm creating a new project statistics computation job.""" permission_required = "variants.view_data" template_name = "variants/project_stats_job_create.html" @@ -3881,8 +3929,7 @@ class ProjectStatsJobDetailView( ProjectContextMixin, DetailView, ): - """Display status and further details of project-wide statistics computation job. - """ + """Display status and further details of project-wide statistics computation job.""" permission_required = "variants.view_data" template_name = "variants/project_stats_job_detail.html" @@ -3993,7 +4040,7 @@ def post(self, *_args, **_kwargs): kwargs["user"] = self.request.user try: comment = SmallVariantComment.objects.get(**kwargs) - except ObjectDoesNotExist as e: + except ObjectDoesNotExist as _e: # noqa: F841 return HttpResponse( json.dumps({"result": "Not authorized to delete comment or no comment found."}), content_type="application/json", @@ -4035,7 +4082,11 @@ def post(self, *_args, **_kwargs): class MultiSmallVariantFlagsAndCommentApiView( - LoginRequiredMixin, LoggedInPermissionMixin, ProjectPermissionMixin, ProjectContextMixin, View, + LoginRequiredMixin, + LoggedInPermissionMixin, + ProjectPermissionMixin, + ProjectContextMixin, + View, ): """A view that returns JSON for the ``SmallVariantFlags`` for a variant of a case and allows updates.""" @@ -4225,7 +4276,7 @@ def post(self, *_args, **_kwargs): try: comment = SmallVariantComment.objects.get(**kwargs) - except ObjectDoesNotExist as e: + except ObjectDoesNotExist as _e: # noqa: F841 return HttpResponse( json.dumps({"result": "Not authorized to delete comment or no comment found."}), content_type="application/json", @@ -4427,7 +4478,10 @@ def post(self, *args, **kwargs): return render( self.request, self.template_name, - self.get_context_data(user=self.request.user, response=dict(result),), + self.get_context_data( + user=self.request.user, + response=dict(result), + ), ) @@ -4621,7 +4675,9 @@ def form_valid(self, form): def get_kiosk_project(self): """Return Project object for the Kiosk cases (or create it).""" cat, _ = Project.objects.get_or_create( - parent=None, type="CATEGORY", title=settings.KIOSK_CAT, + parent=None, + type="CATEGORY", + title=settings.KIOSK_CAT, ) proj = Project.objects.create( parent=cat, @@ -4721,7 +4777,8 @@ def get(self, *args, **kwargs): class ClearExpiredExportedFilesJobDetailView( - LoggedInPermissionMixin, DetailView, + LoggedInPermissionMixin, + DetailView, ): permission_required = "bgjobs.view_site_bgjobs" template_name = "variants/maintenance_job_detail.html" @@ -4732,7 +4789,8 @@ class ClearExpiredExportedFilesJobDetailView( class ClearInactiveVariantSetsJobDetailView( - LoggedInPermissionMixin, DetailView, + LoggedInPermissionMixin, + DetailView, ): permission_required = "bgjobs.view_site_bgjobs" template_name = "variants/maintenance_job_detail.html" @@ -4743,7 +4801,8 @@ class ClearInactiveVariantSetsJobDetailView( class ClearOldKioskCasesJobDetailView( - LoggedInPermissionMixin, DetailView, + LoggedInPermissionMixin, + DetailView, ): permission_required = "bgjobs.view_site_bgjobs" template_name = "variants/maintenance_job_detail.html" @@ -4754,7 +4813,8 @@ class ClearOldKioskCasesJobDetailView( class RefreshSmallVariantSummaryJobDetailView( - LoggedInPermissionMixin, DetailView, + LoggedInPermissionMixin, + DetailView, ): permission_required = "bgjobs.view_site_bgjobs" template_name = "variants/maintenance_job_detail.html" diff --git a/variants/views_api.py b/variants/views_api.py index b9170ba39..944f3f3a2 100644 --- a/variants/views_api.py +++ b/variants/views_api.py @@ -2,16 +2,16 @@ import typing import attrs +from bgjobs.models import JOB_STATE_DONE, JOB_STATE_FAILED, JOB_STATE_INITIAL, JOB_STATE_RUNNING import cattr -from bgjobs.models import JOB_STATE_FAILED, JOB_STATE_DONE, JOB_STATE_RUNNING, JOB_STATE_INITIAL from django.db.models import Q from projectroles.views_api import SODARAPIGenericProjectMixin, SODARAPIProjectPermission from rest_framework import serializers from rest_framework.exceptions import NotFound from rest_framework.generics import ( + CreateAPIView, ListAPIView, RetrieveAPIView, - CreateAPIView, UpdateAPIView, get_object_or_404, ) @@ -21,14 +21,14 @@ # # TOOD: timeline update from variants import query_presets -from variants.models import Case, SmallVariantQuery, FilterBgJob, SmallVariant +from variants.models import Case, FilterBgJob, SmallVariant, SmallVariantQuery from variants.serializers import ( CaseSerializer, + SettingsShortcuts, + SettingsShortcutsSerializer, + SmallVariantForResultSerializer, SmallVariantQuerySerializer, SmallVariantQueryUpdateSerializer, - SmallVariantForResultSerializer, - SettingsShortcutsSerializer, - SettingsShortcuts, ) @@ -61,7 +61,8 @@ def get_permission_required(self): class CaseRetrieveApiView( - VariantsApiBaseMixin, RetrieveAPIView, + VariantsApiBaseMixin, + RetrieveAPIView, ): """ Retrieve detail of the specified case. @@ -324,7 +325,8 @@ def get_permission_required(self): class SmallVariantQuerySettingsShortcutApiView( - VariantsApiBaseMixin, RetrieveAPIView, + VariantsApiBaseMixin, + RetrieveAPIView, ): """ Generate query settings for a given case by certain shortcuts. @@ -463,7 +465,7 @@ def get_object(self, *args, **kwargs): ) def _get_quick_presets(self) -> query_presets.QuickPresets: - """"Return quick preset if given in request.query_params""" + """ "Return quick preset if given in request.query_params""" if "quick_preset" in self.request.query_params: qp_name = self.request.query_params["quick_preset"] if qp_name not in attrs.fields_dict(query_presets._QuickPresetList):