From 77a2411c10358c6f549032c2ca39eeed30691143 Mon Sep 17 00:00:00 2001 From: Manuel Holtgrewe Date: Thu, 1 Apr 2021 11:57:43 +0200 Subject: [PATCH] Adding "submit to SPANR" feature. Closes #65 --- HISTORY.rst | 6 +- config/settings/base.py | 3 + docs_manual/variants_filtration.rst | 7 + variants/admin.py | 2 + variants/file_export.py | 1 + variants/forms.py | 1 + .../migrations/0080_spanrsubmissionbgjob.py | 72 ++++++++ variants/models.py | 38 ++++ variants/plugins.py | 2 + variants/submit_external.py | 116 +++++++++++- variants/tasks.py | 6 + variants/templates/variants/_filter_form.html | 11 ++ .../variants/_spanr_resubmit_modal.html | 34 ++++ .../templates/variants/spanr_job_detail.html | 172 ++++++++++++++++++ variants/tests/test_views.py | 40 ++++ variants/urls.py | 11 ++ variants/views.py | 83 +++++++++ 17 files changed, 602 insertions(+), 3 deletions(-) create mode 100644 variants/migrations/0080_spanrsubmissionbgjob.py create mode 100644 variants/templates/variants/_spanr_resubmit_modal.html create mode 100644 variants/templates/variants/spanr_job_detail.html diff --git a/HISTORY.rst b/HISTORY.rst index d9fab69f9..3528ce31f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -30,7 +30,8 @@ End-User Summary - Rebuild of variant summary database table happens every Sunday at 2:22am. - Added celery queues ``maintenance`` and ``export``. - Adding support for connecting two sites via the GAGH Beacon protocol. -- Adding link-out to "GenCC" +- Adding link-out to "GenCC". +- Adding "submit to SPANR" feature. Full Change List ================ @@ -59,7 +60,8 @@ Full Change List - Added celery queues ``maintenance`` and ``export``. - Adding support for connecting two sites via the GAGH Beacon protocol. - Making CADD version behind CADD REST API configurable. -- Adding link-out to "GenCC" +- Adding link-out to "GenCC". +- Adding "submit to SPANR" feature. ------- v0.22.1 diff --git a/config/settings/base.py b/config/settings/base.py index 4aa271b83..f76497a05 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -487,6 +487,9 @@ def fixed_array_type(field): VARFISH_MUTATIONTASTER_BATCH_VARS = env.int("VARFISH_MUTATIONTASTER_BATCH_VARS", 50) VARFISH_MUTATIONTASTER_MAX_VARS = env.int("VARFISH_MUTATIONTASTER_MAX_VARS", 500) +# VarfFish: Enable SPANR +VARFISH_ENABLE_SPANR_SUBMISSION = env.bool("VARFISH_ENABLE_SPANR_SUBMISSION", False) + # Varfish: UMD URL VARFISH_UMD_REST_API_URL = env.str( "VARFISH_UMD_REST_API_URL", "http://umd-predictor.eu/webservice.php" diff --git a/docs_manual/variants_filtration.rst b/docs_manual/variants_filtration.rst index d093826eb..891507e52 100644 --- a/docs_manual/variants_filtration.rst +++ b/docs_manual/variants_filtration.rst @@ -324,6 +324,13 @@ Here are the actions to create the recommended settings for submitting to Mutati The MutationDistiller submission uses the same feature as th VarFish VCF export. Thus, the limitations described in :ref:`download-as-file` apply. +Submit to SPANR +--------------- + +Also, the little triangle next to the :guilabel:`Filter & Display` gives you access to the :guilabel:`Submit to SPANR` action. +This is similar to submitting ot MutationDistiller described above. +Clicking the button will submit the data to SPANR after confirming this once again in popup window. + -------------------------- Variant Filtration Results -------------------------- diff --git a/variants/admin.py b/variants/admin.py index a27f952ca..e0e17e9d8 100644 --- a/variants/admin.py +++ b/variants/admin.py @@ -16,6 +16,7 @@ SyncCaseResultMessage, ImportVariantsBgJob, SmallVariantSet, + SpanrSubmissionBgJob, CasePhenotypeTerms, ) @@ -37,6 +38,7 @@ SyncCaseResultMessage, ImportVariantsBgJob, SmallVariantSet, + SpanrSubmissionBgJob, CasePhenotypeTerms, ) ) diff --git a/variants/file_export.py b/variants/file_export.py index 6d615e1a5..c317bea3f 100644 --- a/variants/file_export.py +++ b/variants/file_export.py @@ -437,6 +437,7 @@ def write_tmp_file(self): self._write_variants() self._write_trailing() #: Rewind temporary file to beginning and return it. + self.tmp_file.flush() self.tmp_file.seek(0) return self.tmp_file diff --git a/variants/forms.py b/variants/forms.py index c1ebd9bd1..d1467a865 100644 --- a/variants/forms.py +++ b/variants/forms.py @@ -1375,6 +1375,7 @@ class FilterForm( ("download", "Generate downloadable file in background"), ("submit-mutationdistiller", "Submit to MutationDistiller"), ("submit-cadd", "Submit to CADD"), + ("submit-spanr", "Submit to SPANR"), ) ) diff --git a/variants/migrations/0080_spanrsubmissionbgjob.py b/variants/migrations/0080_spanrsubmissionbgjob.py new file mode 100644 index 000000000..a42ce96da --- /dev/null +++ b/variants/migrations/0080_spanrsubmissionbgjob.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2021-04-01 08:45 +from __future__ import unicode_literals +import uuid + +import bgjobs.models +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("projectroles", "0015_fix_appsetting_constraint"), + ("bgjobs", "0006_auto_20200526_1657"), + ("variants", "0079_auto_20210204_1006"), + ] + + operations = [ + migrations.CreateModel( + name="SpanrSubmissionBgJob", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ( + "sodar_uuid", + models.UUIDField(default=uuid.uuid4, help_text="Case SODAR UUID", unique=True), + ), + ( + "query_args", + django.contrib.postgres.fields.jsonb.JSONField( + help_text="(Validated) query parameters" + ), + ), + ( + "spanr_job_url", + models.CharField(help_text="The SPANR job URL", max_length=100, null=True), + ), + ( + "bg_job", + models.ForeignKey( + help_text="Background job for state etc.", + on_delete=django.db.models.deletion.CASCADE, + related_name="spanr_submission_bg_job", + to="bgjobs.BackgroundJob", + ), + ), + ( + "case", + models.ForeignKey( + help_text="The case to export", + on_delete=django.db.models.deletion.CASCADE, + to="variants.Case", + ), + ), + ( + "project", + models.ForeignKey( + help_text="Project in which this objects belongs", + on_delete=django.db.models.deletion.CASCADE, + to="projectroles.Project", + ), + ), + ], + bases=(bgjobs.models.JobModelMessageMixin, models.Model), + ), + ] diff --git a/variants/models.py b/variants/models.py index 533ea8614..8f97aa675 100644 --- a/variants/models.py +++ b/variants/models.py @@ -612,6 +612,7 @@ def get_background_jobs(self): Q(variants_exportfilebgjob_related__case=self) | Q(cadd_submission_bg_job__case=self) | Q(distiller_submission_bg_job__case=self) + | Q(spanr_submission_bg_job__case=self) | Q(filter_bg_job__case=self) ) @@ -1228,6 +1229,43 @@ def get_absolute_url(self): ) +class SpanrSubmissionBgJob(JobModelMessageMixin, models.Model): + """Background job for submitting variants to SPANR.""" + + #: Task description for logging. + task_desc = "Submission to SPANR" + + #: String identifying model in BackgroundJob. + spec_name = "variants.spanr_submission_bg_job" + + # Fields required by SODAR + sodar_uuid = models.UUIDField( + default=uuid_object.uuid4, unique=True, help_text="Case SODAR UUID" + ) + project = models.ForeignKey(Project, help_text="Project in which this objects belongs") + + bg_job = models.ForeignKey( + BackgroundJob, + null=False, + related_name="spanr_submission_bg_job", + help_text="Background job for state etc.", + on_delete=models.CASCADE, + ) + case = models.ForeignKey(Case, null=False, help_text="The case to export") + query_args = JSONField(null=False, help_text="(Validated) query parameters") + + spanr_job_url = models.CharField(max_length=100, null=True, help_text="The SPANR job URL") + + def get_human_readable_type(self): + return "SPANR Submission" + + def get_absolute_url(self): + return reverse( + "variants:spanr-job-detail", + kwargs={"project": self.project.sodar_uuid, "job": self.sodar_uuid}, + ) + + class SmallVariantComment(models.Model): """Model for commenting on a ``SmallVariant``.""" diff --git a/variants/plugins.py b/variants/plugins.py index 46d1991c4..0db80f82c 100644 --- a/variants/plugins.py +++ b/variants/plugins.py @@ -12,6 +12,7 @@ ExportFileBgJob, ExportProjectCasesFileBgJob, CaddSubmissionBgJob, + SpanrSubmissionBgJob, DistillerSubmissionBgJob, ComputeProjectVariantsStatsBgJob, FilterBgJob, @@ -280,6 +281,7 @@ class BackgroundJobsPlugin(BackgroundJobsPluginPoint): job_specs = { ExportFileBgJob.spec_name: ExportFileBgJob, CaddSubmissionBgJob.spec_name: CaddSubmissionBgJob, + SpanrSubmissionBgJob.spec_name: SpanrSubmissionBgJob, DistillerSubmissionBgJob.spec_name: DistillerSubmissionBgJob, ComputeProjectVariantsStatsBgJob.spec_name: ComputeProjectVariantsStatsBgJob, ExportProjectCasesFileBgJob.spec_name: ExportProjectCasesFileBgJob, diff --git a/variants/submit_external.py b/variants/submit_external.py index a03691147..b095f6925 100644 --- a/variants/submit_external.py +++ b/variants/submit_external.py @@ -1,5 +1,5 @@ """This module contains the code for file export""" - +import gzip import re from bs4 import BeautifulSoup @@ -143,3 +143,117 @@ def submit_cadd(job): job.mark_success() if timeline: tl_event.set_status("OK", "CADD submission complete for {case_name}") + + +#: URL to SPANR submission form +SPANR_POST_URL = "http://tools.genes.toronto.edu/" +#: Maximum number of lines to write out +SPANR_MAX_LINES = 40 + + +def submit_spanr(job): + """Submit a case to SPANR.""" + job.mark_start() + timeline = get_backend_api("timeline_backend") + if timeline: + tl_event = timeline.add_event( + project=job.project, + app_name="variants", + user=job.bg_job.user, + event_name="case_submit_spanr", + description="submitting {case_name} case to SPANR", + status_type="INIT", + ) + tl_event.add_object(obj=job.case, label="case_name", name=job.case.name) + try: + job.add_log_entry("Getting submission form for CSRF (security) token") + text_input = _submit_spanr_make_text(job) + job.add_log_entry("Getting submission form for CSRF (security) token") + session = requests.Session() + csrf_token = _submit_spanr_obtain_csrf_token(job, session, timeline, tl_event) + if not csrf_token: + return # bail out! + data = {"csrf_token": (None, csrf_token), "text_input": (None, text_input)} + job_id = _submit_spanr_post(data, job, session, timeline, tl_event) + if not job_id: + return # bail out! + # Get target URL + job.spanr_job_url = "%sresults/%s" % (SPANR_POST_URL, job_id) + job.add_log_entry("SPANR job page is %s" % job.spanr_job_url) + job.save() + except Exception as e: + job.mark_error(e) + if timeline: + tl_event.set_status("FAILED", "SPANR submission failed for {case_name}: %s") + raise + else: + job.mark_success() + if timeline: + tl_event.set_status("OK", "SPANR submission complete for {case_name}") + + +def _submit_spanr_post(data, job, session, timeline, tl_event): + job.add_log_entry("Submitting to %s..." % SPANR_POST_URL) + for k in ("job_name", "chrom", "pos", "variant_id", "ref", "alt"): + data[k] = (None, "") + response = session.post( + SPANR_POST_URL, + files=data, + headers={ + "Referer": SPANR_POST_URL, + "Origin": SPANR_POST_URL[:-1], + "Upgrade-Insecure-Requests": "1", + }, + ) + job.add_log_entry("Done submitting to %s" % SPANR_POST_URL) + if not response.ok: + job.mark_error("HTTP status code: {}".format(response.status_code)) + if timeline: + tl_event.set_status("FAILED", "SPANR submission failed for {case_name}") + return None + soup = BeautifulSoup(response.text, "html.parser") + job_id = None + for tag in soup.find_all("title"): + text = tag.string + if text.startswith("Result"): + job_id = text.split()[1] + if not job_id: + job.mark_error('Page title did not start with "Result"') + if timeline: + tl_event.set_status("FAILED", "SPANR submission failed for {case_name}") + return job_id + + +def _submit_spanr_make_text(job): + with CaseExporterVcf(job, job.case) as exporter: + job.add_log_entry("Creating temporary VCF file...") + tmp_file = exporter.write_tmp_file() + job.add_log_entry("Extracting first 40 variants...") + lines = [] + with gzip.open(tmp_file.name, "rt") as inputf: + i = 0 + for line in inputf: + if not line.startswith("#"): + lines.append("\t".join(line.split("\t")[:5])) + i += 1 + if i >= SPANR_MAX_LINES: + break + text_input = "\n".join(lines) + "\n" + return text_input + + +def _submit_spanr_obtain_csrf_token(job, session, timeline, tl_event): + response = session.get(SPANR_POST_URL) + if not response.ok: + job.mark_error("HTTP status code: {}".format(response.status_code)) + if timeline: + tl_event.set_status("FAILED", "SPANR submission failed for {case_name}") + return None + soup = BeautifulSoup(response.text, "html.parser") + tag = soup.find(id="csrf_token") + if not tag: + job.mark_error("Could not extract CSRF token") + if timeline: + tl_event.set_status("FAILED", "SPANR submission failed for {case_name}") + return None + return tag.attrs.get("value") diff --git a/variants/tasks.py b/variants/tasks.py index 2bf093836..254b7e10b 100644 --- a/variants/tasks.py +++ b/variants/tasks.py @@ -34,6 +34,12 @@ def cadd_submission_task(_self, submission_job_pk): submit_external.submit_cadd(models.CaddSubmissionBgJob.objects.get(pk=submission_job_pk)) +@app.task(bind=True) +def spanr_submission_task(_self, submission_job_pk): + """Task to submit a case to SPANR.""" + submit_external.submit_spanr(models.SpanrSubmissionBgJob.objects.get(pk=submission_job_pk)) + + @app.task(bind=True) def export_file_task(_self, export_job_pk): """Task to export single case to a file""" diff --git a/variants/templates/variants/_filter_form.html b/variants/templates/variants/_filter_form.html index c0975b769..f13a27474 100644 --- a/variants/templates/variants/_filter_form.html +++ b/variants/templates/variants/_filter_form.html @@ -7,6 +7,7 @@ {% get_django_setting 'VARFISH_ENABLE_EXOMISER_PRIORITISER' as exomiser_enabled %} {% get_django_setting 'VARFISH_ENABLE_CADD' as cadd_enabled %} {% get_django_setting 'VARFISH_ENABLE_CADD_SUBMISSION' as cadd_submission_enabled %} +{% get_django_setting 'VARFISH_ENABLE_SPANR_SUBMISSION' as spanr_submission_enabled %} {% get_is_testing as is_testing %} {% if form.errors %} @@ -151,6 +152,15 @@ Submit to CADD {% endif %} + {% if spanr_submission_enabled %} + + {% endif %} @@ -160,6 +170,7 @@ {% include "variants/_distiller_resubmit_modal.html" %} {% include "variants/_cadd_resubmit_modal.html" %} + {% include "variants/_spanr_resubmit_modal.html" %}