Adding "submit to SPANR" feature.

Closes #65

HISTORY.rst

@@ -30,7 +30,8 @@ End-User Summary
 - Rebuild of variant summary database table happens every Sunday at 2:22am.
 - Added celery queues ``maintenance`` and ``export``.
 - Adding support for connecting two sites via the GAGH Beacon protocol.
-- Adding link-out to "GenCC"
+- Adding link-out to "GenCC".
+- Adding "submit to SPANR" feature.
 
 Full Change List
 ================
@@ -59,7 +60,8 @@ Full Change List
 - Added celery queues ``maintenance`` and ``export``.
 - Adding support for connecting two sites via the GAGH Beacon protocol.
 - Making CADD version behind CADD REST API configurable.
-- Adding link-out to "GenCC"
+- Adding link-out to "GenCC".
+- Adding "submit to SPANR" feature.
 
 -------
 v0.22.1

config/settings/base.py

@@ -487,6 +487,9 @@ def fixed_array_type(field):
 VARFISH_MUTATIONTASTER_BATCH_VARS = env.int("VARFISH_MUTATIONTASTER_BATCH_VARS", 50)
 VARFISH_MUTATIONTASTER_MAX_VARS = env.int("VARFISH_MUTATIONTASTER_MAX_VARS", 500)
 
+# VarfFish: Enable SPANR
+VARFISH_ENABLE_SPANR_SUBMISSION = env.bool("VARFISH_ENABLE_SPANR_SUBMISSION", False)
+
 # Varfish: UMD URL
 VARFISH_UMD_REST_API_URL = env.str(
     "VARFISH_UMD_REST_API_URL", "http://umd-predictor.eu/webservice.php"

docs_manual/variants_filtration.rst

@@ -324,6 +324,13 @@ Here are the actions to create the recommended settings for submitting to Mutati
     The MutationDistiller submission uses the same feature as th VarFish VCF export.
     Thus, the limitations described in :ref:`download-as-file` apply.
 
+Submit to SPANR
+---------------
+
+Also, the little triangle next to the :guilabel:`Filter & Display` gives you access to the :guilabel:`Submit to SPANR` action.
+This is similar to submitting ot MutationDistiller described above.
+Clicking the button will submit the data to SPANR after confirming this once again in popup window.
+
 --------------------------
 Variant Filtration Results
 --------------------------

variants/admin.py

@@ -16,6 +16,7 @@
     SyncCaseResultMessage,
     ImportVariantsBgJob,
     SmallVariantSet,
+    SpanrSubmissionBgJob,
     CasePhenotypeTerms,
 )
 
@@ -37,6 +38,7 @@
         SyncCaseResultMessage,
         ImportVariantsBgJob,
         SmallVariantSet,
+        SpanrSubmissionBgJob,
         CasePhenotypeTerms,
     )
 )

variants/file_export.py

@@ -437,6 +437,7 @@ def write_tmp_file(self):
         self._write_variants()
         self._write_trailing()
         #: Rewind temporary file to beginning and return it.
+        self.tmp_file.flush()
         self.tmp_file.seek(0)
         return self.tmp_file
 

variants/forms.py

@@ -1375,6 +1375,7 @@ class FilterForm(
             ("download", "Generate downloadable file in background"),
             ("submit-mutationdistiller", "Submit to MutationDistiller"),
             ("submit-cadd", "Submit to CADD"),
+            ("submit-spanr", "Submit to SPANR"),
         )
     )
 

variants/migrations/0080_spanrsubmissionbgjob.py

@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2021-04-01 08:45
+from __future__ import unicode_literals
+import uuid
+
+import bgjobs.models
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("projectroles", "0015_fix_appsetting_constraint"),
+        ("bgjobs", "0006_auto_20200526_1657"),
+        ("variants", "0079_auto_20210204_1006"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="SpanrSubmissionBgJob",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                (
+                    "sodar_uuid",
+                    models.UUIDField(default=uuid.uuid4, help_text="Case SODAR UUID", unique=True),
+                ),
+                (
+                    "query_args",
+                    django.contrib.postgres.fields.jsonb.JSONField(
+                        help_text="(Validated) query parameters"
+                    ),
+                ),
+                (
+                    "spanr_job_url",
+                    models.CharField(help_text="The SPANR job URL", max_length=100, null=True),
+                ),
+                (
+                    "bg_job",
+                    models.ForeignKey(
+                        help_text="Background job for state etc.",
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="spanr_submission_bg_job",
+                        to="bgjobs.BackgroundJob",
+                    ),
+                ),
+                (
+                    "case",
+                    models.ForeignKey(
+                        help_text="The case to export",
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="variants.Case",
+                    ),
+                ),
+                (
+                    "project",
+                    models.ForeignKey(
+                        help_text="Project in which this objects belongs",
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="projectroles.Project",
+                    ),
+                ),
+            ],
+            bases=(bgjobs.models.JobModelMessageMixin, models.Model),
+        ),
+    ]

variants/models.py

@@ -612,6 +612,7 @@ def get_background_jobs(self):
             Q(variants_exportfilebgjob_related__case=self)
             | Q(cadd_submission_bg_job__case=self)
             | Q(distiller_submission_bg_job__case=self)
+            | Q(spanr_submission_bg_job__case=self)
             | Q(filter_bg_job__case=self)
         )
 
@@ -1228,6 +1229,43 @@ def get_absolute_url(self):
         )
 
 
+class SpanrSubmissionBgJob(JobModelMessageMixin, models.Model):
+    """Background job for submitting variants to SPANR."""
+
+    #: Task description for logging.
+    task_desc = "Submission to SPANR"
+
+    #: String identifying model in BackgroundJob.
+    spec_name = "variants.spanr_submission_bg_job"
+
+    # Fields required by SODAR
+    sodar_uuid = models.UUIDField(
+        default=uuid_object.uuid4, unique=True, help_text="Case SODAR UUID"
+    )
+    project = models.ForeignKey(Project, help_text="Project in which this objects belongs")
+
+    bg_job = models.ForeignKey(
+        BackgroundJob,
+        null=False,
+        related_name="spanr_submission_bg_job",
+        help_text="Background job for state etc.",
+        on_delete=models.CASCADE,
+    )
+    case = models.ForeignKey(Case, null=False, help_text="The case to export")
+    query_args = JSONField(null=False, help_text="(Validated) query parameters")
+
+    spanr_job_url = models.CharField(max_length=100, null=True, help_text="The SPANR job URL")
+
+    def get_human_readable_type(self):
+        return "SPANR Submission"
+
+    def get_absolute_url(self):
+        return reverse(
+            "variants:spanr-job-detail",
+            kwargs={"project": self.project.sodar_uuid, "job": self.sodar_uuid},
+        )
+
+
 class SmallVariantComment(models.Model):
     """Model for commenting on a ``SmallVariant``."""
 

variants/plugins.py

@@ -12,6 +12,7 @@
     ExportFileBgJob,
     ExportProjectCasesFileBgJob,
     CaddSubmissionBgJob,
+    SpanrSubmissionBgJob,
     DistillerSubmissionBgJob,
     ComputeProjectVariantsStatsBgJob,
     FilterBgJob,
@@ -280,6 +281,7 @@ class BackgroundJobsPlugin(BackgroundJobsPluginPoint):
     job_specs = {
         ExportFileBgJob.spec_name: ExportFileBgJob,
         CaddSubmissionBgJob.spec_name: CaddSubmissionBgJob,
+        SpanrSubmissionBgJob.spec_name: SpanrSubmissionBgJob,
         DistillerSubmissionBgJob.spec_name: DistillerSubmissionBgJob,
         ComputeProjectVariantsStatsBgJob.spec_name: ComputeProjectVariantsStatsBgJob,
         ExportProjectCasesFileBgJob.spec_name: ExportProjectCasesFileBgJob,

variants/submit_external.py

@@ -1,5 +1,5 @@
 """This module contains the code for file export"""
-
+import gzip
 import re
 
 from bs4 import BeautifulSoup
@@ -143,3 +143,117 @@ def submit_cadd(job):
         job.mark_success()
         if timeline:
             tl_event.set_status("OK", "CADD submission complete for {case_name}")
+
+
+#: URL to SPANR submission form
+SPANR_POST_URL = "http://tools.genes.toronto.edu/"
+#: Maximum number of lines to write out
+SPANR_MAX_LINES = 40
+
+
+def submit_spanr(job):
+    """Submit a case to SPANR."""
+    job.mark_start()
+    timeline = get_backend_api("timeline_backend")
+    if timeline:
+        tl_event = timeline.add_event(
+            project=job.project,
+            app_name="variants",
+            user=job.bg_job.user,
+            event_name="case_submit_spanr",
+            description="submitting {case_name} case to SPANR",
+            status_type="INIT",
+        )
+        tl_event.add_object(obj=job.case, label="case_name", name=job.case.name)
+    try:
+        job.add_log_entry("Getting submission form for CSRF (security) token")
+        text_input = _submit_spanr_make_text(job)
+        job.add_log_entry("Getting submission form for CSRF (security) token")
+        session = requests.Session()
+        csrf_token = _submit_spanr_obtain_csrf_token(job, session, timeline, tl_event)
+        if not csrf_token:
+            return  # bail out!
+        data = {"csrf_token": (None, csrf_token), "text_input": (None, text_input)}
+        job_id = _submit_spanr_post(data, job, session, timeline, tl_event)
+        if not job_id:
+            return  # bail out!
+        # Get target URL
+        job.spanr_job_url = "%sresults/%s" % (SPANR_POST_URL, job_id)
+        job.add_log_entry("SPANR job page is %s" % job.spanr_job_url)
+        job.save()
+    except Exception as e:
+        job.mark_error(e)
+        if timeline:
+            tl_event.set_status("FAILED", "SPANR submission failed for {case_name}: %s")
+        raise
+    else:
+        job.mark_success()
+        if timeline:
+            tl_event.set_status("OK", "SPANR submission complete for {case_name}")
+
+
+def _submit_spanr_post(data, job, session, timeline, tl_event):
+    job.add_log_entry("Submitting to %s..." % SPANR_POST_URL)
+    for k in ("job_name", "chrom", "pos", "variant_id", "ref", "alt"):
+        data[k] = (None, "")
+    response = session.post(
+        SPANR_POST_URL,
+        files=data,
+        headers={
+            "Referer": SPANR_POST_URL,
+            "Origin": SPANR_POST_URL[:-1],
+            "Upgrade-Insecure-Requests": "1",
+        },
+    )
+    job.add_log_entry("Done submitting to %s" % SPANR_POST_URL)
+    if not response.ok:
+        job.mark_error("HTTP status code: {}".format(response.status_code))
+        if timeline:
+            tl_event.set_status("FAILED", "SPANR submission failed for {case_name}")
+        return None
+    soup = BeautifulSoup(response.text, "html.parser")
+    job_id = None
+    for tag in soup.find_all("title"):
+        text = tag.string
+        if text.startswith("Result"):
+            job_id = text.split()[1]
+    if not job_id:
+        job.mark_error('Page title did not start with "Result"')
+        if timeline:
+            tl_event.set_status("FAILED", "SPANR submission failed for {case_name}")
+    return job_id
+
+
+def _submit_spanr_make_text(job):
+    with CaseExporterVcf(job, job.case) as exporter:
+        job.add_log_entry("Creating temporary VCF file...")
+        tmp_file = exporter.write_tmp_file()
+        job.add_log_entry("Extracting first 40 variants...")
+        lines = []
+        with gzip.open(tmp_file.name, "rt") as inputf:
+            i = 0
+            for line in inputf:
+                if not line.startswith("#"):
+                    lines.append("\t".join(line.split("\t")[:5]))
+                i += 1
+                if i >= SPANR_MAX_LINES:
+                    break
+        text_input = "\n".join(lines) + "\n"
+    return text_input
+
+
+def _submit_spanr_obtain_csrf_token(job, session, timeline, tl_event):
+    response = session.get(SPANR_POST_URL)
+    if not response.ok:
+        job.mark_error("HTTP status code: {}".format(response.status_code))
+        if timeline:
+            tl_event.set_status("FAILED", "SPANR submission failed for {case_name}")
+        return None
+    soup = BeautifulSoup(response.text, "html.parser")
+    tag = soup.find(id="csrf_token")
+    if not tag:
+        job.mark_error("Could not extract CSRF token")
+        if timeline:
+            tl_event.set_status("FAILED", "SPANR submission failed for {case_name}")
+        return None
+    return tag.attrs.get("value")

variants/tasks.py

@@ -34,6 +34,12 @@ def cadd_submission_task(_self, submission_job_pk):
     submit_external.submit_cadd(models.CaddSubmissionBgJob.objects.get(pk=submission_job_pk))
 
 
+@app.task(bind=True)
+def spanr_submission_task(_self, submission_job_pk):
+    """Task to submit a case to SPANR."""
+    submit_external.submit_spanr(models.SpanrSubmissionBgJob.objects.get(pk=submission_job_pk))
+
+
 @app.task(bind=True)
 def export_file_task(_self, export_job_pk):
     """Task to export single case to a file"""

variants/templates/variants/_filter_form.html

@@ -7,6 +7,7 @@
 {% get_django_setting 'VARFISH_ENABLE_EXOMISER_PRIORITISER' as exomiser_enabled %}
 {% get_django_setting 'VARFISH_ENABLE_CADD' as cadd_enabled %}
 {% get_django_setting 'VARFISH_ENABLE_CADD_SUBMISSION' as cadd_submission_enabled %}
+{% get_django_setting 'VARFISH_ENABLE_SPANR_SUBMISSION' as spanr_submission_enabled %}
 {% get_is_testing as is_testing %}
 
 {% if form.errors %}
@@ -151,6 +152,15 @@
                   Submit to CADD
                 </button>
               {% endif %}
+              {% if spanr_submission_enabled %}
+                <button type="button" class="dropdown-item" data-toggle="modal" data-target="#confirm-submit-spanr"
+                        data-tooltip="tooltip" aria-haspopup="true" aria-expanded="false"
+                        title="Submit to SPANR"
+                        {% if num_small_vars == 0 or not variant_set_exists %}disabled{% endif %}>
+                  <i class="fa fa-mail-forward"></i>
+                  Submit to SPANR
+                </button>
+              {% endif %}
             </div>
           </div>
         </div>
@@ -160,6 +170,7 @@
 
   {% include "variants/_distiller_resubmit_modal.html" %}
   {% include "variants/_cadd_resubmit_modal.html" %}
+  {% include "variants/_spanr_resubmit_modal.html" %}
 </form>
 
 <script type="text/javascript">