Mccalluc/igvjs visualization (#1349)

* Empty stub page for visualization

* IGV demo with hard-coded data

* Fix flake8 failure

* Find the button
Are the data_set ones actually in use?

* Link to visualization + pass data

* Test failed because $ not defined

* declare angular dependency
Click works, but params are not actually read.

* Better URLs?
Vague sense that we may have multiple visualizations, and it would be good to have a home for all of them.

* Visualization config passed from view
Also removed optional params from JS.

* Pull out URL lookup (still fake)

* Fetch URL referenced by Node

* Hit S3 for reference data
... but have not actually populated S3, pending #1344

* Make json on server side

* Use promises for simultaneous API calls
(API does not actually have the data we need, yet.)

* API calls made in parallel

* Helper script to populate S3

* aws s3 sync

* FAI creation

* cytoband fallback + sync at end

* Check for twoBitToFa install

* Handle 2bit + better error handling

* Stub /api/v2/nodes

* v2 URL to get file store URLs

* Use the new API
Needs testing, but otherwise ready for review.

* Correct reference files

* Script usage message
+ double braces are preferred in bash

* Fails when just one Node
I was also supplying the wrong datastructure to IGV.

deployment/genome-to-s3.sh

@@ -0,0 +1,94 @@
+ #!/bin/bash
+set -o errexit
+set -o nounset
+
+
+### Helper functions
+
+die() { echo "$@" 1>&2; exit 1; }
+warn() { echo "$@" 1>&2; }
+
+download_and_unzip() {
+  # $1 will include one parent directory.
+  BASE=`basename $1`
+  if [ -e $BASE.gz ] || [ -e $BASE ]
+    then warn "$BASE.gz or $BASE already exists: skip download"
+    else ftp ftp://hgdownload.cse.ucsc.edu/goldenPath/$GENOME/$1.gz \
+      || ftp ftp://hgdownload.cse.ucsc.edu/goldenPath/$GENOME/$1 \
+      || warn "neither $1.gz nor $1 is available" 
+  fi
+
+  if [ -e $BASE.gz ]; then
+    if [ -e $BASE ]
+      then warn "$BASE already exists: skip unzip"
+      else gunzip $BASE.gz
+    fi
+  fi
+}
+
+
+### Check for dependencies
+
+which faidx > /dev/null || die 'Install faidx:
+- "pip install pyfaidx" makes "faidx" available on command line.
+- or:
+  - download source from http://www.htslib.org/download/
+  - make and install
+  - make alias for "samtools faidx"'
+
+which twoBitToFa > /dev/null || die 'Install twoBitToFa:
+Choose the directory of your OS on http://hgdownload.soe.ucsc.edu/admin/exe/,
+download "twoBitToFa", and "chmod a+x". (Or build from source.)'
+
+which aws > /dev/null || die 'Install aws-cli'
+
+aws s3 ls > /dev/null || die 'Check aws-cli credentials'
+
+
+### Main
+
+LOCAL=/tmp/genomes
+
+mkdir -p $LOCAL
+
+if [[ $# -eq 0 ]]; then
+  die "USAGE:
+$0 GENOME1 [ GENOME2 ... ]
+Fetches reference genomes from UCSC, unzips, indexes, and uploads to S3."
+fi
+
+for GENOME in $@; do
+  echo # Blank line for readability
+  echo "Starting $GENOME..."
+  cd $LOCAL
+  mkdir -p $GENOME  
+  cd $GENOME
+
+  download_and_unzip bigZips/$GENOME.2bit
+  if [[ -e $GENOME.2bit ]]; then
+    twoBitToFa $GENOME.2bit $GENOME.fa
+  fi
+
+  download_and_unzip bigZips/$GENOME.fa
+  # Replace $GENOME.fa with upstream1000.fa to get a smaller file for testing.
+
+  if [[ -e $GENOME.fa.fai ]]
+    then warn "$GENOME.fa.fai already exists: will not regenerate"
+    else faidx $GENOME.fa > /dev/null || warn 'FAI creation failed'
+  fi
+
+  download_and_unzip database/cytoBand.txt  
+  if [[ ! -e cytoBand.txt ]]; then
+    # "Ideo" seems to be more detailed?
+    download_and_unzip database/cytoBandIdeo.txt \
+      && mv cytoBandIdeo.txt cytoBand.txt \
+      || warn "No cytoBand.txt for $GENOME"
+    # TODO: Make a mock cytoBand, rather than tracking which are not available?
+  fi
+done
+
+aws s3 sync --exclude "*.gz" --exclude "*.2bit" --region us-east-1 \
+    $LOCAL s3://data.cloud.refinery-platform.org/data/igv-reference
+
+echo 'Delete the cache to free up some disk.'
+du -h $LOCAL

refinery/config/urls.py

@@ -21,7 +21,8 @@
                       UserAuthenticationResource, InvitationResource,
                       FastQCResource, UserProfileResource)
 from core.models import DataSet, AuthenticationFormUsernameOrEmail
-from core.views import WorkflowViewSet, CustomRegistrationView, NodeGroups
+from core.views import (WorkflowViewSet, NodeViewSet,
+                        CustomRegistrationView, NodeGroups)
 from file_store.views import FileStoreItems
 from data_set_manager.views import Assays, AssaysFiles, AssaysAttributes
 from data_set_manager.api import (AttributeOrderResource, StudyResource,
@@ -48,7 +49,7 @@
 # Django REST Framework urls
 router = routers.DefaultRouter()
 router.register(r'workflows', WorkflowViewSet)
-
+router.register(r'nodes', NodeViewSet)
 
 # NG: added for tastypie URL
 v1_api = Api(api_name='v1')

refinery/core/serializers.py

@@ -1,8 +1,11 @@
+import logging
 from rest_framework import serializers
 
 from .models import Workflow, NodeGroup
 from data_set_manager.models import Node, Assay, Study
 
+logger = logging.getLogger(__name__)
+
 
 class NodeGroupSerializer(serializers.ModelSerializer):
     # Slug related field associated uuids with model
@@ -60,3 +63,11 @@ class WorkflowSerializer(serializers.HyperlinkedModelSerializer):
 
     class Meta:
         model = Workflow
+
+
+class NodeSerializer(serializers.HyperlinkedModelSerializer):
+
+    class Meta:
+        model = Node
+        fields = ['uuid', 'full_file_store_item_url']
+        lookup_field = 'uuid'

refinery/core/urls.py

@@ -69,6 +69,7 @@
         r'[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12})/$',
         'workflow_engine', name="workflow_engine"),
     url(r'^fastqc_viewer/$', 'fastqc_viewer', name='fastqc_viewer'),
+    url(r'^visualize/genome/$', 'visualize_genome', name='visualize_genome'),
     url(r'^solr/igv/$', 'solr_igv'),
     url(r'^solr/core/select/$', 'solr_core_search', name="solr_core_search"),
     url(r'^solr/(?P<core>.+)/select/$', 'solr_select', name="solr_select"),

refinery/core/views.py

@@ -40,7 +40,8 @@
     ExtendedGroup, Project, DataSet, Workflow, UserProfile, WorkflowEngine,
     Analysis, Invitation, Ontology, NodeGroup,
     CustomRegistrationProfile)
-from core.serializers import WorkflowSerializer, NodeGroupSerializer
+from core.serializers import (
+    WorkflowSerializer, NodeGroupSerializer, NodeSerializer)
 from core.utils import (get_data_sets_annotations, get_anonymous_user,
                         create_current_selection_node_group,
                         filter_nodes_uuids_in_solr, move_obj_to_front)
@@ -594,6 +595,32 @@ def analysis(request, analysis_uuid):
                               context_instance=RequestContext(request))
 
 
+def visualize_genome(request):
+    """Provide IGV.js visualization of requested species + file nodes
+
+    Looks up species by name, and data files by node_id,
+    and passes the information to IGV.js.
+    """
+    species = request.GET.get('species')
+    node_ids = request.GET.getlist('node_ids')
+
+    genome = re.search(r'\(([^)]*)\)', species).group(1)
+    # TODO: Better to pass genome id instead of parsing?
+    url_base = "https://s3.amazonaws.com/data.cloud.refinery-platform.org" \
+        + "/data/igv-reference/" + genome + "/"
+    node_ids_json = json.dumps(node_ids)
+
+    return render_to_response(
+          'core/visualize/genome.html',
+          {
+              "fasta_url": url_base + genome + ".fa",
+              "index_url": url_base + genome + ".fa.fai",
+              "cytoband_url": url_base + "cytoBand.txt",
+              "node_ids_json": node_ids_json
+          },
+          context_instance=RequestContext(request))
+
+
 def solr_core_search(request):
     """Query Solr's core index for search.
 
@@ -1010,6 +1037,16 @@ class WorkflowViewSet(viewsets.ModelViewSet):
     http_method_names = ['get']
 
 
+class NodeViewSet(viewsets.ModelViewSet):
+    """
+    API endpoint that allows Nodes to be viewed
+    """
+    queryset = Node.objects.all()
+    serializer_class = NodeSerializer
+    lookup_field = 'uuid'
+    http_method_names = ['get']
+
+
 class CustomRegistrationView(RegistrationView):
 
     def register(self, request, **cleaned_data):

refinery/data_set_manager/models.py

@@ -451,6 +451,21 @@ def get_file_store_items(self):
         return file_store.models.FileStoreItem.objects.filter(
             uuid=self.file_uuid)
 
+    def full_file_store_item_url(self):
+        try:
+            file_store_item = file_store.models.FileStoreItem.objects.get(
+                uuid=self.file_uuid
+            )
+            return core.utils.get_full_url(
+                file_store_item.get_datafile_url()
+            )
+        except (
+                file_store.models.FileStoreItem.DoesNotExist,
+                file_store.models.FileStoreItem.MultipleObjectsReturned
+        ) as e:
+            logger.error(e)
+            return None
+
 
 class Attribute(models.Model):
     # allowed attribute types

refinery/templates/core/data_set.html

@@ -653,7 +653,7 @@ <h4>Groups</h4>
     <div id="igvModal" class="modal hide fade" tabindex="-1" role="dialog" aria-labelledby="myModalLabel" aria-hidden="true">
       <div class="modal-header">
         <button type="button" class="close" data-dismiss="modal" aria-hidden="true">√ó</button>
-        <h3 id="myModalLabel">Launch IGV</h3>
+        <h3 id="myModalLabel">Launch IGV.js</h3>
       </div>
       <div class="modal-body" id="myModalBody">
       </div>

refinery/templates/core/data_set2.html

@@ -278,7 +278,7 @@ <h3>Analyses</h3>
     <div id="igvModal" class="modal hide fade" tabindex="-1" role="dialog" aria-labelledby="myModalLabel" aria-hidden="true">
       <div class="modal-header">
         <button type="button" class="close" data-dismiss="modal" aria-hidden="true">√ó</button>
-        <h3 id="myModalLabel">Launch IGV</h3>
+        <h3 id="myModalLabel">Launch IGV.js</h3>
       </div>
       <div class="modal-body" id="myModalBody">
       </div>

refinery/templates/core/visualize/genome.html

@@ -0,0 +1,71 @@
+{% extends "plain.html" %}
+
+{% block title %}
+    {{ block.super }} - Visualization
+{% endblock %}
+
+{% block head %}
+    <!-- jQuery UI CSS -->
+    <link rel="stylesheet" type="text/css"
+          href="//ajax.googleapis.com/ajax/libs/jqueryui/1.11.2/themes/smoothness/jquery-ui.css"/>
+
+    <!-- Font Awesome CSS -->
+    <link rel="stylesheet" type="text/css"
+          href="//maxcdn.bootstrapcdn.com/font-awesome/4.2.0/css/font-awesome.min.css"/>
+
+    <!-- IGV CSS -->
+    <link rel="stylesheet" type="text/css" href="//igv.org/web/release/1.0.1/igv-1.0.1.css">
+
+    <!-- jQuery JS -->
+    <script type="text/javascript" src="//ajax.googleapis.com/ajax/libs/jquery/1.11.1/jquery.min.js"></script>
+    <script type="text/javascript" src="//ajax.googleapis.com/ajax/libs/jqueryui/1.11.2/jquery-ui.min.js"></script>
+
+    <!-- IGV JS-->
+    <script type="text/javascript" src="//igv.org/web/release/1.0.1/igv-1.0.1.js"></script>
+{% endblock %}
+
+{% block content %}
+
+    <div id="igv"></div>
+
+    <script type="text/javascript">
+
+        $(document).ready(function () {
+            var node_ids = {{ node_ids_json|safe }};
+            var ajaxes = $.map(node_ids, function(node_id) {
+                return $.getJSON('/api/v2/nodes/' + node_id + '/?format=json');
+            });
+
+            // When more than one node is specified, but if only a single node
+            // is given, the console log below actually returns a 3 element array:
+            // The AJAX response object is being taken as the array, instead of as
+            // the first element of a 1-element array.
+            console.log('ajaxes: ', ajaxes); // TODO
+            $.when.apply(null, ajaxes).then(function() {
+                // The elements of ajaxes correspond
+                // to the elements of the arguments array.
+                console.log('Array from arguments: ', Array.from(arguments)); // TODO
+                var tracks = $.map(arguments, function(response){
+                    return {
+                        name: response[0].full_file_store_item_url.replace(/.*\//, ''),
+                        url: response[0].full_file_store_item_url
+                    };
+                });
+                var div = $("#igv")[0];
+                var options = {
+                    showKaryo: true, // Default is false.
+                    reference: {
+                        fastaURL: '{{ fasta_url|escapejs }}',
+                        indexURL: '{{ index_url|escapejs }}',
+                        cytobandURL: '{{ cytoband_url|escapejs }}'
+                    },
+                    tracks: tracks
+                };
+
+                igv.createBrowser(div, options);
+            });
+        });
+
+    </script>
+
+{% endblock %}

refinery/templates/plain.html

@@ -0,0 +1,33 @@
+<!DOCTYPE html>
+
+{% load humanize %}
+{% load static %}
+
+<html lang="en">
+<head>
+  <meta content="text/html">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <meta name="description" content="">
+  <meta name="author" content="">
+
+  <!-- http://www.jonathantneal.com/blog/understand-the-favicon/ -->
+  <link rel="apple-touch-icon" href="{% static "images/touchicon.png" %}">
+  <link rel="icon" href="{% static "images/favicon.png" %}">
+  <!--[if IE]><link rel="shortcut icon" href="{% static "images/favicon.ico" %}"><![endif]-->
+  <!-- IE10 Win 8 -->
+  <meta name="msapplication-TileColor" content="#FFFFFF">
+  <meta name="msapplication-TileImage" content="{% static "images/tileicon.png" %}">
+
+  <title>{{ REFINERY_INSTANCE_NAME }}{% block title %}{% endblock %}</title>
+
+  {% block head %}
+  {% endblock %}
+</head>
+
+<body>
+
+    {% block content %}
+    {% endblock %}
+
+</body>
+</html>

refinery/ui/source/js/data-set-explorer/controllers/igv.js

@@ -1,7 +1,7 @@
 'use strict';
 
 function IgvCtrl (
-  $scope, $http, $window, $log, $resource
+  $scope, $http, $window, $log, $resource, $httpParamSerializer
 ) {
   $scope.igvConfig = {
     query: null,
@@ -130,7 +130,11 @@ function IgvCtrl (
   };
 
   $scope.launchIgv = function () {
-    $window.open($scope.selectedSpecies.select.url);
+    var params = $httpParamSerializer({
+      species: $scope.selectedSpecies.select.name,
+      node_ids: $scope.igvConfig.node_selection
+    });
+    $window.open('/visualize/genome?' + params);
   };
 }
 
@@ -142,5 +146,6 @@ angular
     '$window',
     '$log',
     '$resource',
+    '$httpParamSerializer',
     IgvCtrl
   ]);

refinery/ui/source/js/data-set-nav/partials/data-set-ui-mode-visualize.html

@@ -41,7 +41,7 @@
       ng-click="launchIgv()"
       ng-disabled="speciesList.length == 0">
       <i class="fa fa-bar-chart-o"></i>
-      &nbsp;&nbsp;Launch IGV
+      &nbsp;&nbsp;Launch IGV.js
     </button>
   </div>
 </div>