Merge pull request #865 from hammerlab/nfs_bams

Don't look for BAMs on HDFS

API.md

@@ -15,7 +15,7 @@ JSON should be POSTed to <tt>/api/runs</tt> with following fields:
 <dl class="dl-horizontal">
   <dt>uri</dt>
   <dd>
-    The path on HDFS where the VCF can be found.
+    The URL where the VCF can be found.
     (e.g. <tt>/users/cycledasher/pt-123.vcf</tt>)
     This should be immutable, as CycleDash expects
     to be able to find the VCF here at any time.
@@ -30,11 +30,11 @@ JSON should be POSTed to <tt>/api/runs</tt> with following fields:
 
 <dl class="dl-horizontal">
   <dt>tumorBamUri</dt>
-  <dd>The path on HDFS of the tumor BAM on which the caller was run. The BAM must already be in the database.</dd>
+  <dd>The URL of the tumor BAM on which the caller was run. The BAM must already be in the database.</dd>
    <dt>tumorBamId</dt>
   <dd>The ID of the BAM.</dd>
    <dt>normalBamUri</dt>
-  <dd>The path on HDFS of the normalBAM on which the caller was run. The BAM must already be in the database.</dd>
+  <dd>The URL of the normalBAM on which the caller was run. The BAM must already be in the database.</dd>
    <dt>normalBamId</dt>
   <dd>The ID of the BAM.</dd>
    <dt>params</dt>
@@ -76,7 +76,7 @@ JSON should be POSTed to <tt>/api/bams</tt> with following fields:
 
 <dl class="dl-horizontal">
   <dt>uri</dt>
-  <dd>The URI of the BAM on HDFS. Should start with 'hdfs://'.</dd>
+  <dd>The URL of the BAM. Should start with 'http[s]://'.</dd>
   <dt>projectName (or projectId)</dt>
   <dd>
     The name of the project that the run related to.

cycledash/__init__.py

@@ -1,7 +1,9 @@
 from flask import Flask, jsonify, request, make_response, current_app
 import flask.json
 from flask_sqlalchemy import SQLAlchemy
-from flask.ext import restful, login, bcrypt
+import flask_restful as restful
+import flask_login as login
+import flask_bcrypt as bcrypt
 import humanize
 import logging
 import sys

cycledash/api/__init__.py

@@ -1,7 +1,7 @@
 from collections import OrderedDict
 from flask import request
-import flask.ext.restful
-from flask.ext.login import current_user
+import flask_restful
+from flask_login import current_user
 import functools
 import voluptuous
 
@@ -10,7 +10,7 @@
 from cycledash.helpers import prepare_request_data, camelcase_dict
 
 
-class Resource(flask.ext.restful.Resource, object):
+class Resource(flask_restful.Resource, object):
     """Extends Resource by adding an authentication check for basic auth or
     valid session cokie.
     """
@@ -33,7 +33,7 @@ def dispatch_request(self, *args, **kwargs):
                     authorized = True
             if not authorized:
                 auth_msg = 'Correct username/password required.'
-                return flask.ext.restful.abort(401, message=auth_msg)
+                return flask_restful.abort(401, message=auth_msg)
         return super(Resource, self).dispatch_request(*args, **kwargs)
 
 
@@ -88,12 +88,12 @@ def decorator(f):
         @functools.wraps(f)
         def wrapper(*args, **kwargs):
             if not (request.json or request.data or request.form):
-                flask.ext.restful.abort(400, message='Validation error.',
+                flask_restful.abort(400, message='Validation error.',
                                         errors=['No data provided.'])
             try:
                 data = schema(prepare_request_data(request))
             except voluptuous.MultipleInvalid as err:
-                flask.ext.restful.abort(400,
+                flask_restful.abort(400,
                                         message='Validation error.',
                                         errors=[str(e) for e in err.errors])
             setattr(request, 'validated_body', data)

cycledash/api/bams.py

@@ -1,23 +1,22 @@
 """Defines the API for BAMs."""
 from flask import request
-from flask.ext.restful import abort, fields
+from flask_restful import abort, fields
 from sqlalchemy import select, desc
 import voluptuous
 from voluptuous import Schema, Required, Any, Exclusive, Coerce
 
 from common.helpers import tables, find
-from cycledash.validations import expect_one_of, PathString, Doc
+from cycledash.validations import expect_one_of, HttpPathString, Doc
 from cycledash import db
 from cycledash.helpers import abort_if_none_for
 from cycledash.validations import Doc
-import workers.indexer
 
 import projects
 from . import Resource, marshal_with, validate_with
 
 
 CreateBam = Schema({
-    Required('uri'): PathString,
+    Required('uri'): HttpPathString,
 
     # One of `project` is required, but not supported in voluptuous, so we
     # enforce this in code. cf. https://github.com/alecthomas/voluptuous/issues/115
@@ -35,7 +34,7 @@
     'notes': unicode,
     'tissues': unicode,
     'resection_date': unicode,
-    'uri': PathString
+    'uri': HttpPathString
 })
 
 BamFields = Schema({
@@ -49,7 +48,7 @@
     Doc('normal',
         'Whether or not the sample is from normal tissue.'): Any(bool, None),
     Doc('tissues', 'Tissue type of sample.'): Any(basestring, None),
-    Doc('uri', 'The URI of the BAM on HDFS.'): PathString
+    Doc('uri', 'The URL of the BAM.'): HttpPathString
 })
 
 
@@ -65,9 +64,7 @@ def get(self):
     @validate_with(CreateBam)
     @marshal_with(BamFields)
     def post(self):
-        """Create a new BAM.
-
-        This will index the BAM index on HDFS if it's not already indexed."""
+        """Create a new BAM."""
         try:
             expect_one_of(request.validated_body, 'project_name', 'project_id')
         except voluptuous.MultipleInvalid as e:
@@ -81,7 +78,6 @@ def post(self):
             result = bams.insert(
                 request.validated_body).returning(*bams.c).execute()
             bam = dict(result.fetchone())
-        workers.indexer.index.delay(bam['id'])
         return bam, 201
 
 

cycledash/api/comments.py

@@ -1,8 +1,8 @@
 """API for user comments."""
 from collections import defaultdict
 from flask import jsonify, request
-from flask.ext.restful import abort, fields
-from flask.ext.login import current_user
+from flask_restful import abort, fields
+from flask_login import current_user
 from sqlalchemy import select, func, desc
 from voluptuous import Any, Required, Coerce, Schema
 

cycledash/api/genotypes.py

@@ -3,7 +3,7 @@
 import copy
 import json
 from flask import request
-import flask.ext.restful as restful
+import flask_restful as restful
 from plone.memoize import forever
 from sqlalchemy import (select, func, types, cast, join, outerjoin, asc, desc,
                         and_, Integer, Float, String, distinct)

cycledash/api/projects.py

@@ -1,6 +1,6 @@
 """Defines the API for Projects."""
 from flask import request, redirect, jsonify, url_for, render_template
-from flask.ext.restful import fields, abort
+from flask_restful import fields, abort
 from sqlalchemy import exc, select, func, desc
 import voluptuous
 from voluptuous import Schema, Required, Any

cycledash/api/runs.py

@@ -1,37 +1,36 @@
 import datetime
 from flask import request
-from flask.ext.restful import abort, fields
+from flask_restful import abort, fields
 from sqlalchemy import select, desc, func
 import voluptuous
 from voluptuous import Schema, Any, Required, Exclusive, Coerce
 
 from cycledash import db
 from cycledash.helpers import get_id_where, get_where, abort_if_none_for
-from cycledash.validations import Doc, expect_one_of, PathString
+from cycledash.validations import Doc, expect_one_of, FilePathString, HttpPathString
 from common.helpers import tables
 import workers.runner
 
 from . import genotypes, bams, marshal_with, validate_with, projects, Resource
 
 
 CreateRun = Schema({
-    Required('uri'): PathString,
+    Required('uri'): FilePathString,
 
     # One of `project` is required, but not supported in voluptuous, so we
     # enforce this in code. cf. https://github.com/alecthomas/voluptuous/issues/115
     Exclusive('project_id', 'project'): Coerce(int),
     Exclusive('project_name', 'project'): unicode,
 
     Exclusive('normal_bam_id', 'normal_bam'): Coerce(int),
-    Exclusive('normal_bam_uri', 'normal_bam'): PathString,
+    Exclusive('normal_bam_uri', 'normal_bam'): HttpPathString,
     Exclusive('tumor_bam_id', 'tumor_bam'): Coerce(int),
-    Exclusive('tumor_bam_uri', 'tumor_bam'): PathString,
+    Exclusive('tumor_bam_uri', 'tumor_bam'): HttpPathString,
 
     'caller_name': unicode,
     'project_id': Coerce(int),
     'tumor_dataset_id': Coerce(int),
     'normal_dataset_id': Coerce(int),
-    'truth_vcf_path': PathString,
     'is_validation': bool,
     'notes': unicode,
     'dataset': unicode,
@@ -44,9 +43,9 @@
     'caller_name': unicode,
 
     Exclusive('normal_bam_id', 'normal_bam'): Coerce(int),
-    Exclusive('normal_bam_uri', 'normal_bam'): PathString,
+    Exclusive('normal_bam_uri', 'normal_bam'): HttpPathString,
     Exclusive('tumor_bam_id', 'tumor_bam'): Coerce(int),
-    Exclusive('tumor_bam_uri', 'tumor_bam'): PathString,
+    Exclusive('tumor_bam_uri', 'tumor_bam'): HttpPathString,
 
     'notes': unicode,
     'vcf_header': unicode,
@@ -64,7 +63,7 @@
         long,
     Doc('extant_columns', 'A list of all the columns the Run has.'):
         Any(basestring, None),
-    Doc('uri', 'The HDFS or NFS URI of the VCF this run was based on.'):
+    Doc('uri', 'The URL of the VCF this run was based on.'):
         basestring,
     Doc('caller_name',
         'The name of the variant caller used to generate this Run.'):

cycledash/api/tasks.py

@@ -1,7 +1,7 @@
 """Methods for working with Celery task states."""
 from collections import defaultdict
 from sqlalchemy import select
-from flask.ext.restful import abort, fields
+from flask_restful import abort, fields
 from voluptuous import Schema, Any
 
 from common.helpers import tables

cycledash/auth.py

@@ -1,6 +1,6 @@
 """Module to manage user authentication and identification."""
 from flask import request, redirect, render_template
-from flask.ext.login import login_user, logout_user
+from flask_login import login_user, logout_user
 from sqlalchemy import exc
 import voluptuous
 import base64

cycledash/helpers.py

@@ -9,7 +9,7 @@
 from common.helpers import tables, to_epoch
 
 from flask import jsonify, request, url_for, redirect
-import flask.ext.restful, flask.ext.restful.fields
+import flask_restful, flask_restful.fields
 import voluptuous
 from werkzeug.utils import secure_filename
 
@@ -188,7 +188,7 @@ def abort_if_none_for(obj_name):
     def abort_if_none(obj, obj_id):
         """Abort request with a 404 if object is None."""
         if obj is None:
-            flask.ext.restful.abort(
+            flask_restful.abort(
                 404,
                 message='No {} with id={} found.'.format(obj_name, obj_id))
         else:

cycledash/static/js/examine/components/PileupViewer.js

@@ -55,7 +55,9 @@ var PileupViewer = React.createClass({
                 handleSelectRecord={_.noop}
                 handleOpenViewer={_.noop}
                 handleSetComment={_.noop}
-                handleDeleteComment={_.noop} />
+                handleDeleteComment={_.noop}
+                currentUser={{}}
+                handleStarGenotype={_.noop} />
     );
 
     return (
@@ -68,12 +70,9 @@ var PileupViewer = React.createClass({
       </div>
     );
   },
-  // Convert an HDFS path to a browser-accessible URL via igv-httpfs.
-  hdfsUrl: function(path) {
-    return this.props.igvHttpfsUrl + path;
-  },
   canDisplayPath: function(path) {
-    return path && path.indexOf('file://') == -1;
+    return path &&
+      (path.indexOf('http://') > -1 || path.indexOf('https://') > -1);
   },
   handleClose: function(e) {
     e.preventDefault();
@@ -95,14 +94,14 @@ var PileupViewer = React.createClass({
       name: name,
       viz: pileup.viz.variants(),
       data: pileup.formats.vcf({
-        url: this.hdfsUrl(path)
+        url: path
       })
     });
 
     var bamSource = (name, cssClass, path, chunks) => {
       var data = pileup.formats.bam({
-          url: this.hdfsUrl(path),
-          indexUrl: this.hdfsUrl(path + '.bai'),
+          url: path,
+          indexUrl: path + '.bai',
           indexChunks: chunks
         });
       return [
@@ -189,7 +188,7 @@ var PileupViewer = React.createClass({
         }
 
         var chunkPath = bamPath.replace('.bam', '.bam.bai.json');
-        $.getJSON(this.hdfsUrl(chunkPath))
+        $.getJSON(chunkPath)
           .done((chunks) => {
             this.setState(_.object([propName], [chunks]));
           }).fail((jqXHR, error, textStatus) => {
@@ -207,8 +206,6 @@ var PileupViewer = React.createClass({
   componentDidUpdate: function() {
     this.update();
   },
-  componentWillUnmount: function() {
-  },
   shouldComponentUpdate: function(nextProps, nextState) {
     return ((nextProps.isOpen != this.props.isOpen) ||
             (nextProps.selectedRecord != this.props.selectedRecord));

cycledash/static/js/runs/components/forms.js

@@ -60,7 +60,7 @@ var NewRunForm = React.createClass({
             <TextInput label='Variant Caller Name' name='callerName'
                        placeholder='Guacamole::Somatic' />
             <TextInput label='VCF Path' name='uri'
-                       placeholder='/data/somevcf.vcf'
+                       placeholder='Drag a VCF here to upload it.'
                        required={true}
                        uploadable={true} uploadPath={'/upload'} />
             <TextInput label='Tumor BAM URI' name='tumorBamUri'
@@ -100,7 +100,7 @@ var NewBAMForm = React.createClass({
                        placeholder='...' />
             <TextInput label='BAM URI' name='uri'
                        required={true}
-                       placeholder='hdfs:///data/somebam.bam' />
+                       placeholder='http://cluster.example.com/hdfs/data/somebam.bam' />
             <TextInput label='Resection Date' name='resectionDate'
                        placeholder='2015-08-14' />
             <TextInput label='Tissues' name='tissues'

cycledash/templates/about.html

@@ -32,7 +32,7 @@ <h4>Required</h4>
         <dl class="dl-horizontal">
           <dt>uri</dt>
           <dd>
-            The path on HDFS where the VCF can be found.
+            The URL where the VCF was uploaded to (returned by the /upload endpoint).
             (e.g. <tt>/users/cycledasher/pt-123.vcf</tt>)
             This should be immutable, as Cycledash expects
             to be able to find the VCF here at any time.
@@ -45,13 +45,13 @@ <h4>Required</h4>
         <h4>Optional</h4>
         <dl class="dl-horizontal">
           <dt>tumorBamUri</dt>
-          <dd>The path on HDFS of the tumor BAM on which the caller was run. The BAM must already be in the database.</dd>
+          <dd>The URL where the tumor BAM on which the caller was run can be found. The BAM must already be in the database.</dd>
 
           <dt>tumorBamId</dt>
           <dd>The ID of the BAM.</dd>
 
           <dt>normalBamUri</dt>
-          <dd>The path on HDFS of the normalBAM on which the caller was run. The BAM must already be in the database.</dd>
+          <dd>The URL where the normal BAM on which the caller was run can be found. The BAM must already be in the database.</dd>
 
           <dt>normalBamId</dt>
           <dd>The ID of the BAM.</dd>
@@ -100,7 +100,7 @@ <h3>BAMs</h3>
         <h4>Required</h4>
         <dl class="dl-horizontal">
           <dt>uri</dt>
-          <dd>The URI of the BAM on HDFS. Should start with 'hdfs://'.</dd>
+          <dd>The URI of the BAM. Should start with 'http[s]://'.</dd>
           <dt>projectName/projectId</dt>
           <dd>
             The name or ID of the project that the run related to.