Merge remote-tracking branch 'upstream/release_18.01' into usegalaxy

client/galaxy/scripts/mvc/form/form-parameters.js

@@ -224,9 +224,7 @@ export default Backbone.Model.extend({
         var self = this;
         return new SelectGenomeSpace.View({
             id: `field-${input_def.id}`,
-            onchange: function() {
-                self.app.trigger("change");
-            }
+            onchange: input_def.onchange
         });
     },
 

client/galaxy/scripts/mvc/history/history-list.js

@@ -1,11 +1,60 @@
 import _l from "utils/localization";
-/** This class renders the grid list. */
+import AjaxQueue from "utils/ajax-queue";
 import Utils from "utils/utils";
+/** This class renders the grid list. */
 import GridView from "mvc/grid/grid-view";
 import HistoryModel from "mvc/history/history-model";
 import historyCopyDialog from "mvc/history/copy-dialog";
 
 var HistoryGridView = GridView.extend({
+    initialize: function(grid_config) {
+        this.ajaxQueue = new AjaxQueue.AjaxQueue();
+        GridView.prototype.initialize.call(this, grid_config);
+    },
+
+    init_grid_elements: function() {
+        const ajaxQueue = this.ajaxQueue;
+        ajaxQueue.stop();
+        GridView.prototype.init_grid_elements.call(this);
+        const fetchDetails = $.makeArray(
+            this.$el.find(".delayed-value-datasets_by_state").map((i, el) => {
+                return () => {
+                    const historyId = $(el).data("history-id");
+                    const url = `${
+                        Galaxy.root
+                    }api/histories/${historyId}?keys=nice_size,contents_active,contents_states`;
+                    const options = {};
+                    options.url = url;
+                    options.type = "GET";
+                    options.success = req => {
+                        const contentsStates = req["contents_states"];
+                        let stateHtml = "";
+                        for (let state of ["ok", "running", "queued", "new", "error"]) {
+                            const stateCount = contentsStates[state];
+                            if (stateCount) {
+                                stateHtml += `<div class="count-box state-color-${state}" title="Datasets in ${state} state">${stateCount}</div> `;
+                            }
+                        }
+                        const contentsActive = req["contents_active"];
+                        const deleted = contentsActive["deleted"];
+                        if (deleted) {
+                            stateHtml += `<div class="count-box state-color-deleted" title="Deleted datasets">${deleted}</div> `;
+                        }
+                        const hidden = contentsActive["hidden"];
+                        if (hidden) {
+                            stateHtml += `<div class="count-box state-color-hidden" title="Hidden datasets">${hidden}</div> `;
+                        }
+                        $(`.delayed-value-datasets_by_state[data-history-id='${historyId}']`).html(stateHtml);
+                        $(`.delayed-value-disk_size[data-history-id='${historyId}']`).html(req["nice_size"]);
+                    };
+                    var xhr = jQuery.ajax(options);
+                    return xhr;
+                };
+            })
+        );
+        fetchDetails.forEach(fn => ajaxQueue.add(fn));
+        ajaxQueue.start();
+    },
     _showCopyDialog: function(id) {
         var history = new HistoryModel.History({ id: id });
         history

client/galaxy/scripts/mvc/tool/tool-genomespace.js

@@ -8,11 +8,12 @@ export default {
 
         var newWin = window.open(GS_UPLOAD_URL, "GenomeSpace File Browser", "height=360px,width=600px");
 
-        successCalBack = options["successCallback"];
         window.addEventListener(
             "message",
             e => {
-                successCalBack(e.data);
+                if (options.successCallback && e.data.destination) {
+                  options.successCallback(e.data);
+                }
             },
             false
         );

client/galaxy/scripts/mvc/ui/ui-select-genomespace.js

@@ -11,14 +11,15 @@ var View = Backbone.View.extend({
     initialize: function(options) {
         // link this
         var self = this;
+        this.options = options;
 
         // create insert new list element button
         this.browse_button = new Ui.ButtonIcon({
             title: _l("Browse"),
             icon: "fa fa-sign-in",
             tooltip: _l("Browse GenomeSpace"),
             onclick: function() {
-                self.browseGenomeSpace();
+                self.browseGenomeSpace(options);
             }
         });
 
@@ -81,10 +82,11 @@ var View = Backbone.View.extend({
     // set value
     _setValue: function(new_value) {
         if (new_value) {
-            values = new_value.split("^");
+            var values = new_value.split("^");
             this.filename_textbox.value(values[0]);
             this.token_textbox.value(values[1]);
         }
+        this.options.onchange && this.options.onchange(new_value);
     }
 });
 

client/galaxy/scripts/onload.js

@@ -179,18 +179,20 @@ $(document).ready(() => {
 
     function onloadWebhooks() {
         if (Galaxy.root !== undefined) {
-            // Load all webhooks with the type 'onload'
-            Webhooks.load({
-                type: "onload",
-                callback: function(webhooks) {
-                    webhooks.each(model => {
-                        var webhook = model.toJSON();
-                        if (webhook.activate && webhook.script) {
-                            Utils.appendScriptStyle(webhook);
-                        }
-                    });
-                }
-            });
+            if (Galaxy.config.enable_webhooks) {
+                // Load all webhooks with the type 'onload'
+                Webhooks.load({
+                    type: "onload",
+                    callback: function(webhooks) {
+                        webhooks.each(model => {
+                            var webhook = model.toJSON();
+                            if (webhook.activate && webhook.script) {
+                                Utils.appendScriptStyle(webhook);
+                            }
+                        });
+                    }
+                });
+            }
         } else {
             setTimeout(onloadWebhooks, 100);
         }

client/galaxy/style/less/base.less

@@ -1177,8 +1177,15 @@ ul.manage-table-actions li {
 }
 
 .state-color-deleted {
-    border-color: @state-deleted-border;
-    background: @state-deleted-bg;
+    border-color: darken(@state-default-border, 30%);
+    border-style: dotted;
+    background: darken(@state-default-bg, 30%);
+}
+
+.state-color-hidden {
+    border-color: @state-default-border;
+    border-style: dotted;
+    background: @state-default-bg;
 }
 
 .state-fg-new {
@@ -1797,7 +1804,7 @@ div.toolTitleNoSection
     display: none;
 }
 
-// communication channel bootstrap modal 
+// communication channel bootstrap modal
 .chat-modal {
     overflow: hidden;
 }
@@ -1812,7 +1819,7 @@ div.toolTitleNoSection
 }
 
 .close-modal {
-    float: right; 
+    float: right;
     cursor: pointer;
 }
 

client/galaxy/style/less/galaxy_variables.less

@@ -1,5 +1,5 @@
 // Additional variables that are not used by bootstrap but are used
-// in Galaxy stylesheets 
+// in Galaxy stylesheets
 
 @base-bg: @white;
 @base-text-color: @black;
@@ -29,8 +29,8 @@
 @state-running-border: #AAAA66;
 @state-running-bg: #FFFFCC;
 
-@state-deleted-border: #330066;
-@state-deleted-bg: #3399FF;
+// @state-deleted-border: #330066;
+// @state-deleted-bg: #3399FF;
 
 // Theme, expects tmp-site-config.less written by grunt
 @import "tmp-site-config.less";

client/package.json

@@ -36,7 +36,7 @@
     "build-production": "NODE_ENV=production gulp stage-libs && concurrently \"yarn run style\" \"yarn run webpack-production\" \"yarn run gulp clean && yarn run gulp-production\" && yarn run save-build-hash",
     "build-production-maps": "NODE_ENV=production gulp stage-libs && concurrently \"yarn run style\" \"yarn run webpack-production-maps\" \"yarn run gulp clean && yarn run gulp-production-maps\" && yarn run save-build-hash",
     "webpack": "webpack -d",
-    "save-build-hash": "git rev-parse HEAD > ../static/client_build_hash.txt",
+    "save-build-hash": "(git rev-parse HEAD 2>/dev/null || echo '') >../static/client_build_hash.txt",
     "webpack-watch": "webpack -d --watch",
     "webpack-production": "webpack -p",
     "webpack-production-maps": "GXY_BUILD_SOURCEMAPS=1 webpack -p",

config/datatypes_conf.xml.sample

@@ -675,6 +675,9 @@
     <datatype extension="allegro_ihaplo" type="galaxy.datatypes.tabular:Tabular" />
     <datatype extension="allegro_descent" type="galaxy.datatypes.tabular:Tabular" />
     <datatype extension="allegro_fparam" type="galaxy.datatypes.genetics:AllegroLOD" />
+    <!-- IDEAS datatypes -->
+    <datatype extension="ideaspre" type="galaxy.datatypes.genetics:IdeasPre" display_in_upload="true"/>
+    <!-- End IDEAS datatypes -->
   </registration>
   <sniffers>
     <!--

config/plugins/webhooks/demo/search/script.js

@@ -471,7 +471,6 @@ $(document).ready(function() {
                 removed_results_key = "removed_results",
                 pinned_results_key = "pinned_results",
                 class_tool_link = "tool-search-link";
-
             _.each( Galaxy.config.toolbox_in_panel, function( section ) {
                 if( section.model_class === "ToolSection" ) {
                     var all_tools = section.elems,
@@ -506,14 +505,15 @@ $(document).ready(function() {
                                                              tools_template );
                     }
                 }
-                else if( section.model_class === "Tool" || section.model_class === "DataSourceTool" ) {
-                    var attributes = section.attributes;
-                    if( item === attributes.id ) {
+                else if( section.model_class === "Tool" ) {
+                    // check if the tool is present in the search results
+                    if( _.contains( search_result, section.id ) ) {
+                        var attributes = section.attributes ? section.attributes : section;
                         if( !self.checkItemPresent( attributes.id, removed_results_key, self ) ) {
                             tool_template = tool_template + self._buildLinkTemplate( attributes.id, attributes.link,
-                                                attributes.name, attributes.description, attributes.target,
-                                                class_tool_link, self.checkItemPresent( attributes.id, pinned_results_key, self ),
-                                                attributes.version, attributes.min_width, attributes.form_style );
+                                attributes.name, attributes.description, attributes.target,
+                                class_tool_link, self.checkItemPresent( attributes.id, pinned_results_key, self ),
+                                attributes.version, attributes.min_width, attributes.form_style );
                         }
                     }
                 }

lib/galaxy/datatypes/binary.py

@@ -12,6 +12,7 @@
 import tarfile
 import tempfile
 import zipfile
+from collections import OrderedDict
 from json import dumps
 
 import h5py
@@ -234,10 +235,10 @@ def set_meta(self, dataset, overwrite=True, **kwd):
             # TODO: Reference names, lengths, read_groups and headers can become very large, truncate when necessary
             dataset.metadata.reference_names = list(bam_file.references)
             dataset.metadata.reference_lengths = list(bam_file.lengths)
-            dataset.metadata.bam_header = bam_file.header
+            dataset.metadata.bam_header = OrderedDict((k, v) for k, v in bam_file.header.items())
             dataset.metadata.read_groups = [read_group['ID'] for read_group in dataset.metadata.bam_header.get('RG', []) if 'ID' in read_group]
-            dataset.metadata.sort_order = bam_file.header.get('HD', {}).get('SO', None)
-            dataset.metadata.bam_version = bam_file.header.get('HD', {}).get('VN', None)
+            dataset.metadata.sort_order = dataset.metadata.bam_header.get('HD', {}).get('SO', None)
+            dataset.metadata.bam_version = dataset.metadata.bam_header.get('HD', {}).get('VN', None)
         except Exception:
             # Per Dan, don't log here because doing so will cause datasets that
             # fail metadata to end in the error state
@@ -383,25 +384,12 @@ def groom_dataset_content(self, file_name):
 
     def set_meta(self, dataset, overwrite=True, **kwd):
         # These metadata values are not accessible by users, always overwrite
+        super(Bam, self).set_meta(dataset=dataset, overwrite=overwrite, **kwd)
         index_file = dataset.metadata.bam_index
         if not index_file:
             index_file = dataset.metadata.spec['bam_index'].param.new_file(dataset=dataset)
         pysam.index(dataset.file_name, index_file.file_name)
         dataset.metadata.bam_index = index_file
-        # Now use pysam with BAI index to determine additional metadata
-        try:
-            bam_file = pysam.AlignmentFile(dataset.file_name, mode='rb', index_filename=index_file.file_name)
-            # TODO: Reference names, lengths, read_groups and headers can become very large, truncate when necessary
-            dataset.metadata.reference_names = list(bam_file.references)
-            dataset.metadata.reference_lengths = list(bam_file.lengths)
-            dataset.metadata.bam_header = bam_file.header
-            dataset.metadata.read_groups = [read_group['ID'] for read_group in dataset.metadata.bam_header.get('RG', []) if 'ID' in read_group]
-            dataset.metadata.sort_order = bam_file.header.get('HD', {}).get('SO', None)
-            dataset.metadata.bam_version = bam_file.header.get('HD', {}).get('VN', None)
-        except Exception:
-            # Per Dan, don't log here because doing so will cause datasets that
-            # fail metadata to end in the error state
-            pass
 
     def sniff(self, file_name):
         return super(Bam, self).sniff(file_name) and not self.dataset_content_needs_grooming(file_name)

lib/galaxy/datatypes/converters/bam_native_to_bam_converter.xml

@@ -1,4 +1,4 @@
-<tool id="CONVERTER_bam_native_to_bam" name="Convert BAM native to BAM" version="1.0.0" hidden="true">
+<tool id="CONVERTER_bam_native_to_bam" name="Convert BAM native to BAM" version="1.0.0" hidden="true" profile="18.01">
     <!--  <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
     <requirements>
         <requirement type="package" version="1.6">samtools</requirement>

lib/galaxy/datatypes/converters/sam_to_bam_native.xml

@@ -1,4 +1,4 @@
-<tool id="CONVERTER_sam_to_bam_native" name="Convert SAM to BAM native - without sorting" version="1.0.0">
+<tool id="CONVERTER_sam_to_bam_native" name="Convert SAM to BAM native - without sorting" version="1.0.0" profile="18.01">
     <!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
     <requirements>
         <requirement type="package" version="1.6">samtools</requirement>

lib/galaxy/datatypes/genetics.py

@@ -518,6 +518,73 @@ class Snptest(Rgenetics):
     file_ext = "snptest"
 
 
+class IdeasPre(Html):
+    """
+    This datatype defines the input format required by IDEAS:
+    https://academic.oup.com/nar/article/44/14/6721/2468150
+    The IDEAS preprocessor tool produces an output using this
+    format.  The extra_files_path of the primary input dataset
+    contains the following files and directories.
+    - chromosome_windows.txt (optional)
+    - chromosomes.bed (optional)
+    - IDEAS_input_config.txt
+    - compressed archived tmp directory containing a number of compressed bed files.
+    """
+
+    MetadataElement(name="base_name", desc="Base name for this dataset", default='IDEASData', readonly=True, set_in_upload=True)
+    MetadataElement(name="chrom_bed", desc="Bed file specifying window positions", default=None, readonly=True)
+    MetadataElement(name="chrom_windows", desc="Chromosome window positions", default=None, readonly=True)
+    MetadataElement(name="input_config", desc="IDEAS input config", default=None, readonly=True)
+    MetadataElement(name="tmp_archive", desc="Compressed archive of compressed bed files", default=None, readonly=True)
+
+    composite_type = 'auto_primary_file'
+    allow_datatype_change = False
+    file_ext = 'ideaspre'
+
+    def __init__(self, **kwd):
+        Html.__init__(self, **kwd)
+        self.add_composite_file('chromosome_windows.txt', description='Chromosome window positions', is_binary=False, optional=True)
+        self.add_composite_file('chromosomes.bed', description='Bed file specifying window positions', is_binary=False, optional=True)
+        self.add_composite_file('IDEAS_input_config.txt', description='IDEAS input config', is_binary=False)
+        self.add_composite_file('tmp.tar.gz', description='Compressed archive of compressed bed files', is_binary=True)
+
+    def set_meta(self, dataset, **kwd):
+        Html.set_meta(self, dataset, **kwd)
+        for fname in os.listdir(dataset.extra_files_path):
+            if fname.startswith("chromosomes"):
+                dataset.metadata.chrom_bed = os.path.join(dataset.extra_files_path, fname)
+            elif fname.startswith("chromosome_windows"):
+                dataset.metadata.chrom_windows = os.path.join(dataset.extra_files_path, fname)
+            elif fname.startswith("IDEAS_input_config"):
+                dataset.metadata.input_config = os.path.join(dataset.extra_files_path, fname)
+            elif fname.startswith("tmp"):
+                dataset.metadata.tmp_archive = os.path.join(dataset.extra_files_path, fname)
+        self.regenerate_primary_file(dataset)
+
+    def generate_primary_file(self, dataset=None):
+        rval = ['<html><head></head><body>']
+        rval.append('<h3>Files prepared for IDEAS</h3>')
+        rval.append('<ul>')
+        for composite_name, composite_file in self.get_composite_files(dataset=dataset).items():
+            fn = composite_name
+            rval.append('<li><a href="%s>%s</a></li>' % (fn, fn))
+        rval.append('</ul></body></html>\n')
+        return "\n".join(rval)
+
+    def regenerate_primary_file(self, dataset):
+        # Cannot do this until we are setting metadata.
+        rval = ['<html><head></head><body>']
+        rval.append('<h3>Files prepared for IDEAS</h3>')
+        rval.append('<ul>')
+        for fname in os.listdir(dataset.extra_files_path):
+            fn = os.path.split(fname)[-1]
+            rval.append('<li><a href="%s">%s</a></li>' % (fn, fn))
+        rval.append('</ul></body></html>')
+        with open(dataset.file_name, 'w') as f:
+            f.write("\n".join(rval))
+            f.write('\n')
+
+
 class Pheno(Tabular):
     """
     base class for pheno files

lib/galaxy/dependencies/pinned-requirements.txt

@@ -11,7 +11,7 @@ mercurial==3.7.3; python_version < '3.0'
 pycrypto==2.6.1
 uWSGI==2.0.15
 # Flexible BAM index naming is new to core pysam
-pysam>=0.13
+pysam==0.14
 
 # Install python_lzo if you want to support indexed access to lzo-compressed
 # locally cached maf files via bx-python