Merge branch 'dev' into remove_samples_000

client/galaxy/scripts/mvc/dataset/dataset-li-edit.js

@@ -432,7 +432,7 @@ define(
                     ) {
                         var editableDbkey = $('<a class="value">?</a>')
                             .attr("href", this.model.urls.edit)
-                            .attr("target", "top");
+                            .attr("target", "_top");
                         $details
                             .find(".dbkey .value")
                             .replaceWith(editableDbkey);
@@ -508,7 +508,7 @@ define(
                         _l(
                             "An error occurred setting the metadata for this dataset"
                         ),
-                        '<br /><a href="<%- dataset.urls.edit %>" target="top">',
+                        '<br /><a href="<%- dataset.urls.edit %>" target="_top">',
                         _l("Set it manually or retry auto-detection"),
                         "</a>",
                         "</div>",

client/galaxy/scripts/mvc/form/form-parameters.js

@@ -260,9 +260,5 @@ define(
                 });
             }
         });
-
-        return {
-            View: View
-        };
     }
 );

config/datatypes_conf.xml.sample

@@ -489,6 +489,9 @@
     <datatype extension="score" type="galaxy.datatypes.data:Text" subclass="true"/>
     <datatype extension="srs" type="galaxy.datatypes.data:Text" subclass="true"/>
     <datatype extension="srspair" type="galaxy.datatypes.data:Text" subclass="true"/>
+    <!-- Annotation Datatypes -->
+    <datatype extension="snaphmm" type="galaxy.datatypes.annotation:SnapHmm" display_in_upload="true" />
+    <datatype extension="augustus" type="galaxy.datatypes.annotation:Augustus" display_in_upload="true" />
     <!-- MSA Datatypes -->
     <datatype extension="hmm2" type="galaxy.datatypes.msa:Hmmer2" display_in_upload="true" />
     <datatype extension="hmm3" type="galaxy.datatypes.msa:Hmmer3" display_in_upload="true" />
@@ -681,6 +684,7 @@
     <sniffer type="galaxy.datatypes.binary:Fast5ArchiveBz2" />
     <sniffer type="galaxy.datatypes.binary:Fast5Archive" />
     <sniffer type="galaxy.datatypes.binary:PostgresqlArchive"/>
+    <sniffer type="galaxy.datatypes.annotation:Augustus" />
     <sniffer type="galaxy.datatypes.triples:Rdf"/>
     <sniffer type="galaxy.datatypes.blast:BlastXml"/>
     <sniffer type="galaxy.datatypes.xml:Phyloxml"/>
@@ -753,6 +757,7 @@
     <sniffer type="galaxy.datatypes.msa:Hmmer3" />
     <sniffer type="galaxy.datatypes.msa:Stockholm_1_0" />
     <sniffer type="galaxy.datatypes.msa:MauveXmfa" />
+    <sniffer type="galaxy.datatypes.annotation:SnapHmm" />
     <sniffer type="galaxy.datatypes.binary:Cel" />
     <sniffer type="galaxy.datatypes.binary:RData" />
     <sniffer type="galaxy.datatypes.images:Jpg"/>

config/plugins/webhooks/demo/tour_generator/static/script.js

@@ -1,5 +1,5 @@
 $(document).ready(function() {
-    require(['libs/toastr'], function(Toastr){
+    require(['libs/toastr', 'mvc/tours'], function(Toastr, Tours) {
         window.TourGenerator = Backbone.View.extend({
             initialize: function(options) {
                 var me = this;

doc/source/slideshow/architecture/galaxy_architecture.md

@@ -565,7 +565,7 @@ All in `config/plugins/visualizations`:
 
 - `csg` - Chemical structure viewer
 - `graphviz` - Visualize graph data using [cytoscape.js](http://www.cytoscape.org/)
-- `charts` - A more elobrate builds on more Galaxy abstractions.
+- `charts` - Classic charts as well as some integrated BioJS visualizations
 - `trackster` - Genome browser, deeply tied to Galaxy internals.
 
 ---
@@ -585,7 +585,7 @@ Data providers process data before sending to browser - slice, filter, reformat,
 
 ### Interactive Environments
 
-Similar to vizualizations: config and template
+Similar to visualizations: config and template
 
 Within the base template, launch a Docker container running a web accessible
 process
@@ -1149,7 +1149,7 @@ galaxy.tools.special_tools DEBUG 2016-06-23 19:13:38,108 Loaded history export t
 
 class: reduce70
 
-### Vizualization Plugins
+### Visualization Plugins
 
 .code[```
 galaxy.web.base.pluginframework INFO 2016-06-23 19:13:38,109 VisualizationsRegistry, loaded plugin: charts

lib/galaxy/datatypes/annotation.py

@@ -0,0 +1,87 @@
+import logging
+import tarfile
+
+from galaxy.datatypes.binary import Binary, CompressedArchive
+from galaxy.datatypes.data import get_file_peek, Text
+from galaxy.util import nice_size
+
+log = logging.getLogger(__name__)
+
+
+class SnapHmm(Text):
+    file_ext = "snaphmm"
+    edam_data = "data_1364"
+
+    def set_peek(self, dataset, is_multi_byte=False):
+        if not dataset.dataset.purged:
+            dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
+            dataset.blurb = "SNAP HMM model"
+        else:
+            dataset.peek = 'file does not exist'
+            dataset.blurb = 'file purged from disc'
+
+    def display_peek(self, dataset):
+        try:
+            return dataset.peek
+        except:
+            return "SNAP HMM model (%s)" % (nice_size(dataset.get_size()))
+
+    def sniff(self, filename):
+        """
+        SNAP model files start with zoeHMM
+        """
+        with open(filename, 'r') as handle:
+            return handle.read(6) == 'zoeHMM'
+        return False
+
+
+class Augustus(CompressedArchive):
+    """
+        Class describing an Augustus prediction model
+    """
+    file_ext = "augustus"
+    edam_data = "data_0950"
+    compressed = True
+
+    def set_peek(self, dataset, is_multi_byte=False):
+        if not dataset.dataset.purged:
+            dataset.peek = "Augustus model"
+            dataset.blurb = nice_size(dataset.get_size())
+        else:
+            dataset.peek = 'file does not exist'
+            dataset.blurb = 'file purged from disk'
+
+    def display_peek(self, dataset):
+        try:
+            return dataset.peek
+        except:
+            return "Augustus model (%s)" % (nice_size(dataset.get_size()))
+
+    def sniff(self, filename):
+        """
+        Augustus archives always contain the same files
+        """
+        try:
+            if filename and tarfile.is_tarfile(filename):
+                with tarfile.open(filename, 'r') as temptar:
+                    for f in temptar:
+                        if not f.isfile():
+                            continue
+                        if f.name.endswith('_exon_probs.pbl') \
+                           or f.name.endswith('_igenic_probs.pbl') \
+                           or f.name.endswith('_intron_probs.pbl') \
+                           or f.name.endswith('_metapars.cfg') \
+                           or f.name.endswith('_metapars.utr.cfg') \
+                           or f.name.endswith('_parameters.cfg') \
+                           or f.name.endswith('_parameters.cgp.cfg') \
+                           or f.name.endswith('_utr_probs.pbl') \
+                           or f.name.endswith('_weightmatrix.txt'):
+                            return True
+                        else:
+                            return False
+        except Exception as e:
+            log.warning('%s, sniff Exception: %s', self, e)
+        return False
+
+
+Binary.register_sniffable_binary_format("augustus", "augustus", Augustus)

lib/galaxy/datatypes/converters/sam_to_bam.py

@@ -65,8 +65,10 @@ def __main__():
     # convert to SAM
     unsorted_bam_filename = os.path.join(tmp_dir, 'unsorted.bam')
     unsorted_stderr_filename = os.path.join(tmp_dir, 'unsorted.stderr')
-    cmd = "samtools view -bS '%s' > '%s'" % (input_filename, unsorted_bam_filename)
-    proc = subprocess.Popen(args=cmd, stderr=open(unsorted_stderr_filename, 'wb'), shell=True, cwd=tmp_dir)
+    proc = subprocess.Popen(['samtools', 'view', '-bS', input_filename],
+                            stdout=open(unsorted_bam_filename, 'wb'),
+                            stderr=open(unsorted_stderr_filename, 'wb'),
+                            cwd=tmp_dir)
     return_code = proc.wait()
     if return_code:
         stderr_target = sys.stderr
@@ -87,10 +89,13 @@ def __main__():
     # samtools changed sort command arguments (starting from version 1.3)
     samtools_version = LooseVersion(_get_samtools_version())
     if samtools_version < LooseVersion('1.0'):
-        cmd = "samtools sort -o '%s' '%s' > '%s'" % (unsorted_bam_filename, sorting_prefix, output_filename)
+        sort_args = ['-o', unsorted_bam_filename, sorting_prefix]
     else:
-        cmd = "samtools sort -T '%s' '%s' > '%s'" % (sorting_prefix, unsorted_bam_filename, output_filename)
-    proc = subprocess.Popen(args=cmd, stderr=open(sorted_stderr_filename, 'wb'), shell=True, cwd=tmp_dir)
+        sort_args = ['-T', sorting_prefix, unsorted_bam_filename]
+    proc = subprocess.Popen(['samtools', 'sort'] + sort_args,
+                            stdout=open(output_filename, 'wb'),
+                            stderr=open(sorted_stderr_filename, 'wb'),
+                            cwd=tmp_dir)
     return_code = proc.wait()
 
     if return_code:

lib/galaxy/jobs/handler.py

@@ -94,6 +94,19 @@ def job_pair_for_id(self, id):
         job = self.sa_session.query(model.Job).get(id)
         return job, self.job_wrapper(job, use_persisted_destination=True)
 
+    def __write_registry_file_if_absent(self, job):
+        # TODO: remove this and the one place it is called in late 2018, this
+        # hack attempts to minimize the job failures due to upgrades from 17.05
+        # Galaxies.
+        job_wrapper = self.job_wrapper(job)
+        cwd = job_wrapper.working_directory
+        datatypes_config = os.path.join(cwd, "registry.xml")
+        if not os.path.exists(datatypes_config):
+            try:
+                self.app.datatypes_registry.to_xml_file(path=datatypes_config)
+            except OSError:
+                pass
+
     def __check_jobs_at_startup(self):
         """
         Checks all jobs that are in the 'new', 'queued' or 'running' state in
@@ -121,6 +134,7 @@ def __check_jobs_at_startup(self):
                         (model.Job.handler == self.app.config.server_name)).all()
 
         for job in jobs_at_startup:
+            self.__write_registry_file_if_absent(job)
             if not self.app.toolbox.has_tool(job.tool_id, job.tool_version, exact=True):
                 log.warning("(%s) Tool '%s' removed from tool config, unable to recover job" % (job.id, job.tool_id))
                 self.job_wrapper(job).fail('This tool was disabled before the job completed.  Please contact your Galaxy administrator.')

lib/galaxy_ext/metadata/set_metadata.py

@@ -81,6 +81,16 @@ def set_metadata():
 
     # Set up datatypes registry
     datatypes_config = sys.argv.pop(1)
+    if not os.path.exists(datatypes_config):
+        # This path should exist, except for jobs that started running on release 17.05, where a global
+        # datatypes_config (instead of a datatypes_config per job) was used. For a while release 17.05
+        # would remove the global datatypes config on shutdown and toolbox reload, which would lead to
+        # failed metadata jobs. To remedy this we scan jobs at startup for missing registry.xml files,
+        # and if we detect such a job we write out the current registry.xml file.
+        datatypes_config = os.path.join(tool_job_working_directory, "registry.xml")
+        if not os.path.exists(datatypes_config):
+            print("Metadata setting failed because registry.xml could not be found. You may retry setting metadata.")
+            sys.exit(1)
     import galaxy.datatypes.registry
     datatypes_registry = galaxy.datatypes.registry.Registry()
     datatypes_registry.load_datatypes(root_dir=galaxy_root, config=datatypes_config)