Merge branch 'dev' into karma_chromeheadless

config/datatypes_conf.xml.sample

@@ -197,7 +197,11 @@
     <datatype extension="nhdr" type="galaxy.datatypes.images:Nrrd" subclass="true"/>
     <datatype extension="rna_eps" type="galaxy.datatypes.sequence:RNADotPlotMatrix" mimetype="image/eps" display_in_upload="true"/>
     <datatype extension="zip" type="galaxy.datatypes.binary:CompressedZipArchive" display_in_upload="true"/>
-    <datatype extension="tar" type="galaxy.datatypes.binary:CompressedArchive" subclass="true" display_in_upload="true"/>
+    <datatype extension="tar" type="galaxy.datatypes.binary:CompressedArchive" subclass="true" display_in_upload="true">
+      <converter file="tar_to_directory.xml" target_datatype="directory"/>
+    </datatype>
+    <datatype extension="directory" type="galaxy.datatypes.data:Directory">
+    </datatype>
     <!-- Proteomics Datatypes -->
     <datatype extension="pepxml" type="galaxy.datatypes.proteomics:PepXml" mimetype="application/xml" display_in_upload="true"/>
     <datatype extension="raw_pepxml" type="galaxy.datatypes.proteomics:PepXml" mimetype="application/xml" subclass="true"/>

config/migrated_tools_conf.xml.sample

@@ -1,3 +1,3 @@
 <?xml version="1.0"?>
-<toolbox tool_path="../shed_tools">
-</toolbox>
+<toolbox tool_path="database/shed_tools">
+</toolbox>

config/shed_tool_conf.xml.sample

@@ -1,3 +1,3 @@
 <?xml version="1.0"?>
-<toolbox tool_path="../shed_tools">
+<toolbox tool_path="database/shed_tools">
 </toolbox>

doc/source/admin/tool_panel.rst

@@ -37,7 +37,7 @@ Sections & Labels
 .. code-block:: xml
 
     <?xml version="1.0"?>
-    <toolbox tool_path="../shed_tools">
+    <toolbox tool_path="database/shed_tools">
         <section id="mts" name="MTS" version="">
           <tool file="toolshed.g2.bx.psu.edu/repos/devteam/fastqc/a00a6402d09a/fastqc/rgFastQC.xml" guid="toolshed.g2.bx.psu.edu/repos/devteam/fastqc/fastqc/0.67" labels="new">
               <tool_shed>toolshed.g2.bx.psu.edu</tool_shed>
@@ -48,4 +48,4 @@ Sections & Labels
                 <version>0.67</version>
             </tool>
         </section>
-    </toolbox>
+    </toolbox>

lib/galaxy/datatypes/converters/tar_to_directory.xml

@@ -0,0 +1,16 @@
+<tool id="CONVERTER_tar_to_directory" name="Convert tar to directory" version="1.0.0" profile="17.05">
+    <!-- Don't use tar directly so we can verify safety of results - tar -xzf '$input1'; -->
+    <command>
+        mkdir '$output1.files_path';
+        cd '$output1.files_path';
+        python -c "from galaxy.util.compression_utils import CompressedFile; CompressedFile('$input1').extract('.');"
+    </command>
+    <inputs>
+        <param format="tar" name="input1" type="data"/>
+    </inputs>
+    <outputs>
+        <data format="directory" name="output1"/>
+    </outputs>
+    <help>
+    </help>
+</tool>

lib/galaxy/datatypes/data.py

@@ -927,6 +927,10 @@ def regex_line_dataprovider(self, dataset, **settings):
         return dataproviders.line.RegexLineDataProvider(dataset_source, **settings)
 
 
+class Directory(Data):
+    """Class representing a directory of files."""
+
+
 class GenericAsn1(Text):
     """Class for generic ASN.1 text format"""
     edam_data = "data_0849"

lib/galaxy/tools/__init__.py

@@ -138,6 +138,7 @@
     "CONVERTER_maf_to_fasta_0",
     "CONVERTER_maf_to_interval_0",
     "CONVERTER_wiggle_to_interval_0",
+    "CONVERTER_tar_to_directory",
     # Tools improperly migrated to the tool shed (devteam)
     "qualityFilter",
     "winSplitter",

lib/galaxy/tools/parameters/grouping.py

@@ -16,6 +16,7 @@
     ConfigDoesNotAllowException,
 )
 from galaxy.util import (
+    asbool,
     inflector,
     relpath,
     sanitize_for_filename
@@ -217,7 +218,7 @@ def get_composite_dataset_name(self, context):
             filenames = list()
             for composite_file in context.get('files', []):
                 if not composite_file.get('ftp_files', ''):
-                    filenames.append(composite_file.get('file_data', {}).get('filename', ''))
+                    filenames.append((composite_file.get('file_data') or {}).get('filename', ''))
                 else:
                     filenames.append(composite_file.get('ftp_files', [])[0])
             dataset_name = os.path.commonprefix(filenames).rstrip('.') or None
@@ -543,13 +544,14 @@ def get_filenames(context):
         d_type = self.get_datatype(trans, context)
         dbkey = self.get_dbkey(context)
         tag_using_filenames = context.get('tag_using_filenames', False)
+        force_composite = asbool(context.get('force_composite', 'False'))
         writable_files = d_type.writable_files
         writable_files_offset = 0
         groups_incoming = [None for _ in range(file_count)]
         for group_incoming in context.get(self.name, []):
             i = int(group_incoming['__index__'])
             groups_incoming[i] = group_incoming
-        if d_type.composite_type is not None:
+        if d_type.composite_type is not None or force_composite:
             # handle uploading of composite datatypes
             # Only one Dataset can be created
             dataset = Bunch()
@@ -594,18 +596,23 @@ def get_filenames(context):
                 dataset.warnings.extend(warnings)
             if dataset.primary_file is None:  # remove this before finish, this should create an empty dataset
                 raise Exception('No primary dataset file was available for composite upload')
-            keys = [value.name for value in writable_files.values()]
-            for i, group_incoming in enumerate(groups_incoming[writable_files_offset :]):
+            if not force_composite:
+                keys = [value.name for value in writable_files.values()]
+            else:
+                keys = [str(index) for index in range(file_count)]
+            for i, group_incoming in enumerate(groups_incoming[writable_files_offset:]):
                 key = keys[i + writable_files_offset]
-                if group_incoming is None and not writable_files[list(writable_files.keys())[keys.index(key)]].optional:
+                if not force_composite and group_incoming is None and not writable_files[list(writable_files.keys())[keys.index(key)]].optional:
                     dataset.warnings.append("A required composite file (%s) was not specified." % (key))
                     dataset.composite_files[key] = None
                 else:
                     file_bunch, warnings = get_one_filename(group_incoming)
                     dataset.warnings.extend(warnings)
                     if file_bunch.path:
+                        if force_composite:
+                            key = group_incoming.get("NAME") or i
                         dataset.composite_files[key] = file_bunch.__dict__
-                    else:
+                    elif not force_composite:
                         dataset.composite_files[key] = None
                         if not writable_files[list(writable_files.keys())[keys.index(key)]].optional:
                             dataset.warnings.append("A required composite file (%s) was not specified." % (key))

lib/galaxy/util/compression_utils.py

@@ -65,6 +65,10 @@ def get_fileobj_raw(filename, mode="r", compressed_formats=None):
 
 class CompressedFile(object):
 
+    @staticmethod
+    def can_decompress(file_path):
+        return tarfile.is_tarfile(file_path) or zipfile.is_zipfile(file_path)
+
     def __init__(self, file_path, mode='r'):
         if tarfile.is_tarfile(file_path):
             self.file_type = 'tar'

lib/galaxy/webapps/galaxy/api/datasets.py

@@ -2,6 +2,7 @@
 API operations on the contents of a history dataset.
 """
 import logging
+import os
 
 from six import string_types
 
@@ -13,6 +14,9 @@
     web
 )
 from galaxy.datatypes import dataproviders
+from galaxy.util.path import (
+    safe_walk
+)
 from galaxy.visualization.data_providers.genome import (
     BamDataProvider,
     FeatureLocationIndexDataProvider,
@@ -302,6 +306,25 @@ def _raw_data(self, trans, dataset, provider=None, **kwargs):
 
         return data
 
+    @web.expose_api_anonymous
+    def extra_files(self, trans, history_content_id, history_id, **kwd):
+        """
+        GET /api/histories/{encoded_history_id}/contents/{encoded_content_id}/extra_files
+        Generate list of extra files.
+        """
+        decoded_content_id = self.decode_id(history_content_id)
+
+        hda = self.hda_manager.get_accessible(decoded_content_id, trans.user)
+        extra_files_path = hda.extra_files_path
+        rval = []
+        for root, directories, files in safe_walk(extra_files_path):
+            for directory in directories:
+                rval.append({"class": "Directory", "path": os.path.relpath(os.path.join(root, directory), extra_files_path)})
+            for file in files:
+                rval.append({"class": "File", "path": os.path.relpath(os.path.join(root, file), extra_files_path)})
+
+        return rval
+
     @web.expose_api_raw_anonymous
     def display(self, trans, history_content_id, history_id,
                 preview=False, filename=None, to_ext=None, raw=False, **kwd):

lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py

@@ -38,8 +38,8 @@ def get_message_for_no_shed_tool_config():
     message = 'The tool_config_file setting in galaxy.ini must include at least one shed tool configuration file name with a <toolbox> '
     message += 'tag that includes a tool_path attribute value which is a directory relative to the Galaxy installation directory in order to '
     message += 'automatically install tools from a tool shed into Galaxy (e.g., the file name shed_tool_conf.xml whose <toolbox> tag is '
-    message += '<toolbox tool_path="../shed_tools">).  For details, see the "Installation of Galaxy tool shed repository tools into a local '
-    message += 'Galaxy instance" section of the Galaxy tool shed wiki at https://galaxyproject.org/installing-repositories-to-galaxy/'
+    message += '<toolbox tool_path="database/shed_tools">).  For details, see the "Installation of Galaxy tool shed repository tools into a '
+    message += 'local Galaxy instance" section of the Galaxy tool shed wiki at https://galaxyproject.org/installing-repositories-to-galaxy/'
     return message
 
 
@@ -517,8 +517,8 @@ def install_repository_revision(self, trans, payload, **kwd):
                                           (e.g., galaxy.ini).  At least one shed-related tool panel config file is required to be configured. Setting
                                           this parameter to a specific file enables you to choose where the specified repository will be installed because
                                           the tool_path attribute of the <toolbox> from the specified file is used as the installation location
-                                          (e.g., <toolbox tool_path="../shed_tools">).  If this parameter is not set, a shed-related tool panel configuration
-                                          file will be selected automatically.
+                                          (e.g., <toolbox tool_path="database/shed_tools">).  If this parameter is not set, a shed-related tool panel
+                                          configuration file will be selected automatically.
         """
         # Get the information about the repository to be installed from the payload.
         tool_shed_url, name, owner, changeset_revision = self.__parse_repository_from_payload(payload, include_changeset=True)
@@ -573,8 +573,8 @@ def install_repository_revisions(self, trans, payload, **kwd):
                                           (e.g., galaxy.ini).  At least one shed-related tool panel config file is required to be configured. Setting
                                           this parameter to a specific file enables you to choose where the specified repository will be installed because
                                           the tool_path attribute of the <toolbox> from the specified file is used as the installation location
-                                          (e.g., <toolbox tool_path="../shed_tools">).  If this parameter is not set, a shed-related tool panel configuration
-                                          file will be selected automatically.
+                                          (e.g., <toolbox tool_path="database/shed_tools">).  If this parameter is not set, a shed-related tool panel
+                                          configuration file will be selected automatically.
         """
         self.__ensure_can_install_repos(trans)
         # Get the information about all of the repositories to be installed.

lib/galaxy/webapps/galaxy/buildapp.py

@@ -223,6 +223,11 @@ def populate_api_routes(webapp, app):
                           controller="history_contents",
                           action="update_permissions",
                           conditions=dict(method=["PUT"]))
+    webapp.mapper.connect("history_contents_extra_files",
+                          "/api/histories/{history_id}/contents/{history_content_id}/extra_files",
+                          controller="datasets",
+                          action="extra_files",
+                          conditions=dict(method=["GET"]))
     webapp.mapper.connect("history_contents_metadata_file",
                           "/api/histories/{history_id}/contents/{history_content_id}/metadata_file",
                           controller="datasets",

lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py

@@ -815,7 +815,7 @@ def prepare_for_install(self, trans, **kwd):
             message += 'shed tool configuration file name with a <b>&lt;toolbox&gt;</b> tag that includes a <b>tool_path</b> '
             message += 'attribute value which is a directory relative to the Galaxy installation directory in order '
             message += 'to automatically install tools from a Galaxy Tool Shed (e.g., the file name <b>shed_tool_conf.xml</b> '
-            message += 'whose <b>&lt;toolbox&gt;</b> tag is <b>&lt;toolbox tool_path="../shed_tools"&gt;</b>).<p/>See the '
+            message += 'whose <b>&lt;toolbox&gt;</b> tag is <b>&lt;toolbox tool_path="database/shed_tools"&gt;</b>).<p/>See the '
             message += '<a href="https://galaxyproject.org/installing-repositories-to-galaxy/" target="_blank">Installation '
             message += 'of Galaxy Tool Shed repository tools into a local Galaxy instance</a> section of the Galaxy Tool '
             message += 'Shed wiki for all of the details.'

lib/tool_shed/galaxy_install/migrate/check.py

@@ -79,8 +79,8 @@ def verify_tools(app, url, galaxy_config_file=None, engine_options={}):
                     msg += "be removed from your file%s named %s.\n\n" % (plural, tool_panel_config_file_names)
                     msg += "CRITICAL NOTE IF YOU PLAN TO INSTALL\n"
                     msg += "The location in which the tool repositories will be installed is the value of the 'tool_path' attribute in the <tool>\n"
-                    msg += 'tag of the file named ./migrated_tool_conf.xml (i.e., <toolbox tool_path="../shed_tools">).  The default location\n'
-                    msg += "setting is '../shed_tools', which may be problematic for some cluster environments, so make sure to change it before\n"
+                    msg += 'tag of the file named ./migrated_tool_conf.xml (i.e., <toolbox tool_path="database/shed_tools">).  The default location\n'
+                    msg += "setting is 'database/shed_tools', which may be problematic for some cluster environments, so make sure to change it before\n"
                     msg += "you execute the installation process if appropriate.  The configured location must be outside of the Galaxy installation\n"
                     msg += "directory or it must be in a sub-directory protected by a properly configured .hgignore file if the directory is within\n"
                     msg += "the Galaxy installation directory hierarchy.  This is because tool shed repositories will be installed using mercurial's\n"

lib/tool_shed/util/common_util.py

@@ -140,7 +140,7 @@ def get_non_shed_tool_panel_configs(app):
     config_filenames = []
     for config_filename in app.config.tool_configs:
         # Any config file that includes a tool_path attribute in the root tag set like the following is shed-related.
-        # <toolbox tool_path="../shed_tools">
+        # <toolbox tool_path="database/shed_tools">
         tree, error_message = xml_util.parse_xml(config_filename)
         if tree is None:
             continue

templates/webapps/galaxy/admin/tool_sheds.mako

@@ -12,30 +12,22 @@
     ${render_msg( message, status )}
 %endif
 
-<div class="toolForm">
-    <div class="toolFormTitle">Accessible Galaxy tool sheds</div>
-    <div class="toolFormBody">
-        <div class="form-row">
-            <table class="grid">
-                <% shed_id = 0 %>
-                %for name, url in trans.app.tool_shed_registry.tool_sheds.items():
-                    <tr class="libraryTitle">
-                        <td>
-                            <div style="float: left; margin-left: 1px;" class="menubutton split popup" id="dataset-${shed_id}-popup">
-                                <a class="view-info" href="${h.url_for( controller='admin_toolshed', action='browse_tool_shed', tool_shed_url=url )}">${name|h}</a>
-                            </div>
-                            <div popupmenu="dataset-${shed_id}-popup">
-                                <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='browse_tool_shed', tool_shed_url=url )}">Browse valid repositories</a>
-                                <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='find_tools_in_tool_shed', tool_shed_url=url )}">Search for valid tools</a>
-                                <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='find_workflows_in_tool_shed', tool_shed_url=url )}">Search for workflows</a>
-                            </div>
-                        </td>
-                    </tr>
-                    <% shed_id += 1 %>
-                %endfor
-                </tr>
-            </table>
-        </div>
-        <div style="clear: both"></div>
+<div class="card">
+    <div class="card-header">Accessible Galaxy tool sheds</div>
+    <div class="card-body">
+        <% shed_id = 0 %>
+        %for name, url in trans.app.tool_shed_registry.tool_sheds.items():
+            <% margin_cls = "mt-2" if shed_id > 0 else "" %>
+            <div class="menubutton split popup ${margin_cls}" id="dataset-${shed_id}-popup">
+                <a class="view-info" href="${h.url_for( controller='admin_toolshed', action='browse_tool_shed', tool_shed_url=url )}">${name|h}</a>
+            </div>
+            <div popupmenu="dataset-${shed_id}-popup">
+                <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='browse_tool_shed', tool_shed_url=url )}">Browse valid repositories</a>
+                <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='find_tools_in_tool_shed', tool_shed_url=url )}">Search for valid tools</a>
+                <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='find_workflows_in_tool_shed', tool_shed_url=url )}">Search for workflows</a>
+            </div>
+            <br/>
+            <% shed_id += 1 %>
+        %endfor
     </div>
 </div>