Merge pull request #3528 from galaxyproject/release_17.01

Merge Release 17.01 into dev

config/galaxy.ini.sample

@@ -1227,7 +1227,7 @@ use_interactive = True
 # tools). Set this to legacy_and_local to preserve the environment for legacy
 # tools and locally managed tools (this might be useful for instance if you are
 # installing software into Galaxy's virtualenv for tool development).
-#python_environment_problem = legacy_only
+#preserve_python_environment = legacy_only
 
 # Clean up various bits of jobs left on the filesystem after completion.  These
 # bits include the job working directory, external metadata temporary files,

lib/galaxy/datatypes/sequence.py

@@ -33,6 +33,8 @@
 
 log = logging.getLogger(__name__)
 
+SNIFF_COMPRESSED_FASTQS = os.environ.get("GALAXY_ENABLE_BETA_COMPRESSED_FASTQ_SNIFFING", "0") == "1"
+
 
 class SequenceSplitLocations( data.Text ):
     """
@@ -720,9 +722,17 @@ class FastqGz ( BaseFastq, Binary ):
     """Class representing a generic compressed FASTQ sequence"""
     edam_format = "format_1930"
     file_ext = "fastq.gz"
+    compressed = True
+
+    def sniff( self, filename ):
+        """Determines whether the file is in gzip-compressed FASTQ format"""
+        if not is_gzip(filename):
+            return False
+        return BaseFastq.sniff( self, filename )
 
 
-Binary.register_sniffable_binary_format("fastq.gz", "fastq.gz", FastqGz)
+if SNIFF_COMPRESSED_FASTQS:
+    Binary.register_sniffable_binary_format("fastq.gz", "fastq.gz", FastqGz)
 
 
 class FastqSangerGz( FastqGz ):
@@ -731,42 +741,38 @@ class FastqSangerGz( FastqGz ):
     file_ext = "fastqsanger.gz"
 
 
-Binary.register_sniffable_binary_format("fastqsanger.gz", "fastqsanger.gz", FastqSangerGz)
-
-
 class FastqSolexaGz( FastqGz ):
     """Class representing a compressed FASTQ sequence ( the Solexa variant )"""
     edam_format = "format_1933"
     file_ext = "fastqsolexa.gz"
 
 
-Binary.register_sniffable_binary_format("fastqsolexa.gz", "fastqsolexa.gz", FastqSolexaGz)
-
-
 class FastqIlluminaGz( FastqGz ):
     """Class representing a compressed FASTQ sequence ( the Illumina 1.3+ variant )"""
     edam_format = "format_1931"
     file_ext = "fastqillumina.gz"
 
 
-Binary.register_sniffable_binary_format("fastqillumina.gz", "fastqillumina.gz", FastqIlluminaGz)
-
-
 class FastqCSSangerGz( FastqGz ):
     """Class representing a Color Space compressed FASTQ sequence ( e.g a SOLiD variant )"""
     file_ext = "fastqcssanger.gz"
 
 
-Binary.register_sniffable_binary_format("fastqcssanger.gz", "fastqcssanger.gz", FastqCSSangerGz)
-
-
 class FastqBz2 ( BaseFastq, Binary ):
     """Class representing a generic compressed FASTQ sequence"""
     edam_format = "format_1930"
-    file_ext = "fastq.gz"
+    file_ext = "fastq.bz2"
+    compressed = True
+
+    def sniff( self, filename ):
+        """Determine whether the file is in bzip2-compressed FASTQ format"""
+        if not is_bz2(filename):
+            return False
+        return BaseFastq.sniff( self, filename )
 
 
-Binary.register_sniffable_binary_format("fastq.gz", "fastq.gz", FastqGz)
+if SNIFF_COMPRESSED_FASTQS:
+    Binary.register_sniffable_binary_format("fastq.bz2", "fastq.bz2", FastqBz2)
 
 
 class FastqSangerBz2( FastqBz2 ):
@@ -775,35 +781,23 @@ class FastqSangerBz2( FastqBz2 ):
     file_ext = "fastqsanger.bz2"
 
 
-Binary.register_sniffable_binary_format("fastqsanger.bz2", "fastqsanger.bz2", FastqSangerBz2)
-
-
 class FastqSolexaBz2( FastqBz2 ):
     """Class representing a compressed FASTQ sequence ( the Solexa variant )"""
     edam_format = "format_1933"
     file_ext = "fastqsolexa.bz2"
 
 
-Binary.register_sniffable_binary_format("fastqsolexa.bz2", "fastqsolexa.bz2", FastqSolexaBz2)
-
-
 class FastqIlluminaBz2( FastqBz2 ):
     """Class representing a compressed FASTQ sequence ( the Illumina 1.3+ variant )"""
     edam_format = "format_1931"
     file_ext = "fastqillumina.bz2"
 
 
-Binary.register_sniffable_binary_format("fastqillumina.bz2", "fastqillumina.bz2", FastqIlluminaBz2)
-
-
 class FastqCSSangerBz2( FastqBz2 ):
     """Class representing a Color Space compressed FASTQ sequence ( e.g a SOLiD variant )"""
     file_ext = "fastqcssanger.bz2"
 
 
-Binary.register_sniffable_binary_format("fastqcssanger.bz2", "fastqcssanger.bz2", FastqCSSangerBz2)
-
-
 class Maf( Alignment ):
     """Class describing a Maf alignment"""
     edam_format = "format_3008"

lib/galaxy/tools/deps/resolvers/__init__.py

@@ -92,20 +92,16 @@ def _mapping_file_to_list(mapping_file):
 
     def _expand_mappings(self, requirement):
         for mapping in self._mappings:
-            if requirement.name == mapping.from_name:
-                if mapping.from_version is not None and mapping.from_version != requirement.version:
-                    continue
-
-                requirement = requirement.copy()
-                requirement.name = mapping.to_name
-                if mapping.to_version is not None:
-                    requirement.version = mapping.to_version
-
+            if mapping.matches_requirement(requirement):
+                requirement = mapping.apply(requirement)
                 break
 
         return requirement
 
 
+FROM_UNVERSIONED = object()
+
+
 class RequirementMapping(object):
 
     def __init__(self, from_name, from_version, to_name, to_version):
@@ -114,12 +110,47 @@ def __init__(self, from_name, from_version, to_name, to_version):
         self.to_name = to_name
         self.to_version = to_version
 
+    def matches_requirement(self, requirement):
+        """Check if supplied ToolRequirement matches this mapping description.
+
+        For it to match - the names must match. Additionally if the
+        requirement is created with a version or with unversioned being set to
+        True additional checks are needed. If a version is specified, it must
+        match the supplied version exactly. If ``unversioned`` is True, then
+        the supplied requirement must be unversioned (i.e. its version must be
+        set to ``None``).
+        """
+
+        if requirement.name != self.from_name:
+            return False
+        elif self.from_version is None:
+            return True
+        elif self.from_version is FROM_UNVERSIONED:
+            return requirement.version is None
+        else:
+            return requirement.version == self.from_version
+
+    def apply(self, requirement):
+        requirement = requirement.copy()
+        requirement.name = self.to_name
+        if self.to_version is not None:
+            requirement.version = self.to_version
+        return requirement
+
     @staticmethod
     def from_dict(raw_mapping):
         from_raw = raw_mapping.get("from")
         if isinstance(from_raw, dict):
             from_name = from_raw.get("name")
-            from_version = str(from_raw.get("version"))
+            raw_version = from_raw.get("version", None)
+            unversioned = from_raw.get("unversioned", False)
+            if unversioned and raw_version:
+                raise Exception("Cannot define both version and set unversioned to True.")
+
+            if unversioned:
+                from_version = FROM_UNVERSIONED
+            else:
+                from_version = str(raw_version) if raw_version is not None else raw_version
         else:
             from_name = from_raw
             from_version = None

lib/galaxy/tools/deps/resolvers/conda.py

@@ -8,6 +8,7 @@
 import re
 
 import galaxy.tools.deps.installable
+import galaxy.tools.deps.requirements
 
 from ..conda_util import (
     build_isolated_environment,
@@ -141,15 +142,12 @@ def resolve_all(self, requirements, **kwds):
             if requirement.type != "package":
                 return False
 
-        conda_targets = []
-        for requirement in requirements:
-            requirement = self._expand_requirement(requirement)
-
-            version = requirement.version
-            if self.versionless:
-                version = None
-
-            conda_targets.append(CondaTarget(requirement.name, version=version))
+        ToolRequirements = galaxy.tools.deps.requirements.ToolRequirements
+        expanded_requirements = ToolRequirements([self._expand_requirement(r) for r in requirements])
+        if self.versionless:
+            conda_targets = [CondaTarget(r.name, version=None) for r in expanded_requirements]
+        else:
+            conda_targets = [CondaTarget(r.name, version=r.version) for r in expanded_requirements]
 
         preserve_python_environment = kwds.get("preserve_python_environment", False)
 

lib/galaxy/tools/deps/resolvers/default_conda_mapping.yml

@@ -2,4 +2,46 @@
   to: r-base
 - from: blast+
   to: blast
+- from:
+    name: samtools
+    unversioned: true
+  to:
+    name: samtools
+    version: 1.3.1
+- from:
+    name: ucsc_tools
+    unversioned: true
+  to:
+    name: ucsc_tools
+    version: 332
+- from:
+    name: bedtools
+    unversioned: true
+  to:
+    name: bedtools
+    version: 2.26.0gx
+- from:
+    name: gnuplot
+    unversioned: true
+  to:
+    name: gnuplot
+    version: 4.6
+- from:
+    name: openbabel
+    unversioned: true
+  to:
+    name: openbabel
+    version: 2.3.2
+- from:
+    name: gnuplot-py
+    unversioned: true
+  to:
+    name: gnuplot-py
+    version: 1.8
+- from:
+    name: bowtie
+    unversioned: true
+  to:
+    name: bowtie
+    version: 1.2.0
 

lib/galaxy/tools/parameters/basic.py

@@ -1795,9 +1795,15 @@ def append( list, id, hid, name, src, keep=False ):
                 m = match.hda
                 has_matched = has_matched or visible_hda == m or visible_hda == hda
                 m_name = '%s (as %s)' % ( match.original_hda.name, match.target_ext ) if match.implicit_conversion else m.name
-                append( d[ 'options' ][ 'hda' ], m.id, m.hid, m_name if m.visible else '(hidden) %s' % m_name, 'hda' )
-        if not has_matched and hasattr( visible_hda, 'id' ) and hasattr( visible_hda, 'hid' ) and hasattr( visible_hda, 'name' ):
-            append( d[ 'options' ][ 'hda' ], visible_hda.id, visible_hda.hid, '(unavailable) %s' % visible_hda.name, 'hda', True )
+                append( d[ 'options' ][ 'hda' ], m.id, m.hid, m_name, 'hda' )
+        if not has_matched and hasattr( visible_hda, 'hid' ):
+            if visible_hda.deleted:
+                hda_state = 'deleted'
+            elif not visible_hda.visible:
+                hda_state = 'hidden'
+            else:
+                hda_state = 'unavailable'
+            append( d[ 'options' ][ 'hda' ], visible_hda.id, visible_hda.hid, '(%s) %s' % ( hda_state, visible_hda.name ), 'hda', True )
 
         # add dataset collections
         dataset_collection_matcher = DatasetCollectionMatcher( dataset_matcher )

test/base/driver_util.py

@@ -173,6 +173,7 @@ def setup_galaxy_config(
         api_allow_run_as='test@bx.psu.edu',
         auto_configure_logging=logging_config_file is None,
         check_migrate_tools=False,
+        conda_auto_init=False,
         cleanup_job='onsuccess',
         data_manager_config_file=data_manager_config_file,
         enable_beta_tool_formats=True,

test/unit/tools/test_data_parameters.py

@@ -54,12 +54,22 @@ def test_field_filter_on_types( self ):
     def test_field_display_hidden_hdas_only_if_selected( self ):
         hda1 = MockHistoryDatasetAssociation( name="hda1", id=1 )
         hda2 = MockHistoryDatasetAssociation( name="hda2", id=2 )
-        self.stub_active_datasets( hda1, hda2 )
         hda1.visible = False
         hda2.visible = False
+        self.stub_active_datasets( hda1, hda2 )
+        field = self._simple_field( other_values={ "data2" : hda2 } )
+        assert len( field[ 'options' ][ 'hda' ] ) == 1  # hda1 not an option, not visible or selected
+        assert field[ 'options' ][ 'hda' ][ 0 ][ 'name' ] == "(hidden) hda2"
+
+    def test_field_display_deleted_hdas_only_if_selected( self ):
+        hda1 = MockHistoryDatasetAssociation( name="hda1", id=1 )
+        hda2 = MockHistoryDatasetAssociation( name="hda2", id=2 )
+        hda1.visible = False
+        hda2.deleted = True
+        self.stub_active_datasets( hda1, hda2 )
         field = self._simple_field( other_values={ "data2" : hda2 } )
-        self.assertEquals( len( field[ 'options' ][ 'hda' ] ), 1 )  # hda1 not an option, not visible or selected
-        assert field[ 'options' ][ 'hda' ][ 0 ][ 'name' ] == "(unavailable) hda2"
+        assert len( field[ 'options' ][ 'hda' ] ) == 1  # hda1 not an option, not visible or selected
+        assert field[ 'options' ][ 'hda' ][ 0 ][ 'name' ] == "(deleted) hda2"
 
     def test_field_implicit_conversion_new( self ):
         hda1 = MockHistoryDatasetAssociation( name="hda1", id=1 )
@@ -153,7 +163,7 @@ def setUp( self ):
         self._param = None
 
     def stub_active_datasets( self, *hdas ):
-        self.test_history._active_datasets_children_and_roles = hdas
+        self.test_history._active_datasets_children_and_roles = [ h for h in hdas if not h.deleted ]
 
     def _simple_field( self, **kwds ):
         return self.param.to_dict( trans=self.trans, **kwds )

test/unit/tools/test_tool_deps.py

@@ -194,6 +194,34 @@ def test_module_resolver_with_mapping():
         assert module.module_version == "2.24", module.module_version
 
 
+def test_module_resolver_with_mapping_specificity_rules():
+    # If a requirement demands a specific version,
+    # do not map to a different version when the version
+    # has not been specified explicitly
+    with __test_base_path() as temp_directory:
+        module_script = _setup_module_command(temp_directory, '''
+-------------------------- /soft/modules/modulefiles ---------------------------
+blast/2.24
+''')
+        mapping_file = os.path.join(temp_directory, "mapping")
+        with open(mapping_file, "w") as f:
+            f.write('''
+- from:
+    name: blast
+    unversioned: true
+  to:
+    name: blast
+    version: 2.24
+''')
+
+        resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script, mapping_files=mapping_file)
+        module = resolver.resolve( ToolRequirement( name="blast", type="package" ) )
+        assert module.module_name == "blast"
+        assert module.module_version == "2.24", module.module_version  # successful resolution, because Requirement does not ask for a specific version
+        module = resolver.resolve( ToolRequirement( name="blast", version="2.22", type="package" ) )
+        assert isinstance(module, NullDependency)  # NullDependency, because we don't map `version: Null` over a Requirement that asks for a specific version
+
+
 def test_module_resolver_with_mapping_versions():
     with __test_base_path() as temp_directory:
         module_script = _setup_module_command(temp_directory, '''