Merge pull request #42 from jmchilton/update_1705

Update to latest dev (up version to 17.05).

HISTORY.rst

@@ -6,10 +6,11 @@ History
 .. to_doc
 
 ---------------------
-17.1.3.dev0
+17.5.0.dev0
 ---------------------
 
-
+* Bring in the last of the Galaxy dev changes.
+* Allow Conda installs to target global Conda config (for Planemo)
 
 ---------------------
 17.1.2 (2017-01-23)

galaxy/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-__version__ = '17.1.3.dev0'
+__version__ = '17.5.0.dev0'
 
 PROJECT_NAME = "galaxy-lib"
 PROJECT_OWNER = PROJECT_USERAME = "galaxyproject"

galaxy/exceptions/__init__.py

@@ -99,6 +99,11 @@ class ToolMetaParameterException( MessageException ):
     err_code = error_codes.USER_TOOL_META_PARAMETER_PROBLEM
 
 
+class ToolMissingException( MessageException ):
+    status_code = 400
+    err_code = error_codes.USER_TOOL_MISSING_PROBLEM
+
+
 class RequestParameterInvalidException( MessageException ):
     status_code = 400
     err_code = error_codes.USER_REQUEST_INVALID_PARAMETER

galaxy/exceptions/error_codes.json

@@ -69,6 +69,11 @@
     "code": 400013,
     "message": "The contents of the request are malformed."
     },
+   {
+    "name": "USER_TOOL_MISSING_PROBLEM",
+    "code": 400014,
+    "message": "Tool could not be found."
+   },
    {
     "name": "USER_AUTHENTICATION_FAILED",
     "code": 401001,

galaxy/jobs/metrics/instrumenters/__init__.py

@@ -3,19 +3,21 @@
 These are responsible for collecting and formatting a coherent set of metrics.
 """
 import os.path
+from abc import (
+    ABCMeta,
+    abstractmethod
+)
 
-from abc import ABCMeta
-from abc import abstractmethod
+import six
 
 from ...metrics import formatting
 
-
 INSTRUMENT_FILE_PREFIX = "__instrument"
 
 
+@six.add_metaclass(ABCMeta)
 class InstrumentPlugin( object ):
     """Describes how to instrument job scripts and retrieve collected metrics."""
-    __metaclass__ = ABCMeta
     formatter = formatting.JobMetricFormatter()
 
     @property

galaxy/tools/deps/__init__.py

@@ -36,7 +36,6 @@ def build_dependency_manager( config ):
             'app_config': config,
         }
         if getattr(config, "use_cached_dependency_manager", False):
-            dependency_manager_kwds['tool_dependency_cache_dir'] = config.tool_dependency_cache_dir
             dependency_manager = CachedDependencyManager(**dependency_manager_kwds)
         else:
             dependency_manager = DependencyManager( **dependency_manager_kwds )
@@ -92,15 +91,20 @@ def get_resolver_option(self, resolver, key, explicit_resolver_options={}):
         global_key = "%s_%s" % (config_prefix, key)
         value = explicit_resolver_options.get(key, CONFIG_VAL_NOT_FOUND)
         if value is CONFIG_VAL_NOT_FOUND:
-            if isinstance(self.__app_config, dict):
-                value = self.__app_config.get(global_key, CONFIG_VAL_NOT_FOUND)
-            else:
-                value = getattr(self.__app_config, global_key, CONFIG_VAL_NOT_FOUND)
+            value = self.get_app_option(global_key, default)
+
+        return value
+
+    def get_app_option(self, key, default=None):
+        value = CONFIG_VAL_NOT_FOUND
+        if isinstance(self.__app_config, dict):
+            value = self.__app_config.get(key, CONFIG_VAL_NOT_FOUND)
+        else:
+            value = getattr(self.__app_config, key, CONFIG_VAL_NOT_FOUND)
         if value is CONFIG_VAL_NOT_FOUND and hasattr(self.__app_config, "config_dict"):
-            value = self.__app_config.config_dict.get(global_key, CONFIG_VAL_NOT_FOUND)
+            value = self.__app_config.config_dict.get(key, CONFIG_VAL_NOT_FOUND)
         if value is CONFIG_VAL_NOT_FOUND:
             value = default
-
         return value
 
     def dependency_shell_commands( self, requirements, **kwds ):
@@ -210,8 +214,9 @@ def __resolvers_dict( self ):
 
 
 class CachedDependencyManager(DependencyManager):
-    def __init__(self, default_base_path, conf_file=None, app_config={}):
+    def __init__(self, default_base_path, conf_file=None, app_config={}, tool_dependency_cache_dir=None):
         super(CachedDependencyManager, self).__init__(default_base_path=default_base_path, conf_file=conf_file, app_config=app_config)
+        self.tool_dependency_cache_dir = self.get_app_option("tool_dependency_cache_dir")
 
     def build_cache(self, requirements, **kwds):
         resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds)
@@ -240,7 +245,7 @@ def dependency_shell_commands( self, requirements, **kwds ):
         resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds)
         cacheable_dependencies = [dep for dep in resolved_dependencies.values() if dep.cacheable]
         hashed_dependencies_dir = self.get_hashed_dependencies_path(cacheable_dependencies)
-        if not os.path.exists(hashed_dependencies_dir) and self.__app_config.getattr("precache_dependencies", False):
+        if not os.path.exists(hashed_dependencies_dir) and self.get_app_option("precache_dependencies", False):
             # Cache not present, try to create it
             self.build_cache(requirements, **kwds)
         if os.path.exists(hashed_dependencies_dir):
@@ -265,4 +270,4 @@ def get_hashed_dependencies_path(self, resolved_dependencies):
         :rtype: str
         """
         req_hashes = self.hash_dependencies(resolved_dependencies)
-        return os.path.abspath(os.path.join(self.extra_config['tool_dependency_cache_dir'], req_hashes))
+        return os.path.abspath(os.path.join(self.tool_dependency_cache_dir, req_hashes))

galaxy/tools/deps/container_resolvers/__init__.py

@@ -5,17 +5,18 @@
     abstractproperty,
 )
 
+import six
+
 from galaxy.util.dictifiable import Dictifiable
 
 
+@six.add_metaclass(ABCMeta)
 class ContainerResolver(Dictifiable, object):
     """Description of a technique for resolving container images for tool execution."""
 
     # Keys for dictification.
     dict_collection_visible_keys = ['resolver_type']
 
-    __metaclass__ = ABCMeta
-
     def __init__(self, app_info=None, **kwds):
         """Default initializer for ``ContainerResolver`` subclasses."""
         self.app_info = app_info

galaxy/tools/deps/containers.py

@@ -279,8 +279,8 @@ def __init__(self, working_directory, tool_directory, job_directory, job_directo
         self.job_directory_type = job_directory_type  # "galaxy" or "pulsar"
 
 
+@six.add_metaclass(ABCMeta)
 class Container( object ):
-    __metaclass__ = ABCMeta
 
     def __init__(self, container_id, app_info, tool_info, destination_info, job_info, container_description):
         self.container_id = container_id
@@ -445,8 +445,9 @@ class NullContainer(object):
     def __init__(self):
         pass
 
-    def __nonzero__(self):
+    def __bool__(self):
         return False
+    __nonzero__ = __bool__
 
 
 NULL_CONTAINER = NullContainer()

galaxy/tools/deps/installable.py

@@ -2,13 +2,14 @@
 
 import logging
 import os
-
 from abc import (
     ABCMeta,
     abstractmethod,
     abstractproperty,
 )
 
+import six
+
 from galaxy.util.filelock import (
     FileLock,
     FileLockException
@@ -17,11 +18,10 @@
 log = logging.getLogger(__name__)
 
 
+@six.add_metaclass(ABCMeta)
 class InstallableContext(object):
     """Represent a directory/configuration of something that can be installed."""
 
-    __metaclass__ = ABCMeta
-
     @abstractmethod
     def is_installed(self):
         """Return bool indicating if the configured software is installed."""

galaxy/tools/deps/requirements.py

@@ -1,5 +1,7 @@
 import copy
 
+import six
+
 from galaxy.util import (
     asbool,
     xml_text,
@@ -11,6 +13,7 @@
 DEFAULT_REQUIREMENT_VERSION = None
 
 
+@six.python_2_unicode_compatible
 class ToolRequirement( object ):
     """
     Represents an external requirement that must be available for the tool to
@@ -47,6 +50,11 @@ def __ne__(self, other):
     def __hash__(self):
         return hash((self.name, self.type, self.version, frozenset(self.specs)))
 
+    def __str__(self):
+        return "ToolRequirement[%s,version=%s,type=%s,specs=%s]" % (self.name, self.version, self.type, self.specs)
+
+    __repr__ = __str__
+
 
 class RequirementSpecification(object):
     """Refine a requirement using a URI."""

galaxy/tools/deps/resolvers/__init__.py

@@ -5,6 +5,7 @@
     abstractproperty,
 )
 
+import six
 import yaml
 
 from galaxy.util import listify
@@ -13,6 +14,7 @@
 from ..requirements import ToolRequirement
 
 
+@six.add_metaclass(ABCMeta)
 class DependencyResolver(Dictifiable, object):
     """Abstract description of a technique for resolving container images for tool execution."""
 
@@ -26,7 +28,6 @@ class DependencyResolver(Dictifiable, object):
     disabled = False
     resolves_simple_dependencies = True
     config_options = {}
-    __metaclass__ = ABCMeta
 
     @abstractmethod
     def resolve( self, requirement, **kwds ):
@@ -51,12 +52,12 @@ def resolve_all( self, requirements, **kwds ):
         """
 
 
+@six.add_metaclass(ABCMeta)
 class ListableDependencyResolver:
     """ Mix this into a ``DependencyResolver`` and implement to indicate
     the dependency resolver can iterate over its dependencies and generate
     requirements.
     """
-    __metaclass__ = ABCMeta
 
     @abstractmethod
     def list_dependencies(self):
@@ -87,24 +88,20 @@ def _setup_mapping(self, dependency_manager, **kwds):
     def _mapping_file_to_list(mapping_file):
         with open(mapping_file, "r") as f:
             raw_mapping = yaml.load(f) or []
-            return map(RequirementMapping.from_dict, raw_mapping)
+        return map(RequirementMapping.from_dict, raw_mapping)
 
     def _expand_mappings(self, requirement):
         for mapping in self._mappings:
-            if requirement.name == mapping.from_name:
-                if mapping.from_version is not None and mapping.from_version != requirement.version:
-                    continue
-
-                requirement = requirement.copy()
-                requirement.name = mapping.to_name
-                if mapping.to_version is not None:
-                    requirement.version = mapping.to_version
-
+            if mapping.matches_requirement(requirement):
+                requirement = mapping.apply(requirement)
                 break
 
         return requirement
 
 
+FROM_UNVERSIONED = object()
+
+
 class RequirementMapping(object):
 
     def __init__(self, from_name, from_version, to_name, to_version):
@@ -113,12 +110,47 @@ def __init__(self, from_name, from_version, to_name, to_version):
         self.to_name = to_name
         self.to_version = to_version
 
+    def matches_requirement(self, requirement):
+        """Check if supplied ToolRequirement matches this mapping description.
+
+        For it to match - the names must match. Additionally if the
+        requirement is created with a version or with unversioned being set to
+        True additional checks are needed. If a version is specified, it must
+        match the supplied version exactly. If ``unversioned`` is True, then
+        the supplied requirement must be unversioned (i.e. its version must be
+        set to ``None``).
+        """
+
+        if requirement.name != self.from_name:
+            return False
+        elif self.from_version is None:
+            return True
+        elif self.from_version is FROM_UNVERSIONED:
+            return requirement.version is None
+        else:
+            return requirement.version == self.from_version
+
+    def apply(self, requirement):
+        requirement = requirement.copy()
+        requirement.name = self.to_name
+        if self.to_version is not None:
+            requirement.version = self.to_version
+        return requirement
+
     @staticmethod
     def from_dict(raw_mapping):
         from_raw = raw_mapping.get("from")
         if isinstance(from_raw, dict):
             from_name = from_raw.get("name")
-            from_version = str(from_raw.get("version"))
+            raw_version = from_raw.get("version", None)
+            unversioned = from_raw.get("unversioned", False)
+            if unversioned and raw_version:
+                raise Exception("Cannot define both version and set unversioned to True.")
+
+            if unversioned:
+                from_version = FROM_UNVERSIONED
+            else:
+                from_version = str(raw_version) if raw_version is not None else raw_version
         else:
             from_name = from_raw
             from_version = None
@@ -134,13 +166,13 @@ def from_dict(raw_mapping):
         return RequirementMapping(from_name, from_version, to_name, to_version)
 
 
+@six.add_metaclass(ABCMeta)
 class SpecificationAwareDependencyResolver:
     """Mix this into a :class:`DependencyResolver` to implement URI specification matching.
 
     Allows adapting generic requirements to more specific URIs - to tailor name
     or version to specified resolution system.
     """
-    __metaclass__ = ABCMeta
 
     @abstractmethod
     def _expand_specs(self, requirement):
@@ -178,11 +210,11 @@ def _expand_specs(self, requirement):
         return requirement
 
 
+@six.add_metaclass(ABCMeta)
 class InstallableDependencyResolver:
     """ Mix this into a ``DependencyResolver`` and implement to indicate
     the dependency resolver can attempt to install new dependencies.
     """
-    __metaclass__ = ABCMeta
 
     @abstractmethod
     def install_dependency(self, name, version, type, **kwds):
@@ -191,9 +223,9 @@ def install_dependency(self, name, version, type, **kwds):
         """
 
 
+@six.add_metaclass(ABCMeta)
 class Dependency(Dictifiable, object):
     dict_collection_visible_keys = ['dependency_type', 'exact', 'name', 'version', 'cacheable']
-    __metaclass__ = ABCMeta
     cacheable = False
 
     @abstractmethod