Merge pull request #135 from galaxyproject/update_to_galaxy_1809

Update to Galaxy 18.09.

galaxy/containers/docker_swarm.py

@@ -9,7 +9,7 @@
 from functools import partial
 
 try:
-    import docker
+    import docker.types
 except ImportError:
     from galaxy.util.bunch import Bunch
     docker = Bunch(types=Bunch(

galaxy/exceptions/error_codes.py

@@ -6,6 +6,8 @@
 
 from pkg_resources import resource_string
 
+from galaxy.util import unicodify
+
 
 # Error codes are provided as a convience to Galaxy API clients, but at this
 # time they do represent part of the more stable interface. They can change
@@ -42,7 +44,7 @@ def _from_dict(entry):
     return (name, ErrorCode(code, message))
 
 
-error_codes_json = resource_string(__name__, 'error_codes.json').decode("UTF-8")
+error_codes_json = unicodify(resource_string(__name__, 'error_codes.json'))
 for entry in loads(error_codes_json):
     name, error_code_obj = _from_dict(entry)
     globals()[name] = error_code_obj

galaxy/jobs/metrics/instrumenters/cgroup.py

@@ -4,8 +4,8 @@
 from collections import namedtuple
 
 from galaxy.util import asbool, nice_size
-from ..instrumenters import InstrumentPlugin
-from ...metrics import formatting
+from . import InstrumentPlugin
+from .. import formatting
 
 log = logging.getLogger(__name__)
 
@@ -54,6 +54,12 @@ class CgroupPlugin(InstrumentPlugin):
 
     def __init__(self, **kwargs):
         self.verbose = asbool(kwargs.get("verbose", False))
+        params_str = kwargs.get("params", None)
+        if params_str:
+            params = [v.strip() for v in params_str.split(",")]
+        else:
+            params = TITLES.keys()
+        self.params = params
 
     def post_execute_instrument(self, job_directory):
         commands = []
@@ -92,7 +98,7 @@ def __read_metrics(self, path):
         return metrics
 
     def __add_metric(self, metrics, metric):
-        if metric and (metric.subkey in TITLES or self.verbose):
+        if metric and (metric.subkey in self.params or self.verbose):
             metrics[metric.subkey] = metric.value
 
     def __read_key_value(self, line, prev_metric):

galaxy/jobs/metrics/instrumenters/collectl.py

@@ -4,13 +4,13 @@
 import shutil
 
 from galaxy import util
+from . import InstrumentPlugin
+from .. import formatting
 from ..collectl import (
     cli,
     processes,
     subsystems
 )
-from ..instrumenters import InstrumentPlugin
-from ...metrics import formatting
 
 log = logging.getLogger(__name__)
 

galaxy/jobs/metrics/instrumenters/core.py

@@ -2,8 +2,8 @@
 import logging
 import time
 
-from ..instrumenters import InstrumentPlugin
-from ...metrics import formatting
+from . import InstrumentPlugin
+from .. import formatting
 
 log = logging.getLogger(__name__)
 

galaxy/jobs/metrics/instrumenters/cpuinfo.py

@@ -3,8 +3,8 @@
 import re
 
 from galaxy import util
-from ..instrumenters import InstrumentPlugin
-from ...metrics import formatting
+from . import InstrumentPlugin
+from .. import formatting
 
 log = logging.getLogger(__name__)
 

galaxy/jobs/metrics/instrumenters/env.py

@@ -2,8 +2,8 @@
 import logging
 import re
 
-from ..instrumenters import InstrumentPlugin
-from ...metrics import formatting
+from . import InstrumentPlugin
+from .. import formatting
 
 log = logging.getLogger(__name__)
 

galaxy/jobs/metrics/instrumenters/hostname.py

@@ -0,0 +1,36 @@
+"""The module describes the ``cpuinfo`` job metrics plugin."""
+import logging
+
+from . import InstrumentPlugin
+from .. import formatting
+
+log = logging.getLogger(__name__)
+
+
+class HostnameFormatter(formatting.JobMetricFormatter):
+
+    def format(self, key, value):
+        return key, value
+
+
+class HostnamePlugin(InstrumentPlugin):
+    """ Gather hostname
+    """
+    plugin_type = "hostname"
+    formatter = HostnameFormatter()
+
+    def __init__(self, **kwargs):
+        pass
+
+    def pre_execute_instrument(self, job_directory):
+        return "hostname -f > '%s'" % self.__instrument_hostname_path(job_directory)
+
+    def job_properties(self, job_id, job_directory):
+        with open(self.__instrument_hostname_path(job_directory)) as f:
+            return {'hostname': f.read().strip()}
+
+    def __instrument_hostname_path(self, job_directory):
+        return self._instrument_file_path(job_directory, "hostname")
+
+
+__all__ = ('HostnamePlugin', )

galaxy/jobs/metrics/instrumenters/meminfo.py

@@ -3,8 +3,8 @@
 import sys
 
 from galaxy import util
-from ..instrumenters import InstrumentPlugin
-from ...metrics import formatting
+from . import InstrumentPlugin
+from .. import formatting
 
 if sys.version_info > (3,):
     long = int

galaxy/jobs/metrics/instrumenters/uname.py

@@ -1,6 +1,6 @@
 """The module describes the ``uname`` job metrics plugin."""
-from ..instrumenters import InstrumentPlugin
-from ...metrics import formatting
+from . import InstrumentPlugin
+from .. import formatting
 
 
 class UnameFormatter(formatting.JobMetricFormatter):

galaxy/objectstore/__init__.py

@@ -632,7 +632,7 @@ def __filesystem_monitor(self):
     def create(self, obj, **kwargs):
         """The only method in which obj.object_store_id may be None."""
         if obj.object_store_id is None or not self.exists(obj, **kwargs):
-            if obj.object_store_id is None or obj.object_store_id not in self.weighted_backend_ids:
+            if obj.object_store_id is None or obj.object_store_id not in self.backends:
                 try:
                     obj.object_store_id = random.choice(self.weighted_backend_ids)
                 except IndexError:
@@ -808,3 +808,24 @@ def _create_object_in_session(obj):
         object_session(obj).flush()
     else:
         raise Exception(NO_SESSION_ERROR_MESSAGE)
+
+
+class ObjectStorePopulator(object):
+    """ Small helper for interacting with the object store and making sure all
+    datasets from a job end up with the same object_store_id.
+    """
+
+    def __init__(self, app):
+        self.object_store = app.object_store
+        self.object_store_id = None
+
+    def set_object_store_id(self, data):
+        # Create an empty file immediately.  The first dataset will be
+        # created in the "default" store, all others will be created in
+        # the same store as the first.
+        data.dataset.object_store_id = self.object_store_id
+        try:
+            self.object_store.create(data.dataset)
+        except ObjectInvalid:
+            raise Exception('Unable to create output dataset: object store is full')
+        self.object_store_id = data.dataset.object_store_id  # these will be the same thing after the first output

galaxy/tools/deps/__init__.py

@@ -182,7 +182,7 @@ def find_dep(self, name, version=None, type='package', **kwds):
         requirements = ToolRequirements([ToolRequirement(name=name, version=version, type=type)])
         dep_dict = self._requirements_to_dependencies_dict(requirements, **kwds)
         if len(dep_dict) > 0:
-            return dep_dict.values()[0]
+            return next(iter(dep_dict.values()))  # get first dep
         else:
             return NullDependency(name=name, version=version)
 

galaxy/tools/deps/conda_util.py

@@ -13,7 +13,10 @@
 from six.moves import shlex_quote
 
 from galaxy.tools.deps.commands import CommandLineException
-from galaxy.util import unicodify
+from galaxy.util import (
+    smart_str,
+    unicodify
+)
 from . import (
     commands,
     installable
@@ -417,7 +420,7 @@ def hash_conda_packages(conda_packages, conda_target=None):
     """
     h = hashlib.new('sha256')
     for conda_package in conda_packages:
-        h.update(conda_package.install_environment)
+        h.update(smart_str(conda_package.install_environment))
     return h.hexdigest()
 
 

galaxy/tools/deps/container_resolvers/mulled.py

@@ -242,9 +242,9 @@ class CachedMulledSingularityContainerResolver(ContainerResolver):
     resolver_type = "cached_mulled_singularity"
     container_type = "singularity"
 
-    def __init__(self, app_info=None, hash_func="v2"):
+    def __init__(self, app_info=None, hash_func="v2", **kwds):
         super(CachedMulledSingularityContainerResolver, self).__init__(app_info)
-        self.cache_directory = os.path.join(app_info.container_image_cache_path, "singularity", "mulled")
+        self.cache_directory = kwds.get("cache_directory", os.path.join(app_info.container_image_cache_path, "singularity", "mulled"))
         self.hash_func = hash_func
 
     def resolve(self, enabled_container_types, tool_info):
@@ -393,7 +393,7 @@ def __init__(self, app_info=None, hash_func="v2", **kwds):
         self._involucro_context_kwds = {
             'involucro_bin': self._get_config_option("involucro_path", None)
         }
-        self.cache_directory = os.path.join(app_info.container_image_cache_path, "singularity", "mulled")
+        self.cache_directory = kwds.get("cache_directory", os.path.join(app_info.container_image_cache_path, "singularity", "mulled"))
         self.hash_func = hash_func
         self._mulled_kwds = {
             'channels': self._get_config_option("channels", DEFAULT_CHANNELS, prefix="mulled"),

galaxy/tools/deps/containers.py

@@ -106,12 +106,18 @@ def __destination_container(container_description=None, container_id=None, conta
             )
             return container
 
+        def container_from_description_from_dicts(destination_container_dicts):
+            for destination_container_dict in destination_container_dicts:
+                container_description = ContainerDescription.from_dict(destination_container_dict)
+                if container_description:
+                    container = __destination_container(container_description)
+                    if container:
+                        return container
+
         if "container_override" in destination_info:
-            container_description = ContainerDescription.from_dict(destination_info["container_override"][0])
-            if container_description:
-                container = __destination_container(container_description)
-                if container:
-                    return container
+            container = container_from_description_from_dicts(destination_info["container_override"])
+            if container:
+                return container
 
         # If destination forcing Galaxy to use a particular container do it,
         # this is likely kind of a corner case. For instance if deployers
@@ -132,11 +138,9 @@ def __destination_container(container_description=None, container_id=None, conta
         # If we still don't have a container, check to see if any container
         # types define a default container id and use that.
         if "container" in destination_info:
-            container_description = ContainerDescription.from_dict(destination_info["container"][0])
-            if container_description:
-                container = __destination_container(container_description)
-                if container:
-                    return container
+            container = container_from_description_from_dicts(destination_info["container"])
+            if container:
+                return container
 
         for container_type in CONTAINER_CLASSES.keys():
             container_id = self.__default_container_id(container_type, destination_info)

galaxy/tools/parser/cwl.py

@@ -4,6 +4,7 @@
 from galaxy.tools.cwl import tool_proxy
 from galaxy.tools.deps import requirements
 from galaxy.util.odict import odict
+from .error_level import StdioErrorLevel
 from .interface import (
     PageSource,
     PagesSource,
@@ -77,8 +78,6 @@ def parse_strict_shell(self):
 
     def parse_stdio(self):
         # TODO: remove duplication with YAML
-        from galaxy.jobs.error_level import StdioErrorLevel
-
         # New format - starting out just using exit code.
         exit_code_lower = ToolStdioExitCode()
         exit_code_lower.range_start = float("-inf")

galaxy/tools/parser/error_level.py

@@ -0,0 +1,26 @@
+
+
+# These determine stdio-based error levels from matching on regular expressions
+# and exit codes. They are meant to be used comparatively, such as showing
+# that warning < fatal. This is really meant to just be an enum.
+class StdioErrorLevel(object):
+    NO_ERROR = 0
+    LOG = 1
+    WARNING = 2
+    FATAL = 3
+    FATAL_OOM = 4
+    MAX = 4
+    descs = {
+        NO_ERROR: 'No error',
+        LOG: 'Log',
+        WARNING: 'Warning',
+        FATAL: 'Fatal error',
+        FATAL_OOM: 'Out of memory error',
+    }
+
+    @staticmethod
+    def desc(error_level):
+        err_msg = "Unknown error"
+        if error_level > 0 and error_level <= StdioErrorLevel.MAX:
+            err_msg = StdioErrorLevel.descs[error_level]
+        return err_msg

galaxy/tools/parser/interface.py

@@ -5,6 +5,8 @@
 
 import six
 
+from .error_level import StdioErrorLevel
+
 NOT_IMPLEMENTED_MESSAGE = "Galaxy tool format does not yet support this tool feature."
 
 
@@ -211,6 +213,9 @@ def parse_profile(self):
         """ Return tool profile version as Galaxy major e.g. 16.01 or 16.04.
         """
 
+    def macro_paths(self):
+        return []
+
     def parse_tests_to_dict(self):
         return {'tests': []}
 
@@ -337,8 +342,7 @@ def __init__(self):
         self.match = ""
         self.stdout_match = False
         self.stderr_match = False
-        # TODO: Define a common class or constant for error level:
-        self.error_level = "fatal"
+        self.error_level = StdioErrorLevel.FATAL
         self.desc = ""
 
 
@@ -351,8 +355,7 @@ class ToolStdioExitCode(object):
     def __init__(self):
         self.range_start = float("-inf")
         self.range_end = float("inf")
-        # TODO: Define a common class or constant for error level:
-        self.error_level = "fatal"
+        self.error_level = StdioErrorLevel.FATAL
         self.desc = ""