Merge pull request #3772 from galaxyproject/release_17.01

[usegalaxy] Merge release 17.01

doc/source/releases/17.01_announce.rst

@@ -29,15 +29,17 @@ Highlights
   Thanks to `@abretaud <https://github.com/abretaud>`__, `@ashvark <https://github.com/ashvark>`__, `@jvolkening <https://github.com/jvolkening>`__, and `@mvdbeek <https://github.com/mvdbeek>`__.
   Implemented in `Pull Request 3145`_, `PullRequest 3510`_ and `PullRequest 3514`_.
 
-`Github <https://github.com/galaxyproject/galaxy>`__
-===========================================================
+Get Galaxy
+==========
+
+The code lives at `Github <https://github.com/galaxyproject/galaxy>`__ and you should have `Git <https://git-scm.com/>`__ to obtain it.
 
-New Galaxy repository
+To get a new Galaxy repository run:
   .. code-block:: shell
 
       $ git clone -b release_17.01 https://github.com/galaxyproject/galaxy.git
 
-Update of existing Galaxy repository
+To update an existing Galaxy repository run:
   .. code-block:: shell
 
       $ git checkout release_17.01 && git pull --ff-only origin release_17.01

lib/galaxy/managers/workflows.py

@@ -398,8 +398,12 @@ def step_title(step, default_name):
                 step_model["name"] = step_title(step, step_model.get("name"))
             else:
                 inputs = step.module.get_runtime_inputs( connections=step.output_connections )
+                step_name = step.module.name
+                if hasattr( step, 'tool_inputs' ):
+                    if isinstance( step.tool_inputs, dict ):
+                        step_name = step.tool_inputs.get( 'name' ) or step_name
                 step_model = {
-                    'name'   : step_title(step, step.module.name),
+                    'name'   : step_title(step, step_name),
                     'inputs' : [ input.to_dict( trans ) for input in inputs.itervalues() ]
                 }
             step_model[ 'step_type' ] = step.type

lib/galaxy/queue_worker.py

@@ -9,6 +9,7 @@
 
 import galaxy.queues
 from galaxy import util
+from galaxy.model.util import pgcalc
 
 from kombu import Connection
 from kombu.mixins import ConsumerMixin
@@ -18,7 +19,31 @@
 log = logging.getLogger(__name__)
 
 
+def send_local_control_task(app, task, kwargs={}):
+    """
+    This sends a message to the process-local control worker, which is useful
+    for one-time asynchronous tasks like recalculating user disk usage.
+    """
+    log.info("Queuing async task %s." % task)
+    payload = {'task': task,
+               'kwargs': kwargs}
+    try:
+        c = Connection(app.config.amqp_internal_connection)
+        with producers[c].acquire(block=True) as producer:
+            producer.publish(payload,
+                             exchange=galaxy.queues.galaxy_exchange,
+                             declare=[galaxy.queues.galaxy_exchange] + [galaxy.queues.control_queue_from_config(app.config)],
+                             routing_key='control')
+    except Exception:
+        log.exception("Error queueing async task: %s." % payload)
+
+
 def send_control_task(app, task, noop_self=False, kwargs={}):
+    """
+    This sends a control task out to all processes, useful for things like
+    reloading a data table, which needs to happen individually in all
+    processes.
+    """
     log.info("Sending %s control task." % task)
     payload = {'task': task,
                'kwargs': kwargs}
@@ -112,6 +137,21 @@ def reload_sanitize_whitelist(app):
     app.config.reload_sanitize_whitelist()
 
 
+def recalculate_user_disk_usage(app, **kwargs):
+    user_id = kwargs.get('user_id', None)
+    sa_session = app.model.context
+    if user_id:
+        user = sa_session.query( app.model.User ).get( app.security.decode_id( user_id ) )
+        if user:
+            if sa_session.get_bind().dialect.name not in ( 'postgres', 'postgresql' ):
+                new = user.calculate_disk_usage()
+            else:
+                new = pgcalc(sa_session, user.id)
+            user.set_disk_usage(new)
+            sa_session.add(user)
+            sa_session.flush()
+
+
 def reload_tool_data_tables(app, **kwargs):
     params = util.Params(kwargs)
     log.debug("Executing tool data table reload for %s" % params.get('table_names', 'all tables'))
@@ -135,7 +175,8 @@ def admin_job_lock(app, **kwargs):
                             'reload_display_application': reload_display_application,
                             'reload_tool_data_tables': reload_tool_data_tables,
                             'admin_job_lock': admin_job_lock,
-                            'reload_sanitize_whitelist': reload_sanitize_whitelist}
+                            'reload_sanitize_whitelist': reload_sanitize_whitelist,
+                            'recalculate_user_disk_usage': recalculate_user_disk_usage}
 
 
 class GalaxyQueueWorker(ConsumerMixin, threading.Thread):

lib/galaxy/tools/__init__.py

@@ -122,6 +122,12 @@
     "maf_limit_size1",
     "maf_by_block_number1",
     "wiggle2simple1",
+    # Converters
+    "CONVERTER_fastq_to_fqtoc0",
+    "CONVERTER_gff_to_interval_index_0",
+    "CONVERTER_maf_to_fasta_0",
+    "CONVERTER_maf_to_interval_0",
+    "CONVERTER_wiggle_to_interval_0",
     # Tools improperly migrated to the tool shed (devteam)
     "lastz_wrapper_2",
     "qualityFilter",

lib/galaxy/tools/deps/container_resolvers/__init__.py

@@ -5,9 +5,12 @@
     abstractproperty,
 )
 
+import six
+
 from galaxy.util.dictifiable import Dictifiable
 
 
+@six.python_2_unicode_compatible
 class ContainerResolver(Dictifiable, object):
     """Description of a technique for resolving container images for tool execution."""
 
@@ -48,3 +51,6 @@ def resolver_type(self):
     def _container_type_enabled(self, container_description, enabled_container_types):
         """Return a boolean indicating if the specified container type is enabled."""
         return container_description.type in enabled_container_types
+
+    def __str__(self):
+        return "%s[]" % self.__class__.__name__

lib/galaxy/tools/deps/container_resolvers/mulled.py

@@ -3,6 +3,8 @@
 import collections
 import logging
 
+import six
+
 from ..container_resolvers import (
     ContainerResolver,
 )
@@ -102,6 +104,7 @@ def cached_container_description(targets, namespace):
     return container
 
 
+@six.python_2_unicode_compatible
 class CachedMulledContainerResolver(ContainerResolver):
 
     resolver_type = "cached_mulled"
@@ -114,7 +117,11 @@ def resolve(self, enabled_container_types, tool_info):
         targets = mulled_targets(tool_info)
         return cached_container_description(targets, self.namespace)
 
+    def __str__(self):
+        return "CachedMulledContainerResolver[namespace=%s]" % self.namespace
+
 
+@six.python_2_unicode_compatible
 class MulledContainerResolver(ContainerResolver):
     """Look for mulled images matching tool dependencies."""
 
@@ -160,7 +167,11 @@ def resolve(self, enabled_container_types, tool_info):
                 type="docker",
             )
 
+    def __str__(self):
+        return "MulledContainerResolver[namespace=%s]" % self.namespace
 
+
+@six.python_2_unicode_compatible
 class BuildMulledContainerResolver(ContainerResolver):
     """Look for mulled images matching tool dependencies."""
 
@@ -195,6 +206,9 @@ def _get_involucro_context(self):
         self.enabled = ensure_installed(involucro_context, self.auto_init)
         return involucro_context
 
+    def __str__(self):
+        return "BuildContainerResolver[namespace=%s]" % self.namespace
+
 
 def mulled_targets(tool_info):
     return requirements_to_mulled_targets(tool_info.requirements)

lib/galaxy/tools/deps/requirements.py

@@ -1,5 +1,7 @@
 import copy
 
+import six
+
 from galaxy.util import (
     asbool,
     xml_text,
@@ -143,6 +145,7 @@ class ToolRequirementsException(Exception):
 DEFAULT_CONTAINER_SHELL = "/bin/sh"  # Galaxy assumes bash, but containers are usually thinner.
 
 
+@six.python_2_unicode_compatible
 class ContainerDescription( object ):
 
     def __init__(
@@ -178,6 +181,9 @@ def from_dict( dict ):
             shell=shell,
         )
 
+    def __str__(self):
+        return "ContainerDescription[identifier=%s,type=%s]" % (self.identifier, self.type)
+
 
 def parse_requirements_from_dict( root_dict ):
     requirements = root_dict.get("requirements", [])

lib/galaxy/tools/search/__init__.py

@@ -49,8 +49,9 @@ def __init__( self, toolbox, index_help=True ):
         self.build_index( index_help )
 
     def build_index( self, index_help=True ):
-        # Works around https://bitbucket.org/mchaput/whoosh/issues/391/race-conditions-with-temp-storage
+        """Prepare search index for tools loaded in toolbox."""
         RamStorage.temp_storage = _temp_storage
+        # Works around https://bitbucket.org/mchaput/whoosh/issues/391/race-conditions-with-temp-storage
         self.storage = RamStorage()
         self.index = self.storage.create_index( self.schema )
         writer = self.index.writer()
@@ -66,16 +67,15 @@ def build_index( self, index_help=True ):
                 "section": to_unicode( tool.get_panel_section()[1] if len( tool.get_panel_section() ) == 2 else '' ),
                 "help": to_unicode( "" )
             }
-            # Hyphens are wildcards in Whoosh causing bad things
             if tool.name.find( '-' ) != -1:
+                # Hyphens are wildcards in Whoosh causing bad things
                 add_doc_kwds['name'] = (' ').join( [ token.text for token in self.rex( to_unicode( tool.name ) ) ] )
             else:
                 add_doc_kwds['name'] = to_unicode( tool.name )
-            # We do not want to search Tool Shed or version parts
-            # of the long ids
-            if id.find( '/' ) != -1:
-                slash_indexes = [ m.start() for m in re.finditer( '/', id ) ]
-                id_stub = id[ ( slash_indexes[1] + 1 ): slash_indexes[4] ]
+            if tool.guid:
+                # Create a stub consisting of owner, repo, and tool from guid
+                slash_indexes = [ m.start() for m in re.finditer( '/', tool.guid ) ]
+                id_stub = tool.guid[ ( slash_indexes[1] + 1 ): slash_indexes[4] ]
                 add_doc_kwds['stub'] = (' ').join( [ token.text for token in self.rex( to_unicode( id_stub ) ) ] )
             else:
                 add_doc_kwds['stub'] = to_unicode( id )