Merge branch 'feature/plugin-system' of github.com:bgruening/galaxy i…

…nto feature/plugin-system

.ci/flake8_lint_include_list.txt

@@ -95,6 +95,7 @@ lib/galaxy/main.py
 lib/galaxy/managers/annotatable.py
 lib/galaxy/managers/api_keys.py
 lib/galaxy/managers/base.py
+lib/galaxy/managers/collections_util.py
 lib/galaxy/managers/context.py
 lib/galaxy/managers/deletable.py
 lib/galaxy/managers/__init__.py
@@ -103,7 +104,9 @@ lib/galaxy/managers/libraries.py
 lib/galaxy/managers/secured.py
 lib/galaxy/managers/taggable.py
 lib/galaxy/model/__init__.py
+lib/galaxy/model/item_attrs.py
 lib/galaxy/model/mapping.py
+lib/galaxy/model/metadata.py
 lib/galaxy/model/migrate/__init__.py
 lib/galaxy/model/migrate/versions/0001_initial_tables.py
 lib/galaxy/model/migrate/versions/0002_metadata_file_table.py
@@ -246,21 +249,27 @@ lib/galaxy/sample_tracking/sample.py
 lib/galaxy/security/validate_user_input.py
 lib/galaxy/tags/
 lib/galaxy/tools/actions/metadata.py
-lib/galaxy/tools/cwl
+lib/galaxy/tools/cwl/
 lib/galaxy/tools/data_manager/__init__.py
-lib/galaxy/tools/deps
+lib/galaxy/tools/deps/
+lib/galaxy/tools/exception_handling.py
+lib/galaxy/tools/execute.py
 lib/galaxy/tools/filters/
 lib/galaxy/tools/imp_exp/export_history.py
 lib/galaxy/tools/imp_exp/__init__.py
 lib/galaxy/tools/linters/
 lib/galaxy/tools/lint.py
 lib/galaxy/tools/lint_util.py
 lib/galaxy/tools/loader_directory.py
+lib/galaxy/tools/loader.py
+lib/galaxy/tools/parameters/dataset_matcher.py
 lib/galaxy/tools/parameters/history_query.py
+lib/galaxy/tools/parameters/__init__.py
 lib/galaxy/tools/parameters/input_translation.py
 lib/galaxy/tools/parameters/sanitize.py
 lib/galaxy/tools/parameters/validation.py
 lib/galaxy/tools/parameters/wrapped_json.py
+lib/galaxy/tools/parameters/wrapped.py
 lib/galaxy/tools/parser/
 lib/galaxy/tools/special_tools.py
 lib/galaxy/tools/test.py
@@ -347,9 +356,11 @@ lib/galaxy/web/framework/middleware/statsd.py
 lib/galaxy/web/framework/middleware/translogger.py
 lib/galaxy/web/framework/middleware/xforwardedhost.py
 lib/galaxy/web/params.py
+lib/galaxy/workflow/extract.py
 lib/galaxy/workflow/__init__.py
 lib/galaxy/workflow/render.py
-lib/galaxy/workflow/schedulers/__init__.py
+lib/galaxy/workflow/run.py
+lib/galaxy/workflow/schedulers/
 lib/galaxy/workflow/steps.py
 lib/galaxy/work/__init__.py
 lib/mimeparse.py
@@ -517,6 +528,7 @@ test/unit/tools/test_tool_deps.py
 test/unit/tools/test_tool_loader.py
 test/unit/tools/test_toolbox_filters.py
 test/unit/tools/test_watcher.py
+test/unit/tools_support.py
 test/unit/unittest_utils/__init__.py
 test/unit/unittest_utils/utility.py
 test/unit/visualizations/__init__.py

.ci/py3_sources.txt

@@ -28,8 +28,11 @@ lib/galaxy/forms/
 lib/galaxy/jobs/
 lib/galaxy/managers/
 lib/galaxy/model/__init__.py
+lib/galaxy/model/item_attrs.py
 lib/galaxy/model/mapping.py
+lib/galaxy/model/metadata.py
 lib/galaxy/model/migrate/
+lib/galaxy/model/orm/now.py
 lib/galaxy/objectstore/
 lib/galaxy/openid/
 lib/galaxy/quota/
@@ -38,17 +41,26 @@ lib/galaxy/security/
 lib/galaxy/tags/
 lib/galaxy/tools/cwl/
 lib/galaxy/tools/deps/
+lib/galaxy/tools/exception_handling.py
+lib/galaxy/tools/execute.py
 lib/galaxy/tools/lint.py
 lib/galaxy/tools/lint_util.py
 lib/galaxy/tools/linters/
 lib/galaxy/tools/loader.py
 lib/galaxy/tools/loader_directory.py
+lib/galaxy/tools/parameters/dataset_matcher.py
+lib/galaxy/tools/parameters/__init__.py
+lib/galaxy/tools/parameters/wrapped_json.py
+lib/galaxy/tools/parameters/wrapped.py
 lib/galaxy/tools/parser/
 lib/galaxy/tools/toolbox/
 lib/galaxy/tours/
 lib/galaxy/util/
 lib/galaxy/visualization/
 lib/galaxy/work/
+lib/galaxy/workflow/extract.py
+lib/galaxy/workflow/run.py
+lib/galaxy/workflow/schedulers/core.py
 lib/galaxy_ext/
 lib/galaxy_utils/
 lib/log_tempfile.py

config/galaxy.ini.sample

@@ -890,8 +890,7 @@ use_interactive = True
 # Galaxy encodes various internal values when these values will be output in
 # some format (for example, in a URL or cookie).  You should set a key to be
 # used by the algorithm that encodes and decodes these values.  It can be any
-# string.  If left unchanged, anyone could construct a cookie that would grant
-# them access to others' sessions.
+# string.
 # One simple way to generate a value for this is with the shell command:
 #   python -c 'import time; print time.time()' | md5sum | cut -f 1 -d ' '
 #id_secret = USING THE DEFAULT IS NOT SECURE!

config/plugins/webhooks/demo/trans_object/helper/__init__.py

@@ -1,2 +1,6 @@
 def main(trans, webhook):
-    return {'username': trans.user.username}
+    if trans.user:
+        user = trans.user.username
+    else:
+        user = 'No user is logged in.'
+    return {'username': user}

doc/source/admin/webhooks.rst

@@ -7,17 +7,11 @@ can consists out of simple HTML, JS or dynamically generated content from a pyth
 
 Plugin activation
 -----------------
-Webhooks are deactivated by default.
-To activate webhooks make sure the path in your `galaxy.ini` it set and that you activate all plugins that you want to use manually by changing the config of each webhook:
-
-.. code-block:: none
-
-  config/plugins/webhooks/{TOOL_NAME}/config/{TOOL_NAME}.yml
-and set the parameter activate to true:
-
-.. code-block:: python
-
-  activate: true 
+All webhooks that are included in the main Galaxy distribution are located in the `config/plugins/webhooks/demo` folder
+and are deactivated by default.
+To activate these demo webhooks make sure this path is added to `webhooks_dir` in your `galaxy.ini`. You can add as many 
+webhook folders as you like as a comma separated list. 
+Webhooks supports one additionl layer of activating/deactivating by changing the `activate: true` in each config of each webhook.
 
 
 Entry points

lib/galaxy/managers/collections_util.py

@@ -1,8 +1,7 @@
-from galaxy import exceptions
-from galaxy import web
-from galaxy import model
-
 import logging
+
+from galaxy import exceptions, model, web
+
 log = logging.getLogger( __name__ )
 
 ERROR_MESSAGE_UNKNOWN_SRC = "Unknown dataset source (src) %s."
@@ -77,7 +76,7 @@ def dictify_dataset_collection_instance( dataset_collection_instance, parent, se
         dict_value[ 'url' ] = web.url_for( 'library_content', library_id=encoded_library_id, id=encoded_id, folder_id=encoded_folder_id )
     if view == "element":
         collection = dataset_collection_instance.collection
-        dict_value[ 'elements' ] = map( dictify_element, collection.elements )
+        dict_value[ 'elements' ] = [ dictify_element(_) for _ in collection.elements ]
         dict_value[ 'populated' ] = collection.populated
     security.encode_all_ids( dict_value, recursive=True )  # TODO: Use Kyle's recursive formulation of this.
     return dict_value
@@ -89,10 +88,10 @@ def dictify_element( element ):
     if element.child_collection:
         # Recursively yield elements for each nested collection...
         child_collection = element.child_collection
-        object_detials[ "elements" ] = map( dictify_element, child_collection.elements )
+        object_detials[ "elements" ] = [ dictify_element(_) for _ in child_collection.elements ]
         object_detials[ "populated" ] = child_collection.populated
 
     dictified[ "object" ] = object_detials
     return dictified
 
-__all__ = [ api_payload_to_create_params, dictify_dataset_collection_instance ]
+__all__ = ( 'api_payload_to_create_params', 'dictify_dataset_collection_instance' )

lib/galaxy/model/item_attrs.py

@@ -1,7 +1,9 @@
+import logging
+
 from sqlalchemy.sql.expression import func
+
 # Cannot import galaxy.model b/c it creates a circular import graph.
 import galaxy
-import logging
 
 log = logging.getLogger( __name__ )
 
@@ -162,8 +164,8 @@ def _get_annotation_assoc_class( self, item ):
         return getattr( galaxy.model, class_name, None )
 
 
-__all__ = [
+__all__ = (
     'UsesAnnotations',
     'UsesItemRatings',
     'RuntimeException',
-]
+)

lib/galaxy/model/metadata.py

@@ -4,7 +4,6 @@
 """
 
 import copy
-import cPickle
 import json
 import logging
 import os
@@ -15,6 +14,7 @@
 from os.path import abspath
 
 from six import string_types
+from six.moves import cPickle
 from sqlalchemy.orm import object_session
 
 import galaxy.model
@@ -93,13 +93,14 @@ def get( self, key, default=None ):
             return default
 
     def items(self):
-        return iter( [ ( k, self.get( k ) ) for k in self.spec.iterkeys() ] )
+        return iter( [ ( k, self.get( k ) ) for k in self.spec.keys() ] )
 
     def __str__(self):
         return dict( self.items() ).__str__()
 
-    def __nonzero__( self ):
+    def __bool__( self ):
         return bool( self.parent._metadata )
+    __nonzero__ = __bool__
 
     def __getattr__( self, name ):
         if name in self.spec:
@@ -205,7 +206,7 @@ def append( self, item ):
         self[item.name] = item
 
     def iter( self ):
-        return self.itervalues()
+        return iter(self.values())
 
     def __getattr__( self, name ):
         return self.get( name )
@@ -452,15 +453,15 @@ def get_html_field( self, value=None, context=None, other_values=None, values=No
         other_values = other_values or {}
 
         if values is None:
-            values = zip( range( self.min, self.max, self.step ), range( self.min, self.max, self.step ))
+            values = list(zip( range( self.min, self.max, self.step ), range( self.min, self.max, self.step ) ))
         return SelectParameter.get_html_field( self, value=value, context=context, other_values=other_values, values=values, **kwd )
 
     def get_html( self, value, context=None, other_values=None, values=None, **kwd ):
         context = context or {}
         other_values = other_values or {}
 
         if values is None:
-            values = zip( range( self.min, self.max, self.step ), range( self.min, self.max, self.step ))
+            values = list(zip( range( self.min, self.max, self.step ), range( self.min, self.max, self.step ) ))
         return SelectParameter.get_html( self, value, context=context, other_values=other_values, values=values, **kwd )
 
     @classmethod
@@ -478,7 +479,7 @@ def get_html_field( self, value=None, context=None, other_values=None, values=No
 
         if values is None and context:
             column_range = range( 1, ( context.columns or 0 ) + 1, 1 )
-            values = zip( column_range, column_range )
+            values = list(zip( column_range, column_range ))
         return RangeParameter.get_html_field( self, value=value, context=context, other_values=other_values, values=values, **kwd )
 
     def get_html( self, value, context=None, other_values=None, values=None, **kwd ):
@@ -487,7 +488,7 @@ def get_html( self, value, context=None, other_values=None, values=None, **kwd )
 
         if values is None and context:
             column_range = range( 1, ( context.columns or 0 ) + 1, 1 )
-            values = zip( column_range, column_range )
+            values = list(zip( column_range, column_range ))
         return RangeParameter.get_html( self, value, context=context, other_values=other_values, values=values, **kwd )
 
 
@@ -815,7 +816,7 @@ def __get_filename_override():
                 metadata_files.filename_override_metadata = abspath( tempfile.NamedTemporaryFile( dir=tmp_dir, prefix="metadata_override_%s_" % key ).name )
                 open( metadata_files.filename_override_metadata, 'wb+' )  # create the file on disk, so it cannot be reused by tempfile (unlikely, but possible)
                 override_metadata = []
-                for meta_key, spec_value in dataset.metadata.spec.iteritems():
+                for meta_key, spec_value in dataset.metadata.spec.items():
                     if isinstance( spec_value.param, FileParameter ) and dataset.metadata.get( meta_key, None ) is not None:
                         metadata_temp = MetadataTempFile()
                         shutil.copy( dataset.metadata.get( meta_key, None ).file_name, metadata_temp.file_name )
@@ -871,7 +872,7 @@ def set_job_runner_external_pid( self, pid, sa_session ):
             sa_session.add( metadata_files )
             sa_session.flush()
 
-__all__ = [
+__all__ = (
     "Statement",
     "MetadataElement",
     "MetadataCollection",
@@ -889,4 +890,4 @@ def set_job_runner_external_pid( self, pid, sa_session ):
     "FileParameter",
     "MetadataTempFile",
     "JobExternalOutputMetadataWrapper",
-]
+)

lib/galaxy/model/orm/now.py

@@ -10,4 +10,4 @@
 # Return the current time in UTC without any timezone information
 now = datetime.utcnow
 
-__all__ = [now]
+__all__ = ('now', )

lib/galaxy/tools/cwl/__init__.py

@@ -2,18 +2,17 @@
     needs_shell_quoting,
     shellescape,
 )
-from .parser import tool_proxy
-from .parser import workflow_proxy
+from .parser import tool_proxy, workflow_proxy
 from .representation import to_cwl_job, to_galaxy_parameters
 from .runtime_actions import handle_outputs
 
 
-__all__ = [
+__all__ = (
     'tool_proxy',
     'workflow_proxy',
     'handle_outputs',
     'to_cwl_job',
     'to_galaxy_parameters',
     'needs_shell_quoting',
     'shellescape',
-]
+)

lib/galaxy/tools/cwl/cwltool_deps.py

@@ -4,6 +4,7 @@
 :func:`ensure_cwltool_available` before using any of the imported
 functionality at runtime.
 """
+import re
 
 try:
     import requests
@@ -40,8 +41,6 @@
     # Drop SyntaxError once schema_salad supports Python 3
     schema_salad = None
 
-import re
-
 needs_shell_quoting = re.compile(r"""(^$|[\s|&;()<>\'"$@])""").search
 
 
@@ -65,7 +64,7 @@ def ensure_cwltool_available():
         raise ImportError(message)
 
 
-__all__ = [
+__all__ = (
     'main',
     'load_tool',
     'workflow',
@@ -74,4 +73,4 @@ def ensure_cwltool_available():
     'schema_salad',
     'shellescape',
     'needs_shell_quoting',
-]
+)