Merging with dev

.ci/flake8_lint_include_list.txt

@@ -264,35 +264,7 @@ lib/galaxy/sample_tracking/__init__.py
 lib/galaxy/sample_tracking/sample.py
 lib/galaxy/security/validate_user_input.py
 lib/galaxy/tags/
-lib/galaxy/tools/actions/
-lib/galaxy/tools/cwl/
-lib/galaxy/tools/data_manager/__init__.py
-lib/galaxy/tools/deps/
-lib/galaxy/tools/exception_handling.py
-lib/galaxy/tools/execute.py
-lib/galaxy/tools/filters/
-lib/galaxy/tools/imp_exp/export_history.py
-lib/galaxy/tools/imp_exp/__init__.py
-lib/galaxy/tools/linters/
-lib/galaxy/tools/lint.py
-lib/galaxy/tools/lint_util.py
-lib/galaxy/tools/loader_directory.py
-lib/galaxy/tools/loader.py
-lib/galaxy/tools/parameters/dataset_matcher.py
-lib/galaxy/tools/parameters/history_query.py
-lib/galaxy/tools/parameters/__init__.py
-lib/galaxy/tools/parameters/input_translation.py
-lib/galaxy/tools/parameters/sanitize.py
-lib/galaxy/tools/parameters/validation.py
-lib/galaxy/tools/parameters/wrapped_json.py
-lib/galaxy/tools/parameters/wrapped.py
-lib/galaxy/tools/parser/
-lib/galaxy/tools/special_tools.py
-lib/galaxy/tools/test.py
-lib/galaxy/tools/toolbox/
-lib/galaxy/tools/util/galaxyops/
-lib/galaxy/tools/util/__init__.py
-lib/galaxy/tools/verify/
+lib/galaxy/tools/
 lib/galaxy/util/
 lib/galaxy_utils/__init__.py
 lib/galaxy/util/sleeper.py

.ci/py3_sources.txt

@@ -40,26 +40,7 @@ lib/galaxy/quota/
 lib/galaxy/sample_tracking/
 lib/galaxy/security/
 lib/galaxy/tags/
-lib/galaxy/tools/actions/
-lib/galaxy/tools/cwl/
-lib/galaxy/tools/deps/
-lib/galaxy/tools/exception_handling.py
-lib/galaxy/tools/execute.py
-lib/galaxy/tools/lint.py
-lib/galaxy/tools/lint_util.py
-lib/galaxy/tools/linters/
-lib/galaxy/tools/loader.py
-lib/galaxy/tools/loader_directory.py
-lib/galaxy/tools/parameters/dataset_matcher.py
-lib/galaxy/tools/parameters/__init__.py
-lib/galaxy/tools/parameters/input_translation.py
-lib/galaxy/tools/parameters/sanitize.py
-lib/galaxy/tools/parameters/validation.py
-lib/galaxy/tools/parameters/wrapped_json.py
-lib/galaxy/tools/parameters/wrapped.py
-lib/galaxy/tools/parser/
-lib/galaxy/tools/test.py
-lib/galaxy/tools/toolbox/
+lib/galaxy/tools/
 lib/galaxy/tours/
 lib/galaxy/util/
 lib/galaxy/visualization/

lib/galaxy/jobs/__init__.py

@@ -243,7 +243,7 @@ def __parse_job_conf_xml(self, tree):
             job_destination['env'] = self.__get_envs(destination)
             destination_resubmits = self.__get_resubmits(destination)
             if destination_resubmits:
-                resubmits = self.default_resubmits
+                resubmits = destination_resubmits
             else:
                 resubmits = self.default_resubmits
             job_destination["resubmit"] = resubmits

lib/galaxy/tools/data/__init__.py

@@ -6,21 +6,20 @@
 to modify the tool configurations.
 """
 
+import hashlib
 import logging
 import os
 import os.path
 import re
 import string
-import hashlib
-
 from glob import glob
 from tempfile import NamedTemporaryFile
-from urllib2 import urlopen
 
-from galaxy import util
-from galaxy.util.odict import odict
+from six.moves.urllib.request import urlopen
 
+from galaxy import util
 from galaxy.util.dictifiable import Dictifiable
+from galaxy.util.odict import odict
 
 log = logging.getLogger( __name__ )
 
@@ -158,15 +157,15 @@ def to_xml_file( self, shed_tool_data_table_config, new_elems=None, remove_elems
             for elem in out_elems:
                 out.write( util.xml_to_string( elem, pretty=True ) )
             out.write( '</tables>\n' )
-        os.chmod( full_path, 0644 )
+        os.chmod( full_path, 0o644 )
 
     def reload_tables( self, table_names=None ):
         """
         Reload tool data tables.
         """
         tables = self.get_tables()
         if not table_names:
-            table_names = tables.keys()
+            table_names = list(tables.keys())
         elif not isinstance( table_names, list ):
             table_names = [ table_names ]
         for table_name in table_names:
@@ -349,15 +348,15 @@ def configure_and_load( self, config_element, tool_data_path, from_shed_config=F
                 self.filenames[ filename ] = dict( found=found, filename=filename, from_shed_config=from_shed_config, tool_data_path=tool_data_path,
                                                    config_element=config_element, tool_shed_repository=repo_info, errors=errors )
             else:
-                log.debug( "Filename '%s' already exists in filenames (%s), not adding", filename, self.filenames.keys() )
+                log.debug( "Filename '%s' already exists in filenames (%s), not adding", filename, list(self.filenames.keys()) )
             # Remove URL tmp file
             if tmp_file is not None:
                 tmp_file.close()
 
     def merge_tool_data_table( self, other_table, allow_duplicates=True, persist=False, persist_on_error=False, entry_source=None, **kwd ):
         assert self.columns == other_table.columns, "Merging tabular data tables with non matching columns is not allowed: %s:%s != %s:%s" % ( self.name, self.columns, other_table.name, other_table.columns )
         # merge filename info
-        for filename, info in other_table.filenames.iteritems():
+        for filename, info in other_table.filenames.items():
             if filename not in self.filenames:
                 self.filenames[ filename ] = info
         # save info about table
@@ -473,7 +472,7 @@ def get_column_name_list( self ):
         rval = []
         for i in range( self.largest_index + 1 ):
             found_column = False
-            for name, index in self.columns.iteritems():
+            for name, index in self.columns.items():
                 if index == i:
                     if not found_column:
                         rval.append( name )
@@ -530,7 +529,7 @@ def get_filename_for_source( self, source, default=None ):
         else:
             source_repo_info = None
         filename = default
-        for name, value in self.filenames.iteritems():
+        for name, value in self.filenames.items():
             repo_info = value.get( 'tool_shed_repository', None )
             if ( not source_repo_info and not repo_info ) or ( source_repo_info and repo_info and source_repo_info == repo_info ):
                 filename = name
@@ -637,7 +636,7 @@ def _replace_field_separators( self, fields, separator=None, replace=None, comme
                     replace = "_"
                 else:
                     replace = " "
-        return map( lambda x: x.replace( separator, replace ), fields )
+        return [x.replace( separator, replace ) for x in fields]
 
     def _deduplicate_data( self ):
         # Remove duplicate entries, without recreating self.data object

lib/galaxy/tools/data_manager/manager.py

@@ -1,20 +1,24 @@
 import errno
 import json
+import logging
 import os
+
 from six import string_types
 
 from galaxy import util
-from galaxy.util.odict import odict
-from galaxy.util.template import fill_template
+from galaxy.queue_worker import (
+    reload_data_managers,
+    send_control_task
+)
 from galaxy.tools.data import TabularToolDataTable
 from galaxy.tools.toolbox.watcher import get_tool_conf_watcher
-from tool_shed.util import common_util
-from tool_shed.util import repository_util
-from galaxy.queue_worker import reload_data_managers
-from galaxy.queue_worker import send_control_task
+from galaxy.util.odict import odict
+from galaxy.util.template import fill_template
+from tool_shed.util import (
+    common_util,
+    repository_util
+)
 
-# set up logger
-import logging
 log = logging.getLogger( __name__ )
 
 SUPPORTED_DATA_TABLE_TYPES = ( TabularToolDataTable )
@@ -112,7 +116,7 @@ def remove_manager( self, manager_ids ):
                 # determine if any data_tables are no longer tracked
                 for data_table_name in data_manager.data_tables.keys():
                     remove_data_table_tracking = True
-                    for other_data_manager in self.data_managers.itervalues():
+                    for other_data_manager in self.data_managers.values():
                         if data_table_name in other_data_manager.data_tables:
                             remove_data_table_tracking = False
                             break
@@ -279,21 +283,21 @@ def process_result( self, out_data ):
         data_manager_dicts = {}
         data_manager_dict = {}
         # TODO: fix this merging below
-        for output_name, output_dataset in out_data.iteritems():
+        for output_name, output_dataset in out_data.items():
             try:
                 output_dict = json.loads( open( output_dataset.file_name ).read() )
             except Exception as e:
                 log.warning( 'Error reading DataManagerTool json for "%s": %s' % ( output_name, e ) )
                 continue
             data_manager_dicts[ output_name ] = output_dict
-            for key, value in output_dict.iteritems():
+            for key, value in output_dict.items():
                 if key not in data_manager_dict:
                     data_manager_dict[ key ] = {}
                 data_manager_dict[ key ].update( value )
             data_manager_dict.update( output_dict )
 
         data_tables_dict = data_manager_dict.get( 'data_tables', {} )
-        for data_table_name in self.data_tables.iterkeys():
+        for data_table_name in self.data_tables.keys():
             data_table_values = data_tables_dict.pop( data_table_name, None )
             if not data_table_values:
                 log.warning( 'No values for data table "%s" were returned by the data manager "%s".' % ( data_table_name, self.id ) )
@@ -307,7 +311,7 @@ def process_result( self, out_data ):
                 continue  # next table name
             output_ref_values = {}
             if data_table_name in self.output_ref_by_data_table:
-                for data_table_column, output_ref in self.output_ref_by_data_table[ data_table_name ].iteritems():
+                for data_table_column, output_ref in self.output_ref_by_data_table[ data_table_name ].items():
                     output_ref_dataset = out_data.get( output_ref, None )
                     assert output_ref_dataset is not None, "Referenced output was not found."
                     output_ref_values[ data_table_column ] = output_ref_dataset
@@ -316,7 +320,7 @@ def process_result( self, out_data ):
                 data_table_values = [ data_table_values ]
             for data_table_row in data_table_values:
                 data_table_value = dict( **data_table_row )  # keep original values here
-                for name, value in data_table_row.iteritems():  # FIXME: need to loop through here based upon order listed in data_manager config
+                for name, value in data_table_row.items():  # FIXME: need to loop through here based upon order listed in data_manager config
                     if name in output_ref_values:
                         self.process_move( data_table_name, name, output_ref_values[ name ].extra_files_path, **data_table_value )
                         data_table_value[ name ] = self.process_value_translation( data_table_name, name, **data_table_value )
@@ -332,21 +336,21 @@ def process_result( self, out_data ):
             for ref_file in out_data.values():
                 util.move_merge( ref_file.extra_files_path, self.data_managers.app.config.galaxy_data_manager_data_path )
             path_column_names = [ 'path' ]
-            for data_table_name, data_table_values in data_tables_dict.iteritems():
+            for data_table_name, data_table_values in data_tables_dict.items():
                 data_table = self.data_managers.app.tool_data_tables.get( data_table_name, None )
                 if not isinstance( data_table_values, list ):
                     data_table_values = [ data_table_values ]
                 for data_table_row in data_table_values:
                     data_table_value = dict( **data_table_row )  # keep original values here
-                    for name, value in data_table_row.iteritems():
+                    for name, value in data_table_row.items():
                         if name in path_column_names:
                             data_table_value[ name ] = os.path.abspath( os.path.join( self.data_managers.app.config.galaxy_data_manager_data_path, value ) )
                     data_table.add_entry( data_table_value, persist=True, entry_source=self )
                     send_control_task(self.data_managers.app, 'reload_tool_data_tables',
                                       noop_self=True,
                                       kwargs={'table_name': data_table_name} )
         else:
-            for data_table_name, data_table_values in data_tables_dict.iteritems():
+            for data_table_name, data_table_values in data_tables_dict.items():
                 # tool returned extra data table entries, but data table was not declared in data manager
                 # do not add these values, but do provide messages
                 log.warning( 'The data manager "%s" returned an undeclared data table "%s" with new entries "%s". These entries will not be created. Please confirm that an entry for "%s" exists in your "%s" file.' % ( self.id, data_table_name, data_table_values, data_table_name, self.data_managers.filename ) )

lib/galaxy/tools/deps/container_resolvers/mulled.py

@@ -33,7 +33,7 @@
 
 
 def list_cached_mulled_images(namespace=None):
-    command = build_docker_images_command(truncate=True, sudo_docker=False)
+    command = build_docker_images_command(truncate=True, sudo=False)
     command = "%s | tail -n +2 | tr -s ' ' | cut -d' ' -f1,2" % command
     images_and_versions = check_output(command)
     name_filter = get_filter(namespace)

lib/galaxy/tools/deps/resolvers/resolver_mixins.py

@@ -41,7 +41,7 @@ def _installed_versions(self, recipe):
             return []
 
         names = os.listdir(recipe_base_path)
-        return filter(lambda n: os.path.isdir(os.path.join(recipe_base_path, n)), names)
+        return [n for n in names if os.path.isdir(os.path.join(recipe_base_path, n))]
 
 
 class UsesToolDependencyDirMixin:
@@ -53,14 +53,10 @@ def _init_base_path(self, dependency_manager, **kwds):
 class UsesInstalledRepositoriesMixin:
 
     def _get_installed_dependency( self, name, type, version=None, **kwds ):
-        installed_tool_dependencies = kwds.get("installed_tool_dependencies", [])
-        for installed_tool_dependency in (installed_tool_dependencies or []):
-            name_and_type_equal = installed_tool_dependency.name == name and installed_tool_dependency.type == type
-            if version:
-                if name_and_type_equal and installed_tool_dependency.version == version:
-                    return installed_tool_dependency
-            else:
-                if name_and_type_equal:
+        installed_tool_dependencies = kwds.get("installed_tool_dependencies") or []
+        for installed_tool_dependency in installed_tool_dependencies:
+            if installed_tool_dependency.name == name and installed_tool_dependency.type == type:
+                if not version or installed_tool_dependency.version == version:
                     return installed_tool_dependency
         return None