Python3 support and import order fixes for lib/tool_shed/

.ci/flake8_lint_include_list.txt

@@ -384,45 +384,7 @@ lib/galaxy/workflow/steps.py
 lib/galaxy/work/__init__.py
 lib/mimeparse.py
 lib/psyco_full.py
-lib/tool_shed/capsule/__init__.py
-lib/tool_shed/dependencies/__init__.py
-lib/tool_shed/dependencies/repository/__init__.py
-lib/tool_shed/dependencies/tool/
-lib/tool_shed/galaxy_install/datatypes/__init__.py
-lib/tool_shed/galaxy_install/grids/__init__.py
-lib/tool_shed/galaxy_install/__init__.py
-lib/tool_shed/galaxy_install/metadata/__init__.py
-lib/tool_shed/galaxy_install/migrate/__init__.py
-lib/tool_shed/galaxy_install/migrate/versions/
-lib/tool_shed/galaxy_install/repository_dependencies/__init__.py
-lib/tool_shed/galaxy_install/tool_dependencies/env_manager.py
-lib/tool_shed/galaxy_install/tool_dependencies/__init__.py
-lib/tool_shed/galaxy_install/tool_dependencies/recipe/asynchronous_reader.py
-lib/tool_shed/galaxy_install/tool_dependencies/recipe/env_file_builder.py
-lib/tool_shed/galaxy_install/tool_dependencies/recipe/__init__.py
-lib/tool_shed/galaxy_install/tool_dependencies/recipe/recipe_manager.py
-lib/tool_shed/galaxy_install/tools/
-lib/tool_shed/galaxy_install/utility_containers/
-lib/tool_shed/grids/__init__.py
-lib/tool_shed/grids/repository_grid_filter_manager.py
-lib/tool_shed/__init__.py
-lib/tool_shed/managers/groups.py
-lib/tool_shed/managers/__init__.py
-lib/tool_shed/metadata/__init__.py
-lib/tool_shed/repository_types/__init__.py
-lib/tool_shed/repository_types/metadata.py
-lib/tool_shed/repository_types/unrestricted.py
-lib/tool_shed/repository_types/util.py
-lib/tool_shed/tools/data_table_manager.py
-lib/tool_shed/tools/__init__.py
-lib/tool_shed/tools/tool_version_manager.py
-lib/tool_shed/util/basic_util.py
-lib/tool_shed/util/container_util.py
-lib/tool_shed/util/encoding_util.py
-lib/tool_shed/util/__init__.py
-lib/tool_shed/util/search_util.py
-lib/tool_shed/util/web_util.py
-lib/tool_shed/util/xml_util.py
+lib/tool_shed/
 scripts/api/common.py
 scripts/api/copy_hda_to_library_folder.py
 scripts/api/create.py

.ci/py3_sources.txt

@@ -50,15 +50,7 @@ lib/galaxy_ext/
 lib/galaxy_utils/
 lib/log_tempfile.py
 lib/psyco_full.py
-lib/tool_shed/capsule/
-lib/tool_shed/dependencies/
-lib/tool_shed/grids/
-lib/tool_shed/managers/
-lib/tool_shed/metadata/
-lib/tool_shed/repository_types/
-lib/tool_shed/tools/
-lib/tool_shed/util/
-lib/tool_shed/utility_containers/
+lib/tool_shed/
 scripts/api/
 scripts/auth/
 scripts/bootstrap_history.py

lib/tool_shed/capsule/capsule_manager.py

@@ -5,32 +5,21 @@
 import tarfile
 import tempfile
 import threading
-import urllib
-from time import gmtime
-from time import strftime
+from time import gmtime, strftime
 
+from six.moves.urllib.request import urlopen
 from sqlalchemy import and_, false
 
 import tool_shed.repository_types.util as rt_util
 from galaxy import web
-from galaxy.util import asbool
-from galaxy.util import CHUNK_SIZE
-from galaxy.util import safe_relpath
-from galaxy.util import build_url
+from galaxy.util import asbool, build_url, CHUNK_SIZE, safe_relpath
 from galaxy.util.odict import odict
-from tool_shed.dependencies.repository.relation_builder import RelationBuilder
 from tool_shed.dependencies import attribute_handlers
+from tool_shed.dependencies.repository.relation_builder import RelationBuilder
 from tool_shed.galaxy_install.repository_dependencies.repository_dependency_manager import RepositoryDependencyInstallManager
 from tool_shed.metadata import repository_metadata_manager
-from tool_shed.util import basic_util
-from tool_shed.util import commit_util
-from tool_shed.util import common_util
-from tool_shed.util import encoding_util
-from tool_shed.util import hg_util
-from tool_shed.util import metadata_util
-from tool_shed.util import repository_util
-from tool_shed.util import shed_util_common as suc
-from tool_shed.util import xml_util
+from tool_shed.util import (basic_util, commit_util, common_util, encoding_util,
+    hg_util, metadata_util, repository_util, shed_util_common as suc, xml_util)
 
 log = logging.getLogger( __name__ )
 
@@ -822,7 +811,7 @@ def upload_capsule( self, **kwd ):
         if url:
             valid_url = True
             try:
-                stream = urllib.urlopen( url )
+                stream = urlopen( url )
             except Exception as e:
                 valid_url = False
                 return_dict['error_message'] = 'Error importing file via http: %s' % str( e )

lib/tool_shed/dependencies/repository/relation_builder.py

@@ -1,10 +1,9 @@
 import logging
 
-from galaxy.util import asbool, listify
-
 import tool_shed.util.repository_util
-from tool_shed.util import common_util, container_util, hg_util, metadata_util
-from tool_shed.util import shed_util_common as suc
+from galaxy.util import asbool, listify
+from tool_shed.util import (common_util, container_util, hg_util, metadata_util,
+    shed_util_common as suc)
 
 log = logging.getLogger( __name__ )
 
@@ -28,11 +27,11 @@ def __init__( self, app, repository, repository_metadata, tool_shed_url ):
 
     def can_add_to_key_rd_dicts( self, key_rd_dict, key_rd_dicts ):
         """Handle the case where an update to the changeset revision was done."""
-        k = key_rd_dict.keys()[ 0 ]
+        k = next(iter(key_rd_dict))
         rd = key_rd_dict[ k ]
         partial_rd = rd[ 0:3 ]
         for kr_dict in key_rd_dicts:
-            key = kr_dict.keys()[ 0 ]
+            key = next(iter(kr_dict))
             if key == k:
                 repository_dependency = kr_dict[ key ]
                 if repository_dependency[ 0:3 ] == partial_rd:
@@ -175,7 +174,7 @@ def get_repository_dependency_as_key( self, repository_dependency ):
     def get_updated_changeset_revisions_for_repository_dependencies( self, key_rd_dicts ):
         updated_key_rd_dicts = []
         for key_rd_dict in key_rd_dicts:
-            key = key_rd_dict.keys()[ 0 ]
+            key = next(iter(key_rd_dict))
             repository_dependency = key_rd_dict[ key ]
             rd_toolshed, rd_name, rd_owner, rd_changeset_revision, \
                 rd_prior_installation_required, \
@@ -293,7 +292,7 @@ def handle_key_rd_dicts_for_repository( self, current_repository_key, repository
     def handle_next_repository_dependency( self ):
         next_repository_key_rd_dict = self.key_rd_dicts_to_be_processed.pop( 0 )
         next_repository_key_rd_dicts = [ next_repository_key_rd_dict ]
-        next_repository_key = next_repository_key_rd_dict.keys()[ 0 ]
+        next_repository_key = next(iter(next_repository_key_rd_dict))
         self.handle_key_rd_dicts_for_repository( next_repository_key, next_repository_key_rd_dicts )
         return self.get_repository_dependencies_for_changeset_revision()
 
@@ -326,7 +325,7 @@ def in_circular_repository_dependencies( self, repository_key_rd_dict ):
 
     def in_key_rd_dicts( self, key_rd_dict, key_rd_dicts ):
         """Return True if key_rd_dict is contained in the list of key_rd_dicts."""
-        k = key_rd_dict.keys()[ 0 ]
+        k = next(iter(key_rd_dict))
         v = key_rd_dict[ k ]
         for key_rd_dict in key_rd_dicts:
             for key, val in key_rd_dict.items():
@@ -436,11 +435,11 @@ def prune_invalid_repository_dependencies( self, repository_dependencies ):
 
     def remove_from_key_rd_dicts( self, key_rd_dict, key_rd_dicts ):
         """Eliminate the key_rd_dict from the list of key_rd_dicts if it is contained in the list."""
-        k = key_rd_dict.keys()[ 0 ]
+        k = next(iter(key_rd_dict))
         v = key_rd_dict[ k ]
         clean_key_rd_dicts = []
         for krd_dict in key_rd_dicts:
-            key = krd_dict.keys()[ 0 ]
+            key = next(iter(krd_dict))
             val = krd_dict[ key ]
             if key == k and val == v:
                 continue
@@ -450,15 +449,15 @@ def remove_from_key_rd_dicts( self, key_rd_dict, key_rd_dicts ):
     def remove_repository_dependency_reference_to_self( self, key_rd_dicts ):
         """Remove all repository dependencies that point to a revision within its own repository."""
         clean_key_rd_dicts = []
-        key = key_rd_dicts[ 0 ].keys()[ 0 ]
+        key = next(iter(key_rd_dicts[ 0 ]))
         repository_tup = key.split( container_util.STRSEP )
         rd_toolshed, rd_name, rd_owner, rd_changeset_revision, \
             rd_prior_installation_required, \
             rd_only_if_compiling_contained_td = \
             common_util.parse_repository_dependency_tuple( repository_tup )
         cleaned_rd_toolshed = common_util.remove_protocol_from_tool_shed_url( rd_toolshed )
         for key_rd_dict in key_rd_dicts:
-            k = key_rd_dict.keys()[ 0 ]
+            k = next(iter(key_rd_dict))
             repository_dependency = key_rd_dict[ k ]
             toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \
                 common_util.parse_repository_dependency_tuple( repository_dependency )

lib/tool_shed/galaxy_install/datatypes/custom_datatype_manager.py

@@ -91,7 +91,7 @@ def alter_config_and_load_prorietary_datatypes( self, datatypes_config, relative
             os.write( fd, '%s' % xml_util.xml_to_string( sniffers ) )
         os.write( fd, '</datatypes>\n' )
         os.close( fd )
-        os.chmod( proprietary_datatypes_config, 0644 )
+        os.chmod( proprietary_datatypes_config, 0o644 )
         # Load custom datatypes
         self.app.datatypes_registry.load_datatypes( root_dir=self.app.config.root,
                                                     config=proprietary_datatypes_config,

lib/tool_shed/galaxy_install/grids/admin_toolshed_grids.py

@@ -5,9 +5,8 @@
 from galaxy import util
 from galaxy.model import tool_shed_install
 from galaxy.web import url_for
-from galaxy.web.framework.helpers import iff, grids
-from tool_shed.util import tool_dependency_util
-from tool_shed.util import repository_util
+from galaxy.web.framework.helpers import grids, iff
+from tool_shed.util import repository_util, tool_dependency_util
 
 log = logging.getLogger( __name__ )
 

lib/tool_shed/galaxy_install/install_manager.py

@@ -11,10 +11,6 @@
 
 from galaxy import exceptions, util
 from galaxy.tools.deps import views
-from tool_shed.util import basic_util, common_util, encoding_util, hg_util, repository_util
-from tool_shed.util import shed_util_common as suc, tool_dependency_util
-from tool_shed.util import tool_util, xml_util
-
 from tool_shed.galaxy_install.datatypes import custom_datatype_manager
 from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager
 from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager
@@ -23,8 +19,10 @@
 from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import StepManager
 from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import TagManager
 from tool_shed.galaxy_install.tools import data_manager, tool_panel_manager
-
 from tool_shed.tools import data_table_manager, tool_version_manager
+from tool_shed.util import basic_util, common_util, encoding_util, hg_util, repository_util
+from tool_shed.util import shed_util_common as suc, tool_dependency_util
+from tool_shed.util import tool_util, xml_util
 
 log = logging.getLogger( __name__ )
 
@@ -478,8 +476,8 @@ def __get_install_info_from_tool_shed( self, tool_shed_url, name, owner, changes
             repository_revision_dict = items[ 1 ]
             repo_info_dict = items[ 2 ]
         else:
-            message = "Unable to retrieve installation information from tool shed %s for revision %s of repository %s owned by %s: %s" % \
-                ( str( tool_shed_url ), str( changeset_revision ), str( name ), str( owner ), str( e ) )
+            message = "Unable to retrieve installation information from tool shed %s for revision %s of repository %s owned by %s" % \
+                ( str( tool_shed_url ), str( changeset_revision ), str( name ), str( owner ) )
             log.warning( message )
             raise exceptions.InternalServerError( message )
         # Make sure the tool shed returned everything we need for installing the repository.

lib/tool_shed/galaxy_install/installed_repository_manager.py

@@ -8,19 +8,18 @@
 from sqlalchemy import and_, false, true
 
 from galaxy import util
+from tool_shed.galaxy_install.datatypes import custom_datatype_manager
+from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager
+from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager
+from tool_shed.galaxy_install.tools import data_manager
+from tool_shed.galaxy_install.tools import tool_panel_manager
 from tool_shed.util import common_util
 from tool_shed.util import container_util
 from tool_shed.util import repository_util
 from tool_shed.util import shed_util_common as suc
 from tool_shed.util import tool_dependency_util
 from tool_shed.util import xml_util
 
-from tool_shed.galaxy_install.datatypes import custom_datatype_manager
-from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager
-from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager
-from tool_shed.galaxy_install.tools import data_manager
-from tool_shed.galaxy_install.tools import tool_panel_manager
-
 log = logging.getLogger( __name__ )
 
 
@@ -242,7 +241,7 @@ def get_dependencies_for_repository( self, tool_shed_url, repo_info_dict, includ
         installed_td = {}
         missing_rd = {}
         missing_td = {}
-        name = repo_info_dict.keys()[ 0 ]
+        name = next(iter(repo_info_dict))
         repo_info_tuple = repo_info_dict[ name ]
         description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = \
             repository_util.get_repo_info_tuple_contents( repo_info_tuple )

lib/tool_shed/galaxy_install/migrate/common.py

@@ -1,9 +1,12 @@
-import ConfigParser
+from __future__ import print_function
+
 import os
 import sys
 
+from six.moves import configparser
+
 import galaxy.config
-from tool_shed.galaxy_install import tool_migration_manager, installed_repository_manager
+from tool_shed.galaxy_install import installed_repository_manager, tool_migration_manager
 
 
 class MigrateToolsApplication( object, galaxy.config.ConfiguresGalaxyMixin ):
@@ -18,9 +21,9 @@ def __init__( self, tools_migration_config ):
             sys.argv.pop( pos )
             galaxy_config_file = sys.argv.pop( pos )
         if not os.path.exists( galaxy_config_file ):
-            print "Galaxy config file does not exist (hint: use '-c config.ini' for non-standard locations): %s" % galaxy_config_file
+            print("Galaxy config file does not exist (hint: use '-c config.ini' for non-standard locations): %s" % galaxy_config_file)
             sys.exit( 1 )
-        config_parser = ConfigParser.ConfigParser( { 'here': os.getcwd() } )
+        config_parser = configparser.ConfigParser( { 'here': os.getcwd() } )
         config_parser.read( galaxy_config_file )
         galaxy_config_dict = {}
         for key, value in config_parser.items( "app:main" ):

lib/tool_shed/galaxy_install/migrate/versions/0001_tools.py

@@ -1,10 +1,11 @@
 """
 Initialize the version column of the migrate_tools database table to 1.  No tool migrations are handled in this version.
 """
+from __future__ import print_function
 
 
 def upgrade(migrate_engine):
-    print __doc__
+    print(__doc__)
 
 
 def downgrade(migrate_engine):

lib/tool_shed/galaxy_install/migrate/versions/0002_tools.py

@@ -4,10 +4,11 @@
 repositories named emboss_5 and emboss_datatypes from the main Galaxy tool shed at http://toolshed.g2.bx.psu.edu
 will be installed into your local Galaxy instance at the location discussed above by running the following command.
 """
+from __future__ import print_function
 
 
 def upgrade(migrate_engine):
-    print __doc__
+    print(__doc__)
 
 
 def downgrade(migrate_engine):

lib/tool_shed/galaxy_install/migrate/versions/0003_tools.py

@@ -3,10 +3,11 @@
 Galaxy tool shed at http://toolshed.g2.bx.psu.edu will be installed into your local Galaxy instance at the
 location discussed above by running the following command.
 """
+from __future__ import print_function
 
 
 def upgrade(migrate_engine):
-    print __doc__
+    print(__doc__)
 
 
 def downgrade(migrate_engine):

lib/tool_shed/galaxy_install/migrate/versions/0004_tools.py

@@ -5,10 +5,11 @@
 These repositories will be installed into your local Galaxy instance at the
 location discussed above by running the following command.
 """
+from __future__ import print_function
 
 
 def upgrade(migrate_engine):
-    print __doc__
+    print(__doc__)
 
 
 def downgrade(migrate_engine):

lib/tool_shed/galaxy_install/migrate/versions/0005_tools.py

@@ -6,10 +6,11 @@
 Galaxy instance at the location discussed above by running the following
 command.
 """
+from __future__ import print_function
 
 
 def upgrade(migrate_engine):
-    print __doc__
+    print(__doc__)
 
 
 def downgrade(migrate_engine):

lib/tool_shed/galaxy_install/migrate/versions/0006_tools.py

@@ -11,10 +11,11 @@
 local Galaxy instance at the location discussed above by running
 the following command.
 """
+from __future__ import print_function
 
 
 def upgrade(migrate_engine):
-    print __doc__
+    print(__doc__)
 
 
 def downgrade(migrate_engine):