Cleanup and fix tests.

lib/galaxy/util/__init__.py

@@ -30,6 +30,7 @@
 from six import binary_type, iteritems, PY3, string_types, text_type
 from six.moves import email_mime_text, xrange, zip
 from six.moves.urllib import parse as urlparse
+from six.moves.urllib import request as urlrequest
 
 try:
     import docutils.core as docutils_core
@@ -1342,20 +1343,40 @@ def parse_int(value, min_val=None, max_val=None, default=None, allow_none=False)
             raise
 
 
-def url_get( app, base_url, toolshed=True, pathspec=None, params=None ):
+def build_url( base_url, port=80, scheme='http', pathspec=None, params=None, doseq=False ):
+    if params is None:
+        params = dict()
+    if pathspec is None:
+        pathspec = []
+    parsed_url = urlparse.urlparse( base_url )
+    if scheme != 'http':
+        parsed_url.scheme = scheme
+    if port != 80:
+        url = '%s://%s:%d/%s' % ( parsed_url.scheme, parsed_url.netloc.rstrip( '/' ), int( port ), parsed_url.path )
+    else:
+        url = '%s://%s/%s' % ( parsed_url.scheme, parsed_url.netloc.rstrip( '/' ), parsed_url.path.lstrip( '/' ) )
+    if len( pathspec ) > 0:
+        url = '%s/%s' % ( url.rstrip( '/' ), '/'.join( pathspec ) )
+    if parsed_url.query:
+        for query_parameter in parsed_url.query.split( '&' ):
+            key, value = query_parameter.split( '=' )
+            params[ key ] = value
+    if params:
+        url += '?%s' % urlparse.urlencode( params, doseq=doseq )
+    return url
+
+
+def url_get( base_url, password_mgr=None, pathspec=None, params=None ):
     """Make contact with the uri provided and return any contents."""
-    # urllib2 auto-detects system proxies, when passed a Proxyhandler.
-    # Refer: https://docs.python.org/2/howto/urllib2.html#proxies
-    proxy = urllib2.ProxyHandler()
-    urlopener = urllib2.build_opener( proxy )
-    urllib2.install_opener( urlopener )
-    if toolshed:
-        registry = app.tool_shed_registry
-        password_mgr = registry.password_manager_for_url( base_url )
-        if password_mgr is not None:
-            auth_handler = urllib2.HTTPBasicAuthHandler( password_mgr )
-            urlopener.add_handler( auth_handler )
-    full_url = url_join( base_url, pathspec=pathspec, params=params )
+    # Uses system proxy settings if they exist.
+    proxy = urlrequest.ProxyHandler()
+    if password_mgr is not None:
+        auth = urlrequest.HTTPDigestAuthHandler( password_mgr )
+        urlopener = urlrequest.build_opener( proxy, auth )
+    else:
+        urlopener = urlrequest.build_opener( proxy )
+    urlrequest.install_opener( urlopener )
+    full_url = build_url( base_url, pathspec=pathspec, params=params )
     response = urlopener.open( full_url )
     content = response.read()
     response.close()

lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py

@@ -100,7 +100,7 @@ def get_latest_installable_revision( self, trans, payload, **kwd ):
         params = dict(name=name, owner=owner)
         pathspec = ['api', 'repositories', 'get_ordered_installable_revisions']
         try:
-            raw_text = util.url_get( trans.app, tool_shed_url, pathspec, params )
+            raw_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
         except Exception, e:
             message = "Error attempting to retrieve the latest installable revision from tool shed %s for repository %s owned by %s: %s" % \
                 ( str( tool_shed_url ), str( name ), str( owner ), str( e ) )

lib/galaxy/webapps/galaxy/controllers/admin.py

@@ -11,7 +11,7 @@
 from galaxy.exceptions import MessageException
 from galaxy.model import tool_shed_install as install_model
 from galaxy.model.util import pgcalc
-from galaxy.util import nice_size, sanitize_text
+from galaxy.util import nice_size, sanitize_text, url_get
 from galaxy.util.odict import odict
 from galaxy.web import url_for
 from galaxy.web.base.controller import BaseUIController, UsesQuotaMixin
@@ -770,9 +770,9 @@ def check_for_tool_dependencies( self, trans, migration_stage ):
         tree = galaxy.util.parse_xml( tools_xml_file_path )
         root = tree.getroot()
         tool_shed = root.get( 'name' )
-        tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, tool_shed )
+        shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, tool_shed )
         repo_name_dependency_tups = []
-        if tool_shed_url:
+        if shed_url:
             for elem in root:
                 if elem.tag == 'repository':
                     tool_dependencies = []
@@ -781,7 +781,7 @@ def check_for_tool_dependencies( self, trans, migration_stage ):
                     changeset_revision = elem.get( 'changeset_revision' )
                     params = dict( name=repository_name, owner='devteam', changeset_revision=changeset_revision )
                     pathspec = [ 'repository', 'get_tool_dependencies' ]
-                    text = url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params )
+                    text = url_get( shed_url, password_mgr=self.app.tool_shed_registry.url_auth( shed_url ), pathspec=pathspec, params=params )
                     if text:
                         tool_dependencies_dict = encoding_util.tool_shed_decode( text )
                         for dependency_key, requirements_dict in tool_dependencies_dict.items():

lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py

@@ -189,7 +189,7 @@ def browse_tool_shed( self, trans, **kwd ):
         tool_shed_url = kwd.get( 'tool_shed_url', '' )
         tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, tool_shed_url )
         params = dict( galaxy_url=web.url_for( '/', qualified=True ) )
-        url = common_util.url_join( tool_shed_url, pathspec=[ 'repository', 'browse_valid_categories' ], params=params )
+        url = util.build_url( tool_shed_url, pathspec=[ 'repository', 'browse_valid_categories' ], params=params )
         return trans.response.send_redirect( url )
 
     @web.expose
@@ -212,7 +212,7 @@ def check_for_updates( self, trans, **kwd ):
                        owner=str( repository.owner ),
                        changeset_revision=str( repository.changeset_revision ) )
         pathspec = [ 'repository', 'check_for_updates' ]
-        url = common_util.url_join( tool_shed_url, pathspec=pathspec, params=params )
+        url = util.build_url( tool_shed_url, pathspec=pathspec, params=params )
         return trans.response.send_redirect( url )
 
     @web.expose
@@ -369,7 +369,7 @@ def find_tools_in_tool_shed( self, trans, **kwd ):
         tool_shed_url = kwd.get( 'tool_shed_url', '' )
         tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, tool_shed_url )
         params = dict( galaxy_url=web.url_for( '/', qualified=True ) )
-        url = common_util.url_join( tool_shed_url, pathspec=[ 'repository', 'find_tools' ], params=params )
+        url = util.build_url( tool_shed_url, pathspec=[ 'repository', 'find_tools' ], params=params )
         return trans.response.send_redirect( url )
 
     @web.expose
@@ -378,7 +378,7 @@ def find_workflows_in_tool_shed( self, trans, **kwd ):
         tool_shed_url = kwd.get( 'tool_shed_url', '' )
         tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, tool_shed_url )
         params = dict( galaxy_url=web.url_for( '/', qualified=True ) )
-        url = common_util.url_join( tool_shed_url, pathspec=[ 'repository', 'find_workflows' ], params=params )
+        url = util.build_url( tool_shed_url, pathspec=[ 'repository', 'find_workflows' ], params=params )
         return trans.response.send_redirect( url )
 
     @web.expose
@@ -413,7 +413,7 @@ def get_tool_dependencies( self, trans, repository_id, repository_name, reposito
             raise Exception( message )
         params = dict( name=repository_name, owner=repository_owner, changeset_revision=changeset_revision )
         pathspec = [ 'repository', 'get_tool_dependencies' ]
-        raw_text = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params )
+        raw_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
         if len( raw_text ) > 2:
             encoded_text = json.loads( raw_text )
             text = encoding_util.tool_shed_decode( encoded_text )
@@ -439,7 +439,7 @@ def get_updated_repository_information( self, trans, repository_id, repository_n
                        owner=str( repository_owner ),
                        changeset_revision=changeset_revision )
         pathspec = [ 'repository', 'get_updated_repository_information' ]
-        raw_text = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params )
+        raw_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
         repo_information_dict = json.loads( raw_text )
         return repo_information_dict
 
@@ -529,8 +529,8 @@ def install_latest_repository_revision( self, trans, **kwd ):
                                name=name,
                                owner=owner )
                 pathspec = [ 'repository', 'get_latest_downloadable_changeset_revision' ]
-                raw_text = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params )
-                url = common_util.url_join( tool_shed_url, pathspec=pathspec, params=params )
+                raw_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
+                url = util.build_url( tool_shed_url, pathspec=pathspec, params=params )
                 latest_downloadable_revision = json.loads( raw_text )
                 if latest_downloadable_revision == hg_util.INITIAL_CHANGELOG_HASH:
                     message = 'Error retrieving the latest downloadable revision for this repository via the url <b>%s</b>.' % url
@@ -541,7 +541,7 @@ def install_latest_repository_revision( self, trans, **kwd ):
                     # appropriate repository revision if one exists.  We need to create a temporary repo_info_tuple
                     # with the following entries to handle this.
                     # ( description, clone_url, changeset_revision, ctx_rev, owner, repository_dependencies, tool_dependencies )
-                    tmp_clone_url = common_util.url_join( tool_shed_url, pathspec=[ 'repos', owner, name ] )
+                    tmp_clone_url = util.build_url( tool_shed_url, pathspec=[ 'repos', owner, name ] )
                     tmp_repo_info_tuple = ( None, tmp_clone_url, latest_downloadable_revision, None, owner, None, None )
                     installed_repository, installed_changeset_revision = \
                         suc.repository_was_previously_installed( trans.app, tool_shed_url, name, tmp_repo_info_tuple, from_tip=False )
@@ -559,7 +559,7 @@ def install_latest_repository_revision( self, trans, **kwd ):
                                        changeset_revisions=str( latest_downloadable_revision ),
                                        galaxy_url=web.url_for( '/', qualified=True ) )
                         pathspec = [ 'repository', 'install_repositories_by_revision' ]
-                        url = common_util.url_join( tool_shed_url, pathspec=pathspec, params=params )
+                        url = util.build_url( tool_shed_url, pathspec=pathspec, params=params )
                         return trans.response.send_redirect( url )
             else:
                 message = 'Cannot locate installed tool shed repository with encoded id <b>%s</b>.' % str( repository_id )
@@ -727,7 +727,7 @@ def manage_repository( self, trans, **kwd ):
                            changeset_revisions=installed_changeset_revision,
                            galaxy_url=web.url_for( '/', qualified=True ) )
             pathspec = [ 'repository', 'install_repositories_by_revision' ]
-            url = common_util.url_join( tool_shed_url, pathspec=pathspec, params=params )
+            url = util.build_url( tool_shed_url, pathspec=pathspec, params=params )
             return trans.response.send_redirect( url )
         description = kwd.get( 'description', repository.description )
         shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( trans.app, repository )
@@ -985,7 +985,7 @@ def prepare_for_install( self, trans, **kwd ):
                 try:
                     params = dict( name=str( repository.name ), owner=str( repository.owner ) )
                     pathspec = [ 'repository', 'get_repository_id' ]
-                    repository_ids = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params )
+                    repository_ids = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
                 except Exception, e:
                     # The Tool Shed cannot handle the get_repository_id request, so the code must be older than the
                     # 04/2014 Galaxy release when it was introduced.  It will be safest to error out and let the
@@ -1005,7 +1005,7 @@ def prepare_for_install( self, trans, **kwd ):
             # Get the information necessary to install each repository.
             params = dict( repository_ids=str( repository_ids ), changeset_revisions=str( changeset_revisions ) )
             pathspec = [ 'repository', 'get_repository_information' ]
-            raw_text = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params )
+            raw_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
             repo_information_dict = json.loads( raw_text )
             for encoded_repo_info_dict in repo_information_dict.get( 'repo_info_dicts', [] ):
                 decoded_repo_info_dict = encoding_util.tool_shed_decode( encoded_repo_info_dict )
@@ -1539,7 +1539,7 @@ def reselect_tool_panel_section( self, trans, **kwd ):
                                owner=tool_shed_repository.owner,
                                changeset_revision=tool_shed_repository.installed_changeset_revision )
                 pathspec = [ 'repository', 'get_readme_files' ]
-                raw_text = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params )
+                raw_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
                 readme_files_dict = json.loads( raw_text )
                 tool_dependencies = metadata.get( 'tool_dependencies', None )
             rdim = repository_dependency_manager.RepositoryDependencyInstallManager( trans.app )
@@ -1739,7 +1739,7 @@ def set_tool_versions( self, trans, **kwd ):
         tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( repository.tool_shed ) )
         params = dict( name=repository.name, owner=repository.owner, changeset_revision=repository.changeset_revision )
         pathspec = [ 'repository', 'get_tool_versions' ]
-        text = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params )
+        text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
         if text:
             tool_version_dicts = json.loads( text )
             tvm = tool_version_manager.ToolVersionManager( trans.app )

lib/galaxy/webapps/galaxy/controllers/workflow.py

@@ -862,7 +862,7 @@ def import_workflow( self, trans, cntrller='workflow', **kwd ):
                            workflow_name=encoding_util.tool_shed_encode( workflow_name ),
                            open_for_url=True )
             pathspec = [ 'workflow', 'import_workflow' ]
-            workflow_text = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params )
+            workflow_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
             import_button = True
         if import_button:
             workflow_data = None

lib/galaxy/webapps/tool_shed/controllers/repository.py

@@ -809,7 +809,7 @@ def check_for_updates( self, trans, **kwd ):
                     return update
         params['latest_changeset_revision'] = str( latest_changeset_revision )
         params['latest_ctx_rev'] = str( update_to_ctx.rev() )
-        url = common_util.url_join( galaxy_url, pathspec=pathspec, params=params )
+        url = util.build_url( galaxy_url, pathspec=pathspec, params=params )
         return trans.response.send_redirect( url )
 
     @web.expose
@@ -1065,7 +1065,7 @@ def download( self, trans, repository_id, changeset_revision, file_type, **kwd )
         trans.sa_session.flush()
         tool_shed_url = web.url_for( '/', qualified=True )
         pathspec = [ 'repos', str( repository.user.username ), str( repository.name ), 'archive', file_type_str ]
-        download_url = common_util.url_join( tool_shed_url, pathspec=pathspec )
+        download_url = util.build_url( tool_shed_url, pathspec=pathspec )
         return trans.response.send_redirect( download_url )
 
     @web.expose
@@ -1534,7 +1534,7 @@ def get_functional_test_rss( self, trans, **kwd ):
                         time_tested = repository_metadata.time_last_tested.strftime( '%a, %d %b %Y %H:%M:%S UT' )
                     # Generate a citable URL for this repository with owner and changeset revision.
                     pathspec = [ 'view', str( user.username ), str( repository.name ), str( repository_metadata.changeset_revision ) ]
-                    repository_citable_url = common_util.url_join( tool_shed_url, pathspec=pathspec )
+                    repository_citable_url = util.build_url( tool_shed_url, pathspec=pathspec )
                     passed_tests = len( tool_test_results.get( 'passed_tests', [] ) )
                     failed_tests = len( tool_test_results.get( 'failed_tests', [] ) )
                     missing_test_components = len( tool_test_results.get( 'missing_test_components', [] ) )
@@ -1982,7 +1982,7 @@ def install_repositories_by_revision( self, trans, **kwd ):
                            repository_ids=','.join( util.listify( repository_ids ) ),
                            changeset_revisions=','.join( util.listify( changeset_revisions ) ) )
             pathspec = [ 'admin_toolshed', 'prepare_for_install' ]
-            url = common_util.url_join( galaxy_url, pathspec=pathspec, params=params )
+            url = util.build_url( galaxy_url, pathspec=pathspec, params=params )
             return trans.response.send_redirect( url )
         else:
             message = 'Repository installation is not possible due to an invalid Galaxy URL: <b>%s</b>.  ' % galaxy_url

lib/tool_shed/capsule/capsule_manager.py

@@ -15,6 +15,7 @@
 from galaxy.util import asbool
 from galaxy.util import CHUNK_SIZE
 from galaxy.util import safe_relpath
+from galaxy.util import build_url
 from galaxy.util.odict import odict
 from tool_shed.dependencies.repository.relation_builder import RelationBuilder
 from tool_shed.dependencies import attribute_handlers
@@ -123,7 +124,7 @@ def export_repository( self ):
             params = dict( encoded_repositories_archive_name=encoded_repositories_archive_name )
             pathspec = [ 'repository', 'export_via_api' ]
             tool_shed_url = web.url_for( '/', qualified=True )
-            download_url = common_util.url_join( tool_shed_url, pathspec=pathspec, params=params )
+            download_url = build_url( tool_shed_url, pathspec=pathspec, params=params )
             return dict( download_url=download_url, error_messages=error_messages )
         return repositories_archive, error_messages