Fix import order and Python3 compatibility for most of test/

.ci/flake8_blacklist.txt

@@ -1,8 +1,8 @@
 .venv/
 .tox/
-eggs/
-client/
+client/node_modules/
 database/
 doc/patch.py
 doc/source/conf.py
+eggs/
 lib/galaxy/util/jstree.py

.ci/flake8_lint_include_list.txt

@@ -1,8 +1,10 @@
+client/galaxy/style/source_material/circle.py
 contrib/
 cron/add_manual_builds.py
 cron/cleanup_datasets.py
 cron/parse_builds_3_sites.py
 cron/parse_builds.py
+doc/parse_gx_xsd.py
 lib/galaxy/actions/
 lib/galaxy/auth/__init__.py
 lib/galaxy/auth/providers/__init__.py
@@ -440,34 +442,47 @@ scripts/tool_shed/migrate_tools_to_repositories.py
 scripts/tool_shed/show_tool_dependency_installation_dir_contents.py
 scripts/tools/re_escape_output.py
 test/api/__init__.py
+test/api/test_dataset_collections.py
 test/api/test_datasets.py
 test/api/test_datatypes.py
 test/api/test_framework.py
 test/api/test_histories.py
 test/api/test_history_contents_provenance.py
 test/api/test_jobs.py
 test/api/test_libraries.py
+test/api/test_page_revisions.py
 test/api/test_search.py
+test/api/test_tool_data.py
+test/api/test_tools.py
 test/api/test_tours.py
+test/api/test_workflow_extraction.py
 test/api/test_workflows_from_yaml.py
-test/api/workflows_format_2/interface.py
-test/api/workflows_format_2/main.py
+test/api/workflows_format_2/
 test/base/api_asserts.py
+test/base/api.py
 test/base/api_util.py
 test/base/__init__.py
 test/base/instrument.py
+test/base/integration_util.py
+test/base/interactor.py
 test/base/nose_util.py
+test/base/test_logging.py
 test/base/tool_shed_util.py
+test/base/twilltestcase.py
 test/casperjs/
 test/functional/database_contexts.py
 test/functional/__init__.py
+test/functional/test_data_managers.py
+test/functional/test_toolbox.py
+test/functional/tool-data/
 test/functional/tools/
-test/integration/__init__.py
-test/integration/test_pulsar_embedded.py
+test/functional/workflow.py
+test/integration/
 test/manual/__init__.py
 test/shed_functional/base/common.py
 test/shed_functional/base/__init__.py
 test/shed_functional/functional/
+test/shed_functional/functional_tests.py
 test/shed_functional/__init__.py
 test/unit/dataset_collections/__init__.py
 test/unit/datatypes/dataproviders/__init__.py

.ci/py3_sources.txt

@@ -1,5 +1,7 @@
+client/galaxy/style/source_material/circle.py
 contrib/
 cron/
+doc/parse_gx_xsd.py
 lib/galaxy/actions/
 lib/galaxy/auth/
 lib/galaxy/config.py

client/galaxy/style/source_material/circle.py

@@ -1,15 +1,14 @@
 #!/usr/bin/env python
-
 """
 usage: %prog width height bg_color hatch_color [color alpha stop_pos] +
 """
-
 from __future__ import division
 
 import sys
-import cairo
 from math import pi
 
+import cairo
+
 assert sys.version_info[:2] >= ( 2, 4 )
 
 

doc/parse_gx_xsd.py

@@ -2,14 +2,12 @@
 # TODO: Add examples, tables and best practice links to command
 # TODO: Examples of truevalue, falsevalue
 # TODO: Test param extra_file
-from __future__ import print_function
-
 # Things dropped from TOC (still documented inside schema).
 #  - request_parameter_translation
+from __future__ import print_function
 
 from lxml import etree
-from StringIO import StringIO
-
+from six import StringIO
 
 with open("doc/schema_template.md", "r") as f:
     MARKDOWN_TEMPLATE = f.read()

test/api/helpers.py

@@ -1,10 +1,11 @@
-from base import api_asserts
+import json
+import time
 from operator import itemgetter
 
-import time
-import json
-from six import StringIO
 from pkg_resources import resource_string
+from six import StringIO
+
+from base import api_asserts
 
 # Simple workflow that takes an input and call cat wrapper on it.
 workflow_str = resource_string( __name__, "test_workflow_1.ga" )

test/api/test_api_batch.py

@@ -1,7 +1,9 @@
 import json
-import pprint
 import logging
+import pprint
+
 from requests import post
+
 from base import api
 
 log = logging.getLogger( "functional_tests.py" )

test/api/test_authenticate.py

@@ -1,8 +1,8 @@
 import base64
 
 from requests import get
-from base import api
 
+from base import api
 
 TEST_USER_EMAIL = "auth_user_test@bx.psu.edu"
 TEST_USER_PASSWORD = "testpassword1"

test/api/test_dataset_collections.py

@@ -1,7 +1,8 @@
-from base import api
 import json
-from .helpers import DatasetPopulator
-from .helpers import DatasetCollectionPopulator
+
+from base import api
+
+from .helpers import DatasetCollectionPopulator, DatasetPopulator
 
 
 class DatasetCollectionApiTestCase( api.ApiTestCase ):

test/api/test_history_contents.py

@@ -1,11 +1,10 @@
-from base import api
 import json
 
-from .helpers import TestsDatasets
-from .helpers import LibraryPopulator
-from .helpers import DatasetCollectionPopulator
-from requests import delete
-from requests import put
+from requests import delete, put
+
+from base import api
+
+from .helpers import DatasetCollectionPopulator, LibraryPopulator, TestsDatasets
 
 
 # TODO: Test anonymous access.

test/api/test_page_revisions.py

@@ -1,5 +1,6 @@
 from galaxy.exceptions import error_codes
-from api.test_pages import BasePageApiTestCase
+
+from .test_pages import BasePageApiTestCase
 
 
 class PageRevisionsApiTestCase( BasePageApiTestCase ):

test/api/test_pages.py

@@ -1,8 +1,7 @@
-from galaxy.exceptions import error_codes
-from base import api
 from requests import delete
 
-from operator import itemgetter
+from base import api
+from galaxy.exceptions import error_codes
 
 
 class BasePageApiTestCase( api.ApiTestCase ):
@@ -102,4 +101,4 @@ def _users_index_has_page_with_id( self, id ):
         index_response = self._get( "pages" )
         self._assert_status_code_is( index_response, 200 )
         pages = index_response.json()
-        return id in map( itemgetter( "id" ), pages )
+        return id in ( _["id"] for _ in pages )

test/api/test_tool_data.py

@@ -1,9 +1,11 @@
 """ Tests for the tool data API.
 """
 from __future__ import print_function
-from base import api
+
 import operator
 
+from base import api
+
 
 class ToolDataApiTestCase( api.ApiTestCase ):
 

test/api/test_tools.py

@@ -1,13 +1,12 @@
 # Test tools API.
-from base import api
 import json
-from operator import itemgetter
-from .helpers import DatasetPopulator
-from .helpers import DatasetCollectionPopulator
-from .helpers import LibraryPopulator
-from .helpers import skip_without_tool
+
+from base import api
 from galaxy.tools.verify.test_data import TestDataResolver
 
+from .helpers import (DatasetCollectionPopulator, DatasetPopulator,
+    LibraryPopulator, skip_without_tool)
+
 
 class ToolsTestCase( api.ApiTestCase ):
 
@@ -25,7 +24,7 @@ def test_no_panel_index( self ):
         tools_index = index.json()
         # No need to flatten out sections, with in_panel=False, only tools are
         # returned.
-        tool_ids = map( itemgetter( "id" ), tools_index )
+        tool_ids = [_["id"] for _ in tools_index]
         assert "upload1" in tool_ids
 
     @skip_without_tool( "cat1" )
@@ -195,7 +194,7 @@ def test_filter_failed( self ):
         def get_state(dce):
             return dce["object"]["state"]
 
-        mixed_states = map(get_state, mixed_hdca["elements"])
+        mixed_states = [get_state(_) for _ in mixed_hdca["elements"]]
         assert mixed_states == [u"ok", u"error", u"ok", u"error"], mixed_states
         inputs = {
             "input": { "src": "hdca", "id": mixed_hdca["id"] },
@@ -206,7 +205,7 @@ def get_state(dce):
         self.assertEquals( len(filter_output_collections), 1 )
         filtered_hid = filter_output_collections[0]["hid"]
         filtered_hdca = self.dataset_populator.get_history_collection_details(history_id, hid=filtered_hid, wait=False)
-        filtered_states = map(get_state, filtered_hdca["elements"])
+        filtered_states = [get_state(_) for _ in filtered_hdca["elements"]]
         assert filtered_states == [u"ok", u"ok"], filtered_states
 
     @skip_without_tool( "multi_select" )
@@ -525,7 +524,7 @@ def test_multirun_non_data_parameter( self ):
         # Assert we have three outputs with 1, 2, and 3 lines respectively.
         assert len( outputs ) == 3
         outputs_contents = [ self.dataset_populator.get_history_dataset_content( history_id, dataset=o ).strip() for o in outputs ]
-        assert sorted( map( lambda c: len( c.split( "\n" ) ), outputs_contents ) ) == [ 1, 2, 3 ]
+        assert sorted( len( c.split( "\n" ) ) for c in outputs_contents ) == [ 1, 2, 3 ]
 
     @skip_without_tool( "cat1" )
     def test_multirun_in_repeat( self ):
@@ -1230,7 +1229,7 @@ def __tool_ids( self ):
             else:
                 tools.append( tool_or_section )
 
-        tool_ids = map( itemgetter( "id" ), tools )
+        tool_ids = [_["id"] for _ in tools]
         return tool_ids
 
     def __build_nested_list( self, history_id ):

test/api/test_users.py

@@ -1,5 +1,7 @@
 import json
+
 from requests import put
+
 from base import api
 
 TEST_USER_EMAIL = "user_for_users_index_test@bx.psu.edu"

test/api/test_workflow_extraction.py

@@ -1,8 +1,9 @@
 from __future__ import print_function
-from collections import namedtuple
+
 import functools
-from json import dumps, loads
 import operator
+from collections import namedtuple
+from json import dumps, loads
 
 from .helpers import skip_without_tool
 from .test_workflows import BaseWorkflowsApiTestCase

test/api/test_workflows.py

@@ -1,26 +1,24 @@
 from __future__ import print_function
 
 import time
-import yaml
-from json import dumps
 from collections import namedtuple
+from json import dumps
 from uuid import uuid4
 
+import yaml
+from requests import delete, put
+
 from base import api
-from galaxy.tools.verify.test_data import TestDataResolver
 from galaxy.exceptions import error_codes
-from .helpers import WorkflowPopulator
-from .helpers import DatasetPopulator
-from .helpers import DatasetCollectionPopulator
-from .helpers import skip_without_tool
+from galaxy.tools.verify.test_data import TestDataResolver
+
+from .helpers import (DatasetCollectionPopulator, DatasetPopulator,
+    skip_without_tool, WorkflowPopulator)
 from .workflows_format_2 import (
     convert_and_import_workflow,
     ImporterGalaxyInterface,
 )
 
-from requests import delete
-from requests import put
-
 SIMPLE_NESTED_WORKFLOW_YAML = """
 class: GalaxyWorkflow
 inputs:
@@ -277,15 +275,15 @@ def test_show_valid( self ):
         workflow = show_response.json()
         self._assert_looks_like_instance_workflow_representation( workflow )
         assert len(workflow["steps"]) == 3
-        self.assertEqual(sorted([step["id"] for step in workflow["steps"].values()]), [0, 1, 2])
+        self.assertEqual(sorted(step["id"] for step in workflow["steps"].values()), [0, 1, 2])
 
         show_response = self._get( "workflows/%s" % workflow_id, {"legacy": True} )
         workflow = show_response.json()
         self._assert_looks_like_instance_workflow_representation( workflow )
         assert len(workflow["steps"]) == 3
         # Can't reay say what the legacy IDs are but must be greater than 3 because dummy
         # workflow was created first in this instance.
-        self.assertNotEqual(sorted([step["id"] for step in workflow["steps"].values()]), [0, 1, 2])
+        self.assertNotEqual(sorted(step["id"] for step in workflow["steps"].values()), [0, 1, 2])
 
     def test_show_invalid_key_is_400( self ):
         show_response = self._get( "workflows/%s" % self._random_key() )
@@ -457,7 +455,7 @@ def test_import_subworkflows( self ):
         def get_subworkflow_content_id(workflow_id):
             workflow_contents = self._download_workflow(workflow_id, style="editor")
             steps = workflow_contents['steps']
-            subworkflow_step = filter(lambda s: s["type"] == "subworkflow", steps.values())[0]
+            subworkflow_step = next(s for s in steps.values() if s["type"] == "subworkflow")
             return subworkflow_step['content_id']
 
         workflow_id = self._upload_yaml_workflow(SIMPLE_NESTED_WORKFLOW_YAML, publish=True)
@@ -1636,7 +1634,7 @@ def __assert_lines_hid_line_count_is( self, history, hid, lines ):
         contents_url = "histories/%s/contents" % history
         history_contents_response = self._get( contents_url )
         self._assert_status_code_is( history_contents_response, 200 )
-        hda_summary = filter( lambda hc: hc[ "hid" ] == hid, history_contents_response.json() )[ 0 ]
+        hda_summary = next(hc for hc in history_contents_response.json() if hc[ "hid" ] == hid)
         hda_info_response = self._get( "%s/%s" % ( contents_url, hda_summary[ "id" ] ) )
         self._assert_status_code_is( hda_info_response, 200 )
         self.assertEqual( hda_info_response.json()[ "metadata_data_lines" ], lines )

test/api/workflows_format_2/__init__.py

@@ -1,8 +1,8 @@
 """ This module defines the public interface or entry point for the
 Format 2 workflow code.
 """
-from .main import convert_and_import_workflow
 from .interface import ImporterGalaxyInterface
+from .main import convert_and_import_workflow
 
 
 __all__ = [