Merge "Fix py3k issues"

rally/benchmark/context/sahara/sahara_edp.py

@@ -13,7 +13,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import urllib2
+import requests
 
 from rally.benchmark.context import base
 from rally.benchmark.context.cleanup import manager as resource_manager
@@ -144,7 +144,7 @@ def setup_inputs(self, sahara, tenant_id, input_type, input_url):
 
     def download_and_save_lib(self, sahara, lib_type, name, download_url,
                               tenant_id):
-        lib_data = urllib2.urlopen(download_url).read()
+        lib_data = requests.get(download_url).json()
 
         job_binary_internal = sahara.job_binary_internals.create(
             name=name,

rally/benchmark/engine.py

@@ -115,7 +115,7 @@ def _check_cloud(self):
                              _("Task validation of scenarios names."))
     def _validate_config_scenarios_name(self, config):
         available = set(base_scenario.Scenario.list_benchmark_scenarios())
-        specified = set(config.iterkeys())
+        specified = set(six.iterkeys(config))
 
         if not specified.issubset(available):
             names = ", ".join(specified - available)

rally/cmd/envutils.py

@@ -55,7 +55,7 @@ def get_global(global_key, do_raise=False):
 def default_from_global(arg_name, env_name,
                         cli_arg_name):
     def default_from_global(f, *args, **kwargs):
-        id_arg_index = f.func_code.co_varnames.index(arg_name)
+        id_arg_index = f.__code__.co_varnames.index(arg_name)
         args = list(args)
         if args[id_arg_index] is None:
             args[id_arg_index] = get_global(env_name)

rally/common/utils.py

@@ -297,7 +297,7 @@ def parse_docstring(docstring):
 
     if docstring:
         lines = docstrings.prepare_docstring(docstring)
-        lines = filter(lambda line: line != "", lines)
+        lines = [line for line in lines if line]
     else:
         lines = []
 

rally/deploy/serverprovider/providers/openstack.py

@@ -15,7 +15,6 @@
 
 import os
 import time
-import urllib2
 
 import novaclient.exceptions
 
@@ -137,14 +136,11 @@ def get_image_uuid(self):
                 return image.id
 
         LOG.info(_('Downloading new image %s') % self.config['image']['url'])
-        image = self.glance.images.create(name=self.config['image']['name'])
-        try:
-            image.update(data=urllib2.urlopen(self.config['image']['url']),
-                         disk_format=self.config['image']['format'],
-                         container_format='bare')
-        except urllib2.URLError:
-            LOG.error(_('Unable to retrieve %s') % self.config['image']['url'])
-            raise
+        image = self.glance.images.create(
+            name=self.config['image']['name'],
+            copy_from=self.config['image']['url'],
+            disk_format=self.config['image']['format'],
+            container_format='bare')
         image.get()
 
         if image.checksum != self.config['image']['checksum']:

tests/hacking/README.rst

@@ -22,7 +22,7 @@ Rally Specific Commandments
  * [N323] - Ensure that ``assertTrue/assertFalse(A in/not in B)`` are not used with collection contents
  * [N324] - Ensure that ``assertEqual(A in/not in B, True/False)`` and ``assertEqual(True/False, A in/not in B)`` are not used with collection contents
 * [N33x] - Reserved for rules related to Python 3 compatibility
- * [N330] - Ensure that ``dict.iteritems()`` is not used
+ * [N330] - Ensure that ``dict.iterkeys()``, ``dict.itervalues()``, ``dict.iteritems()`` and ``dict.iterlist()`` are not used
  * [N331] - Ensure that ``basestring`` is not used
  * [N332] - Ensure that ``StringIO.StringIO`` is not used
  * [N333] - Ensure that ``urlparse`` is not used

tests/hacking/checks.py

@@ -45,7 +45,6 @@
     r"assertEqual\((\w|[][.'\"])+( not)? in (\w|[][.'\", ])+, (True|False)\)")
 re_assert_equal_in_start_with_true_or_false = re.compile(
     r"assertEqual\((True|False), (\w|[][.'\"])+( not)? in (\w|[][.'\", ])+\)")
-re_iteritems_method = re.compile(r"\.iteritems\(\)")
 re_basestring_method = re.compile(r"(^|[\s,(\[=])basestring([\s,)\]]|$)")
 re_StringIO_method = re.compile(r"StringIO\.StringIO\(")
 re_urlparse_method = re.compile(r"(^|[\s=])urlparse\.")
@@ -233,17 +232,24 @@ def assert_equal_in(logical_line):
 
 
 def check_iteritems_method(logical_line):
-    """Check if iteritems is properly called for compatibility with Python 3
+    """Check that collections are iterated in Python 3 compatible way
 
-    The correct form is six.iteritems(dict) or dict.items(), instead of
-    dict.iteritems()
+    The correct forms:
+      six.iterkeys(collection)
+      six.itervalues(collection)
+      six.iteritems(collection)
+      six.iterlist(collection)
 
     N330
     """
-    res = re_iteritems_method.search(logical_line)
-    if res:
-        yield (0, "N330: Use six.iteritems(dict) or dict.items() rather than "
-                  "dict.iteritems() to iterate a collection.")
+    iter_functions = ["iterkeys()", "itervalues()",
+                      "iteritems()", "iterlist()"]
+    for func in iter_functions:
+        pos = logical_line.find(func)
+        if pos != -1:
+            yield (pos, "N330: Use six.%(func)s(dict) rather than "
+                        "dict.%(func)s() to iterate a collection." %
+                   {"func": func[:-2]})
 
 
 def check_basestring_method(logical_line):

tests/unit/benchmark/context/cleanup/test_resources.py

@@ -131,7 +131,7 @@ def test_list(self):
             "some_resources": some_resources
         }
 
-        self.assertEqual([some_resources[0]], neut.list())
+        self.assertEqual([some_resources[0]], list(neut.list()))
 
         neut.user.neutron().list_some_resources.assert_called_once_with(
             {"tenant_id": neut.tenant_uuid})

tests/unit/benchmark/context/sahara/test_sahara_edp.py

@@ -74,17 +74,17 @@ def context_without_edp_keys(self):
         }
 
     @mock.patch("%s.sahara_edp.resource_manager.cleanup" % CTX)
-    @mock.patch("%s.sahara_edp.urllib2" % CTX)
+    @mock.patch("%s.sahara_edp.requests" % CTX)
     @mock.patch("%s.sahara_edp.osclients" % CTX)
-    def test_setup_and_cleanup(self, mock_osclients, mock_urllib,
+    def test_setup_and_cleanup(self, mock_osclients, mock_requests,
                                mock_cleanup):
 
         mock_sahara = mock_osclients.Clients(mock.MagicMock()).sahara()
         mock_sahara.data_sources.create.return_value = mock.MagicMock(id=42)
         mock_sahara.job_binary_internals.create.return_value = (
             mock.MagicMock(id=42))
 
-        mock_urllib.urlopen().read.return_value = "test_binary"
+        mock_requests.get().json.return_value = "test_binary"
 
         ctx = self.context_without_edp_keys
         sahara_ctx = sahara_edp.SaharaEDP(ctx)
@@ -112,7 +112,7 @@ def test_setup_and_cleanup(self, mock_osclients, mock_urllib,
         sahara_ctx.setup()
 
         mock_sahara.data_sources.create.assert_has_calls(input_ds_crete_calls)
-        mock_urllib.urlopen().read.assert_has_calls(download_calls)
+        mock_requests.get().json.assert_has_calls(download_calls)
         mock_sahara.job_binary_internals.create.assert_has_calls(
             job_binary_internals_calls)
         mock_sahara.job_binaries.create.assert_has_calls(job_binaries_calls)

tests/unit/benchmark/context/test_images.py

@@ -76,7 +76,7 @@ def test_setup(self, mock_osclients, mock_image_create):
 
         tenants = self._gen_tenants(tenants_count)
         users = list()
-        for id in tenants.iterkeys():
+        for id in tenants:
             for i in range(users_per_tenant):
                 users.append({"id": i, "tenant_id": id,
                               "endpoint": "endpoint"})
@@ -123,7 +123,7 @@ def test_cleanup(self, mock_cleanup, mock_osclients):
 
         tenants = self._gen_tenants(tenants_count)
         users = list()
-        for id_ in tenants.iterkeys():
+        for id_ in tenants:
             for i in range(users_per_tenant):
                 users.append({"id": i, "tenant_id": id_,
                               "endpoint": "endpoint"})

tests/unit/cmd/test_cliutils.py

@@ -15,6 +15,7 @@
 
 import mock
 from oslo.config import cfg
+import testtools
 
 from rally.cmd import cliutils
 from rally.cmd.commands import deployment
@@ -141,6 +142,10 @@ def test_run_show(self):
         ret = cliutils.run(["rally", "show", "keypairs"], self.categories)
         self.assertEqual(ret, 1)
 
+    @testtools.skip(
+        "This test is not work with latest(1.6.0) oslo.config(see "
+        "https://review.openstack.org/#/c/135150 for more details). "
+        "Should wait for new release of oslo.config with appropriate fix.")
     @mock.patch("rally.db.task_get",
                 side_effect=exceptions.TaskNotFound(FAKE_TASK_UUID))
     def test_run_task_not_found(self, mock_task_get):
@@ -149,13 +154,21 @@ def test_run_task_not_found(self, mock_task_get):
         self.assertTrue(mock_task_get.called)
         self.assertEqual(ret, 1)
 
+    @testtools.skip(
+        "This test is not work with latest(1.6.0) oslo.config(see "
+        "https://review.openstack.org/#/c/135150 for more details). "
+        "Should wait for new release of oslo.config with appropriate fix.")
     @mock.patch("rally.openstack.common.cliutils.validate_args",
                 side_effect=common_cliutils.MissingArgs("missing"))
     def test_run_show_fails(self, mock_validate_args):
         ret = cliutils.run(["rally", "show", "keypairs"], self.categories)
         self.assertTrue(mock_validate_args.called)
         self.assertEqual(ret, 1)
 
+    @testtools.skip(
+        "This test is not work with latest(1.6.0) oslo.config(see "
+        "https://review.openstack.org/#/c/135150 for more details). "
+        "Should wait for new release of oslo.config with appropriate fix.")
     def test_run_failed_to_open_file(self):
 
         class FailuresCommands(object):

tests/unit/common/test_utils.py

@@ -226,7 +226,7 @@ def test_method_class_for_class_level_method(self):
         class A:
             def m(self):
                 pass
-        self.assertEqual(utils.get_method_class(A.m), A)
+        self.assertEqual(A, utils.get_method_class(A.m))
 
     def test_method_class_for_module_level_method(self):
         self.assertIsNone(utils.get_method_class(module_level_method))

tests/unit/deploy/serverprovider/providers/test_openstack.py

@@ -176,25 +176,21 @@ def test_create_servers(self, bmutils, oscl, m_Server, m_sleep):
             nics='fake_nics', key_name='fake_key_name')
 
     @mock.patch(MOD_NAME + '.osclients')
-    @mock.patch(MOD_NAME + '.urllib2')
-    def test_get_image_found_by_checksum(self, u, oscl):
+    def test_get_image_found_by_checksum(self, oscl):
         self._init_mock_clients()
         oscl.Clients = mock.MagicMock(return_value=self.clients)
         prov = OSProvider(mock.MagicMock(), self._get_valid_config())
         image_uuid = prov.get_image_uuid()
         self.assertEqual(image_uuid, 'fake-uuid')
 
     @mock.patch(MOD_NAME + '.osclients')
-    @mock.patch(MOD_NAME + '.urllib2')
-    def test_get_image_download(self, u, oscl):
+    def test_get_image_download(self, oscl):
         self._init_mock_clients()
         self.glance_client.images.list = mock.Mock(return_value=[])
         oscl.Clients = mock.MagicMock(return_value=self.clients)
         prov = OSProvider(mock.MagicMock(), self._get_valid_config())
         image_uuid = prov.get_image_uuid()
         self.assertEqual(image_uuid, 'fake-uuid')
-        self.assertEqual(u.mock_calls,
-                         [mock.call.urlopen('http://example.net/img.qcow2')])
 
     @mock.patch(MOD_NAME + '.osclients')
     def test_get_image_no_glance_exception(

tests/unit/test_docstrings.py

@@ -62,8 +62,8 @@ def test_all_scenarios_have_docstrings(self):
                                      "One-line description for %s "
                                      "should be declarative and not start "
                                      "with 'Test(s) ...'" % scenario_name)
-                    params_count = scenario.func_code.co_argcount
-                    params = scenario.func_code.co_varnames[:params_count]
+                    params_count = scenario.__code__.co_argcount
+                    params = scenario.__code__.co_varnames[:params_count]
                     documented_params = [p["name"] for p in doc["params"]]
                     for param in params:
                         if param not in ignored_params: