Deprecate rally.common.yamlutils and rally.common.fileutils

* rally.common.fileutils contains stuff that related only to
  rally.cli.envutils + openstack related helper

* rally.common.yamlutils was designed for CLI purpose and should not be
  places under rally.common.

Change-Id: I9d639d2baa9298a26e7670c5b4b356a8b56583a2

CHANGELOG.rst

@@ -45,6 +45,11 @@ Deprecated
 * Module *rally.common.sshutils* is deprecated. Use *rally.utils.sshutils*
   instead.
 
+* Module *rally.common.yamlutils* is deprecated. It was designed for CLI usage
+  and moves to right place.
+
+* Module *rally.common.fileutils* is deprecated.
+
 * All modules from *rally.plugins.common.contexts* are deprecated. Use
   *rally.plugins.task.contexts* instead.
 

rally/cli/commands/deployment.py

@@ -25,10 +25,9 @@
 
 from rally.cli import cliutils
 from rally.cli import envutils
-from rally.common import fileutils
+from rally.cli import yamlutils as yaml
 from rally.common import logging
 from rally.common import utils
-from rally.common import yamlutils as yaml
 from rally.env import env_mgr
 from rally import exceptions
 from rally import plugins
@@ -326,10 +325,10 @@ def use(self, api, deployment):
             return 1
         print("Using deployment: %s" % deployment["uuid"])
 
-        fileutils.update_globals_file(envutils.ENV_DEPLOYMENT,
-                                      deployment["uuid"])
-        fileutils.update_globals_file(envutils.ENV_ENV,
-                                      deployment["uuid"])
+        envutils.update_globals_file(envutils.ENV_DEPLOYMENT,
+                                     deployment["uuid"])
+        envutils.update_globals_file(envutils.ENV_ENV,
+                                     deployment["uuid"])
 
         if "openstack" in deployment["credentials"]:
             creds = deployment["credentials"]["openstack"][0]

rally/cli/commands/env.py

@@ -21,8 +21,7 @@
 
 from rally.cli import cliutils
 from rally.cli import envutils
-from rally.common import fileutils
-from rally.common import yamlutils as yaml
+from rally.cli import yamlutils as yaml
 from rally.env import env_mgr
 from rally import exceptions
 
@@ -336,4 +335,4 @@ def use(self, api, env, to_json=False):
 
     def _use(self, env_uuid, to_json):
         _print("Using environment: %s" % env_uuid, to_json)
-        fileutils.update_globals_file(envutils.ENV_ENV, env_uuid)
+        envutils.update_globals_file(envutils.ENV_ENV, env_uuid)

rally/cli/commands/task.py

@@ -28,11 +28,10 @@
 
 from rally.cli import cliutils
 from rally.cli import envutils
-from rally.common import fileutils
+from rally.cli import yamlutils as yaml
 from rally.common import logging
 from rally.common import utils as rutils
 from rally.common import version
-from rally.common import yamlutils as yaml
 from rally import consts
 from rally import exceptions
 from rally import plugins
@@ -926,7 +925,7 @@ def use(self, api, task_id):
 
         print("Using task: %s" % task_id)
         api.task.get(task_id=task_id)
-        fileutils.update_globals_file("RALLY_TASK", task_id)
+        envutils.update_globals_file("RALLY_TASK", task_id)
 
     @cliutils.args("--uuid", dest="tasks", nargs="+", type=str,
                    help="UUIDs of tasks or json reports of tasks")

rally/cli/commands/verify.py

@@ -25,9 +25,8 @@
 
 from rally.cli import cliutils
 from rally.cli import envutils
-from rally.common import fileutils
+from rally.cli import yamlutils as yaml
 from rally.common import logging
-from rally.common import yamlutils as yaml
 from rally import exceptions
 from rally import plugins
 
@@ -151,7 +150,7 @@ def create_verifier(self, api, name, vtype, platform="", source=None,
     def use_verifier(self, api, verifier_id):
         """Choose a verifier to use for the future operations."""
         verifier = api.verifier.get(verifier_id=verifier_id)
-        fileutils.update_globals_file(envutils.ENV_VERIFIER, verifier["uuid"])
+        envutils.update_globals_file(envutils.ENV_VERIFIER, verifier["uuid"])
         print("Using verifier '%s' (UUID=%s) as the default verifier "
               "for the future CLI operations."
               % (verifier["name"], verifier["uuid"]))
@@ -536,7 +535,7 @@ def use(self, api, verification_uuid):
 
         verification = api.verification.get(
             verification_uuid=verification_uuid)
-        fileutils.update_globals_file(
+        envutils.update_globals_file(
             envutils.ENV_VERIFICATION, verification["uuid"])
         print("Using verification (UUID=%s) as the default verification "
               "for the future operations." % verification["uuid"])

rally/cli/envutils.py

@@ -17,7 +17,6 @@
 
 import decorator
 
-from rally.common import fileutils
 from rally import exceptions
 
 PATH_GLOBALS = "~/.rally/globals"
@@ -31,10 +30,78 @@
 MSG_MISSING_ARG = "Missing argument: --%(arg_name)s"
 
 
+def _read_env_file(path, except_env=None):
+    """Read the environment variable file.
+
+    :param path: the path of the file
+    :param except_env: the environment variable to avoid in the output
+
+    :returns: the content of the original file except the line starting with
+    the except_env parameter
+    """
+    output = []
+    if os.path.exists(path):
+        with open(path, "r") as env_file:
+            content = env_file.readlines()
+            for line in content:
+                if except_env is None or not line.startswith("%s=" %
+                                                             except_env):
+                    output.append(line)
+    return output
+
+
+def _load_env_file(path):
+    """Load the environment variable file into os.environ.
+
+    :param path: the path of the file
+    """
+    if os.path.exists(path):
+        content = _read_env_file(path)
+        for line in content:
+            (key, sep, value) = line.partition("=")
+            os.environ[key] = value.rstrip()
+
+
+def _rewrite_env_file(path, initial_content):
+    """Rewrite the environment variable file.
+
+    :param path: the path of the file
+    :param initial_content: the original content of the file
+    """
+    with open(path, "w+") as env_file:
+        for line in initial_content:
+            env_file.write(line)
+
+
+def _update_env_file(path, env_key, env_value):
+    """Update the environment variable file.
+
+    :param path: the path of the file
+    :param env_key: the key to update
+    :param env_value: the value of the property to update
+    """
+    output = _read_env_file(path, env_key)
+    output.append("%s=%s" % (env_key, env_value))
+    _rewrite_env_file(path, output)
+
+
+def update_globals_file(key, value):
+    """Update the globals variables file.
+
+    :param key: the key to update
+    :param value: the value to update
+    """
+    dir = os.path.expanduser("~/.rally/")
+    if not os.path.exists(dir):
+        os.makedirs(dir)
+    expanded_path = os.path.join(dir, "globals")
+    _update_env_file(expanded_path, key, "%s\n" % value)
+
+
 def clear_global(global_key):
     path = os.path.expanduser(PATH_GLOBALS)
     if os.path.exists(path):
-        fileutils.update_env_file(path, global_key, "\n")
+        _update_env_file(path, global_key, "\n")
     if global_key in os.environ:
         os.environ.pop(global_key)
 
@@ -46,7 +113,7 @@ def clear_env():
 
 def get_global(global_key, do_raise=False):
     if global_key not in os.environ:
-        fileutils.load_env_file(os.path.expanduser(PATH_GLOBALS))
+        _load_env_file(os.path.expanduser(PATH_GLOBALS))
     value = os.environ.get(global_key)
     if not value and do_raise:
         raise exceptions.InvalidArgumentsException("%s env is missing"

rally/common/fileutils.py

@@ -15,75 +15,14 @@
 
 import os
 import tempfile
+import warnings
 import zipfile
 
 
-def _read_env_file(path, except_env=None):
-    """Read the environment variable file.
-
-    :param path: the path of the file
-    :param except_env: the environment variable to avoid in the output
-
-    :returns: the content of the original file except the line starting with
-    the except_env parameter
-    """
-    output = []
-    if os.path.exists(path):
-        with open(path, "r") as env_file:
-            content = env_file.readlines()
-            for line in content:
-                if except_env is None or not line.startswith("%s=" %
-                                                             except_env):
-                    output.append(line)
-    return output
-
-
-def load_env_file(path):
-    """Load the environment variable file into os.environ.
-
-    :param path: the path of the file
-    """
-    if os.path.exists(path):
-        content = _read_env_file(path)
-        for line in content:
-            (key, sep, value) = line.partition("=")
-            os.environ[key] = value.rstrip()
-
-
-def _rewrite_env_file(path, initial_content):
-    """Rewrite the environment variable file.
-
-    :param path: the path of the file
-    :param initial_content: the original content of the file
-    """
-    with open(path, "w+") as env_file:
-        for line in initial_content:
-            env_file.write(line)
-
-
-def update_env_file(path, env_key, env_value):
-    """Update the environment variable file.
-
-    :param path: the path of the file
-    :param env_key: the key to update
-    :param env_value: the value of the property to update
-    """
-    output = _read_env_file(path, env_key)
-    output.append("%s=%s" % (env_key, env_value))
-    _rewrite_env_file(path, output)
-
-
-def update_globals_file(key, value):
-    """Update the globals variables file.
-
-    :param key: the key to update
-    :param value: the value to update
-    """
-    dir = os.path.expanduser("~/.rally/")
-    if not os.path.exists(dir):
-        os.makedirs(dir)
-    expanded_path = os.path.join(dir, "globals")
-    update_env_file(expanded_path, key, "%s\n" % value)
+warnings.warn(
+    f"Module `{__name__}` is deprecated since Rally v3.0.0 and may be "
+    f"removed in further releases."
+)
 
 
 def pack_dir(source_directory, zip_name=None):

rally/common/utils.py

@@ -156,6 +156,7 @@ def reset(self):
             self.__int.value = 0
 
 
+@logging.log_deprecated("it was an inner helper.", rally_version="3.0.0")
 def get_method_class(func):
     """Return the class that defined the given method.
 
@@ -178,6 +179,7 @@ def get_method_class(func):
         return None
 
 
+@logging.log_deprecated("it was an inner helper.", rally_version="3.0.0")
 def first_index(lst, predicate):
     """Return the index of the first element that matches a predicate.
 
@@ -212,6 +214,7 @@ def retry(times, func, *args, **kwargs):
                 raise
 
 
+@logging.log_deprecated("it is openstack specific.", rally_version="3.0.0")
 def iterate_per_tenants(users):
     """Iterate of a single arbitrary user from each tenant
 
@@ -425,6 +428,7 @@ def name_matches_object(cls, name, task_id=None, exact=True):
     return CustomNameMatcher
 
 
+@logging.log_deprecated("it was an inner helper.", rally_version="3.0.0")
 def merge(length, *sources):
     """Merge lists of lists.
 

tests/unit/cli/commands/test_deployment.py

@@ -243,8 +243,8 @@ def test_deploy_no_deployment_id(self, mock_get_global):
     @mock.patch("os.remove")
     @mock.patch("os.symlink")
     @mock.patch("os.path.exists", return_value=True)
-    @mock.patch("rally.common.fileutils.update_env_file")
-    def test_use(self, mock_update_env_file, mock_path_exists,
+    @mock.patch("rally.cli.envutils._update_env_file")
+    def test_use(self, mock__update_env_file, mock_path_exists,
                  mock_symlink, mock_remove):
         deployment_id = "593b683c-4b16-4b2b-a56b-e162bd60f10b"
         self.fake_api.deployment.get.return_value = {
@@ -262,7 +262,7 @@ def test_use(self, mock_update_env_file, mock_path_exists,
                         create=True) as mock_file:
             self.deployment.use(self.fake_api, deployment_id)
             self.assertEqual(3, mock_path_exists.call_count)
-            mock_update_env_file.assert_has_calls([
+            mock__update_env_file.assert_has_calls([
                 mock.call(os.path.expanduser("~/.rally/globals"),
                           "RALLY_DEPLOYMENT", "%s\n" % deployment_id),
                 mock.call(os.path.expanduser("~/.rally/globals"),
@@ -286,8 +286,8 @@ def test_use(self, mock_update_env_file, mock_path_exists,
     @mock.patch("os.remove")
     @mock.patch("os.symlink")
     @mock.patch("os.path.exists", return_value=True)
-    @mock.patch("rally.common.fileutils.update_env_file")
-    def test_use_with_v3_auth(self, mock_update_env_file, mock_path_exists,
+    @mock.patch("rally.cli.envutils._update_env_file")
+    def test_use_with_v3_auth(self, mock__update_env_file, mock_path_exists,
                               mock_symlink, mock_remove):
         deployment_id = "593b683c-4b16-4b2b-a56b-e162bd60f10b"
 
@@ -308,7 +308,7 @@ def test_use_with_v3_auth(self, mock_update_env_file, mock_path_exists,
                         create=True) as mock_file:
             self.deployment.use(self.fake_api, deployment_id)
             self.assertEqual(3, mock_path_exists.call_count)
-            mock_update_env_file.assert_has_calls([
+            mock__update_env_file.assert_has_calls([
                 mock.call(os.path.expanduser("~/.rally/globals"),
                           "RALLY_DEPLOYMENT", "%s\n" % deployment_id),
                 mock.call(os.path.expanduser("~/.rally/globals"),
@@ -334,7 +334,7 @@ def test_use_with_v3_auth(self, mock_update_env_file, mock_path_exists,
 
     @mock.patch("rally.cli.commands.deployment.DeploymentCommands."
                 "_update_openrc_deployment_file")
-    @mock.patch("rally.common.fileutils.update_globals_file")
+    @mock.patch("rally.cli.envutils.update_globals_file")
     def test_use_by_name(self, mock_update_globals_file,
                          mock__update_openrc_deployment_file):
         fake_credentials = {"admin": "foo_admin", "users": ["foo_user"]}

tests/unit/cli/commands/test_env.py

@@ -494,7 +494,7 @@ def test_use(self, mock__use, mock_env_manager_get):
         mock__use.assert_has_calls(
             [mock.call("aa", False), mock.call("bb", True)])
 
-    @mock.patch("rally.cli.commands.env.fileutils.update_globals_file")
+    @mock.patch("rally.cli.commands.env.envutils.update_globals_file")
     @mock.patch("rally.cli.commands.env.print")
     def test__use(self, mock_print, mock_update_globals_file):
         self.env._use("aa", True)

tests/unit/cli/commands/test_task.py

@@ -26,7 +26,7 @@
 from rally import api
 from rally.cli import cliutils
 from rally.cli.commands import task
-from rally.common import yamlutils as yaml
+from rally.cli import yamlutils as yaml
 from rally import consts
 from rally import exceptions
 from tests.unit import fakes
@@ -859,7 +859,7 @@ def test_validate_invalid(self, mock__load_and_validate_task):
             deployment="deployment",
             config=mock__load_and_validate_task.return_value)
 
-    @mock.patch("rally.common.fileutils._rewrite_env_file")
+    @mock.patch("rally.cli.envutils._rewrite_env_file")
     def test_use(self, mock__rewrite_env_file):
         task_id = "80422553-5774-44bd-98ac-38bd8c7a0feb"
         self.task.use(self.fake_api, task_id)