Allow regular expressions in skip-list

This allows you to more easily skip tests belonging to particular class,
as you don't have to list them individually.

Backwards compatability is maintained by treating invalid regexes
as literal strings.

Change-Id: I10d7477cd3321683899d455bf4370ec8ef8a4d05

doc/source/quick_start/tutorial/step_9_verifying_cloud_via_tempest_verifier.rst

@@ -460,7 +460,10 @@ argument.
     Using verification (UUID=af766b2f-cada-44db-a0c2-336ab0c17c27) as the default verification for the future operations.
 
 Moreover, it is possible to skip a certain list of Tempest tests, using the
-**--skip-list** argument.
+**--skip-list** argument; this should be a yaml file containing key-value
+pairs, where the keys are a regex to match against the test name. The values
+are the reason why the test was skipped. If an invalid regex is supplied the
+key is treated as a test id. For example:
 
 .. code-block:: console
 
@@ -471,6 +474,12 @@ Moreover, it is possible to skip a certain list of Tempest tests, using the
     tempest.api.compute.admin.test_flavors.FlavorsAdminTestJSON.test_create_flavor_with_none_id[id-f83fe669-6758-448a-a85e-32d351f36fe0]: Reason 2
     tempest.api.compute.admin.test_flavors.FlavorsAdminTestJSON.test_create_flavor_with_uuid_id[id-94c9bb4e-2c2a-4f3c-bb1f-5f0daf918e6d]:
 
+    # Example: Skipping several tests with a partial match:
+    ^tempest\.api\.compute.servers\.test_attach_interfaces: skip using a regex
+
+The first five keys are invalid regular expressions and are included in the
+skip list as is.
+
 .. code-block:: console
 
     $ rally verify start --pattern tempest.api.compute.admin.test_flavors.FlavorsAdminTestJSON --skip-list skip-list.yaml

rally/cli/commands/verify.py

@@ -455,7 +455,8 @@ def delete_verifier_ext(self, api, verifier_id=None, name=None):
                    required=False,
                    help="Path to a file with a list of tests to skip. "
                         "Format: json or yaml like a dictionary where keys "
-                        "are test names and values are reasons.")
+                        "are regexes matching test names and values are "
+                        "reasons.")
     @cliutils.args("--xfail-list", dest="xfail_list", type=str,
                    metavar="<path>", required=False,
                    help="Path to a file with a list of tests that will be "

rally/plugins/common/verification/testr.py

@@ -41,6 +41,7 @@ def __init__(self, ctx):
         self._tmp_files = []
 
     def setup(self):
+        super(TestrContext, self).setup()
         use_testr = getattr(self.verifier.manager, "_use_testr", True)
 
         if use_testr:
@@ -62,12 +63,10 @@ def setup(self):
                 self.context["testr_cmd"].extend(
                     ["--concurrency", str(concurrency)])
 
-        load_list = run_args.get("load_list")
-        skip_list = run_args.get("skip_list")
+        load_list = self.context.get("load_list")
+        skip_list = self.context.get("skip_list")
 
         if skip_list:
-            if not load_list:
-                load_list = self.verifier.manager.list_tests()
             load_list = set(load_list) - set(skip_list)
         if load_list:
             load_list_file = common_utils.generate_random_path()
@@ -137,15 +136,14 @@ def list_tests(self, pattern=""):
     def run(self, context):
         """Run tests."""
         testr_cmd = context["testr_cmd"]
-        run_args = context.get("run_args", {})
         LOG.debug("Test(s) started by the command: '%s'."
                   % " ".join(testr_cmd))
         stream = subprocess.Popen(testr_cmd, env=self.run_environ,
                                   cwd=self.repo_dir,
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.STDOUT)
-        xfail_list = run_args.get("xfail_list")
-        skip_list = run_args.get("skip_list")
+        xfail_list = context.get("xfail_list")
+        skip_list = context.get("skip_list")
         results = subunit_v2.parse(stream.stdout, live=True,
                                    expected_failures=xfail_list,
                                    skipped_tests=skip_list,

rally/verification/context.py

@@ -13,6 +13,7 @@
 #    under the License.
 
 import functools
+import re
 
 from rally.common.plugin import plugin
 from rally.task import context
@@ -21,6 +22,31 @@
 configure = functools.partial(context.configure, hidden=True)
 
 
+def _expand_skip_list(load_list, regexps):
+    """Returns mapping of test names to the reason why the test was skipped.
+
+    The dictionary includes all tests in``load_list`` that match a key in
+    ``regexps``.
+    """
+    result = {}
+    if not regexps:
+        return result
+    for regex, reason in regexps.items():
+        try:
+            pattern = re.compile(regex)
+            for test in load_list:
+                if pattern.search(test):
+                    result[test] = reason
+        except re.error:
+            # assume regex is a test id, eg: tempest.api.compute.admin.
+            # test_flavors.FlavorsAdminTestJSON.
+            # test_create_flavor_using_string_ram
+            # [id-3b541a2e-2ac2-4b42-8b8d-ba6e22fcd4da]
+            result[regex] = reason
+            continue
+    return result
+
+
 @plugin.base()
 class VerifierContext(context.BaseContext):
     """Verifier context that will be run before starting a verification."""
@@ -35,6 +61,26 @@ def validate(cls, config):
         # do not validate jsonschema.
         pass
 
+    def setup(self):
+        self._process_runargs()
+
+    def _process_runargs(self):
+        # Store the skip and test lists in the context
+
+        run_args = self.context.get("run_args", {})
+
+        load_list = run_args.get("load_list")
+        skip_list = run_args.get("skip_list")
+
+        if skip_list:
+            if not load_list:
+                load_list = self.verifier.manager.list_tests()
+            skip_list = _expand_skip_list(load_list, skip_list)
+
+        self.context["skip_list"] = skip_list
+        self.context["load_list"] = load_list
+        self.context["xfail_list"] = run_args.get("xfail_list")
+
 
 class ContextManager(context.ContextManager):
 

tests/unit/plugins/common/verification/test_testr.py

@@ -111,7 +111,7 @@ def test_setup_with_skip_and_load_lists(self, mock_generate_random_path):
 
         # with load_list and skip_list
         load_list = ["tests.foo", "tests.bar"]
-        skip_list = ["tests.foo"]
+        skip_list = {"tests.foo": "skip reason"}
         cfg = {"verifier": self.verifier,
                "run_args": {"load_list": load_list,
                             "skip_list": skip_list}}
@@ -131,7 +131,55 @@ def test_setup_with_skip_and_load_lists(self, mock_generate_random_path):
         # with skip_list, but without load_list
         load_list = ["tests.foo", "tests.bar"]
         self.verifier.manager.list_tests.return_value = load_list
-        skip_list = ["tests.foo"]
+        skip_list = {"tests.foo": "skip reason"}
+        cfg = {"verifier": self.verifier,
+               "run_args": {"skip_list": skip_list}}
+        ctx = testr.TestrContext(cfg)
+        mock_open = mock.mock_open()
+        with mock.patch("%s.open" % PATH, mock_open):
+            ctx.setup()
+        mock_open.assert_called_once_with(
+            mock_generate_random_path.return_value, "w")
+        handle = mock_open.return_value
+        handle.write.assert_called_once_with(load_list[1])
+        self.assertEqualCmd(["--parallel", "--load-list",
+                             mock_generate_random_path.return_value],
+                            cfg["testr_cmd"])
+        self.verifier.manager.list_tests.assert_called_once_with()
+
+    @mock.patch("%s.common_utils.generate_random_path" % PATH)
+    def test_skip_list_with_regex_positive_match(self,
+                                                 mock_generate_random_path):
+        # using a regex in skip_list
+        load_list = ["tests.foo.bar", "tests.bar"]
+        self.verifier.manager.list_tests.return_value = load_list
+        skip_list = {"^tests.foo": "skip reason"}
+        cfg = {"verifier": self.verifier,
+               "run_args": {"skip_list": skip_list}}
+        ctx = testr.TestrContext(cfg)
+        mock_open = mock.mock_open()
+        with mock.patch("%s.open" % PATH, mock_open):
+            ctx.setup()
+        mock_open.assert_called_once_with(
+            mock_generate_random_path.return_value, "w")
+        handle = mock_open.return_value
+        handle.write.assert_called_once_with(load_list[1])
+        self.assertEqualCmd(["--parallel", "--load-list",
+                             mock_generate_random_path.return_value],
+                            cfg["testr_cmd"])
+        self.verifier.manager.list_tests.assert_called_once_with()
+
+    @mock.patch("%s.common_utils.generate_random_path" % PATH)
+    def test_skip_list_with_invalid_regex(self,
+                                          mock_generate_random_path):
+        load_list = [
+            "tests.foo[e3976dea-bed9-4b14-abaf-59372de9303]",
+            "tests.bar"
+        ]
+        self.verifier.manager.list_tests.return_value = load_list
+        skip_list = {
+            "tests.foo[e3976dea-bed9-4b14-abaf-59372de9303]": "skip reason"
+        }
         cfg = {"verifier": self.verifier,
                "run_args": {"skip_list": skip_list}}
         ctx = testr.TestrContext(cfg)
@@ -301,8 +349,7 @@ def test__init_testr(self, mock_isdir, mock_exists, mock_check_output,
     def test_run(self, mock_popen, mock_parse):
         launcher = testr.TestrLauncher(mock.Mock())
         ctx = {"testr_cmd": ["ls", "-la"],
-               "run_args": {"xfail_list": mock.Mock(),
-                            "skip_list": mock.Mock()}}
+               "xfail_list": None, "skip_list": None}
 
         self.assertEqual(mock_parse.return_value, launcher.run(ctx))
 
@@ -314,8 +361,54 @@ def test_run(self, mock_popen, mock_parse):
         mock_popen.return_value.wait.assert_called_once_with()
         mock_parse.assert_called_once_with(
             mock_popen.return_value.stdout, live=True,
-            expected_failures=ctx["run_args"]["xfail_list"],
-            skipped_tests=ctx["run_args"]["skip_list"],
+            expected_failures=None,
+            skipped_tests=None,
+            logger_name=launcher.verifier.name)
+
+    @mock.patch("%s.subunit_v2.parse" % PATH)
+    @mock.patch("%s.subprocess.Popen" % PATH)
+    def test_run_skip(self, mock_popen, mock_parse):
+        launcher = testr.TestrLauncher(mock.Mock())
+        skip_list = {"test1": "reason1"}
+        ctx = {"testr_cmd": ["ls", "-la"],
+               "xfail_list": None, "skip_list": skip_list}
+
+        self.assertEqual(mock_parse.return_value, launcher.run(ctx))
+
+        mock_popen.assert_called_once_with(ctx["testr_cmd"],
+                                           env=launcher.run_environ,
+                                           cwd=launcher.repo_dir,
+                                           stdout=subprocess.PIPE,
+                                           stderr=subprocess.STDOUT)
+        mock_popen.return_value.wait.assert_called_once_with()
+        mock_parse.assert_called_once_with(
+            mock_popen.return_value.stdout, live=True,
+            expected_failures=None,
+            skipped_tests=ctx["skip_list"],
+            logger_name=launcher.verifier.name)
+
+    @mock.patch("%s.subunit_v2.parse" % PATH)
+    @mock.patch("%s.subprocess.Popen" % PATH)
+    def test_run_exclude(self, mock_popen, mock_parse):
+        launcher = testr.TestrLauncher(mock.Mock())
+        xfail_list = {"test1": "reason1"}
+        ctx = {"testr_cmd": ["ls", "-la"],
+               "xfail_list": xfail_list, "skip_list": None}
+        launcher = testr.TestrLauncher(mock.Mock())
+
+        result = launcher.run(ctx)
+        self.assertEqual(mock_parse.return_value, result)
+
+        mock_popen.assert_called_once_with(ctx["testr_cmd"],
+                                           env=launcher.run_environ,
+                                           cwd=launcher.repo_dir,
+                                           stdout=subprocess.PIPE,
+                                           stderr=subprocess.STDOUT)
+        mock_popen.return_value.wait.assert_called_once_with()
+        mock_parse.assert_called_once_with(
+            mock_popen.return_value.stdout, live=True,
+            expected_failures=xfail_list,
+            skipped_tests=None,
             logger_name=launcher.verifier.name)
 
     @mock.patch("%s.manager.VerifierManager.install" % PATH)

tests/unit/verification/test_context.py

@@ -22,6 +22,7 @@ def cleanup(self):
         pass
 
     def setup(self):
+        super(FakeContext, self).setup()
         pass