Deprecate -m/+m dotest options in favor of test categories

Summary:
This change deprecates -m/+m dotest options (the options are still recognized but they print an
error message pointing to the new options) and adds a new lldb-mi test category instead. To just
run lldb-mi tests, use '-G lldb-mi'. To skip lldb-mi tests, use '--skip-category lldb-mi'. All
lldb-mi tests are marked as such using the getCategories method on the base MiTestCaseBase class
and the @lldbmi_test decorator is not needed. In case one still needs to annotate a specific test
function as an lldb-mi test, one can use the @add_test_categories(['lldb-mi']) decorator to
achieve that.

Reviewers: tfiala, dawn, ki.stfu, abidh

Subscribers: lldb-commits

Differential Revision: http://reviews.llvm.org/D14060

llvm-svn: 251400

lldb/test/dotest.py

@@ -84,12 +84,6 @@ def writeln(self, arg=None):
 # The test suite.
 suite = unittest2.TestSuite()
 
-# By default, lldb-mi tests are performed if lldb-mi can be found.
-# Use @lldbmi_test decorator, defined in lldbtest.py, to mark a test as
-# a lldb-mi test.
-dont_do_lldbmi_test = False
-just_do_lldbmi_test = False
-
 # By default, benchmarks tests are not run.
 just_do_benchmarks_test = False
 
@@ -414,14 +408,25 @@ def setupCrashInfoHook():
     else:
         pass
 
+def shouldSkipBecauseOfCategories(test_categories):
+    global useCategories, categoriesList, skipCategories
+
+    if useCategories:
+        if len(test_categories) == 0 or len(categoriesList & set(test_categories)) == 0:
+            return True
+
+    for category in skipCategories:
+        if category in test_categories:
+            return True
+
+    return False
+
 def parseOptionsAndInitTestdirs():
     """Initialize the list of directories containing our unittest scripts.
 
     '-h/--help as the first option prints out usage info and exit the program.
     """
 
-    global dont_do_lldbmi_test
-    global just_do_lldbmi_test
     global just_do_benchmarks_test
     global dont_do_dsym_test
     global dont_do_dwarf_test
@@ -566,6 +571,11 @@ def parseOptionsAndInitTestdirs():
               "functionality (-G pyapi, --skip-category pyapi) instead.")
         sys.exit(1)
 
+    if args.m or args.plus_m:
+        print("Options '-m' and '+m' have been deprecated. Please use the test category\n"
+              "functionality (-G lldb-mi, --skip-category lldb-mi) instead.")
+        sys.exit(1)
+
     if args.plus_b:
         just_do_benchmarks_test = True
 
@@ -631,15 +641,6 @@ def parseOptionsAndInitTestdirs():
     if args.l:
         skip_long_running_test = False
 
-    if args.m:
-        dont_do_lldbmi_test = True
-
-    if args.plus_m:
-        if dont_do_lldbmi_test:
-            print("Warning: -m and +m can't both be specified! Using only -m")
-        else:
-            just_do_lldbmi_test = True
-
     if args.framework:
         lldbFrameworkPath = args.framework
 
@@ -727,10 +728,6 @@ def parseOptionsAndInitTestdirs():
     if do_help == True:
         usage(parser)
 
-    # Do not specify both '-m' and '+m' at the same time.
-    if dont_do_lldbmi_test and just_do_lldbmi_test:
-        usage(parser)
-
     if args.no_multiprocess:
         no_multiprocess_test_runner = True
 
@@ -1071,8 +1068,6 @@ def setupSysPath():
     # Some of the tests can invoke the 'lldb' command directly.
     # We'll try to locate the appropriate executable right here.
 
-    lldbMiExec = None
-
     # The lldb executable can be set from the command line
     # if it's not set, we try to find it now
     # first, we try the environment
@@ -1113,15 +1108,13 @@ def setupSysPath():
 
     # Assume lldb-mi is in same place as lldb
     # If not found, disable the lldb-mi tests
-    global dont_do_lldbmi_test
+    lldbMiExec = None
     if lldbtest_config.lldbExec and is_exe(lldbtest_config.lldbExec + "-mi"):
         lldbMiExec = lldbtest_config.lldbExec + "-mi"
     if not lldbMiExec:
-        dont_do_lldbmi_test = True
-        if just_do_lldbmi_test:
+        if not shouldSkipBecauseOfCategories(["lldb-mi"]):
             print("The 'lldb-mi' executable cannot be located.  The lldb-mi tests can not be run as a result.")
-        else:
-            print("The 'lldb-mi' executable cannot be located.  Some of the tests may not be run as a result.")
+            skipCategories.append("lldb-mi")
     else:
         os.environ["LLDBMI_EXEC"] = lldbMiExec
 
@@ -1517,8 +1510,6 @@ def getsource_if_available(obj):
     lldb.lldbtest_remote_shell_template = lldbtest_remote_shell_template
 
     # Put all these test decorators in the lldb namespace.
-    lldb.dont_do_lldbmi_test = dont_do_lldbmi_test
-    lldb.just_do_lldbmi_test = just_do_lldbmi_test
     lldb.just_do_benchmarks_test = just_do_benchmarks_test
     lldb.dont_do_dsym_test = dont_do_dsym_test
     lldb.dont_do_dwarf_test = dont_do_dwarf_test
@@ -1807,30 +1798,14 @@ def getCategoriesForTest(self,test):
                         test_categories = []
                     return test_categories
 
-                def shouldSkipBecauseOfCategories(self,test):
-                    global useCategories
-                    import inspect
-                    if useCategories:
-                        global categoriesList
-                        test_categories = self.getCategoriesForTest(test)
-                        if len(test_categories) == 0 or len(categoriesList & set(test_categories)) == 0:
-                            return True
-
-                    global skipCategories
-                    for category in skipCategories:
-                        if category in self.getCategoriesForTest(test):
-                            return True
-
-                    return False
-
                 def hardMarkAsSkipped(self,test):
                     getattr(test, test._testMethodName).__func__.__unittest_skip__ = True
                     getattr(test, test._testMethodName).__func__.__unittest_skip_why__ = "test case does not fall in any category of interest for this run"
                     test.__class__.__unittest_skip__ = True
                     test.__class__.__unittest_skip_why__ = "test case does not fall in any category of interest for this run"
 
                 def startTest(self, test):
-                    if self.shouldSkipBecauseOfCategories(test):
+                    if shouldSkipBecauseOfCategories(self.getCategoriesForTest(test)):
                         self.hardMarkAsSkipped(test)
                     global setCrashInfoHook
                     setCrashInfoHook("%s at %s" % (str(test),inspect.getfile(test.__class__)))

lldb/test/dotest_args.py

@@ -60,8 +60,6 @@ def create_parser():
     group.add_argument('-f', metavar='filterspec', action='append', help='Specify a filter, which consists of the test class name, a dot, followed by the test method, to only admit such test into the test suite')  # FIXME: Example?
     X('-g', 'If specified, the filterspec by -f is not exclusive, i.e., if a test module does not match the filterspec (testclass.testmethod), the whole module is still admitted to the test suite')
     X('-l', "Don't skip long running tests")
-    X('-m', "Don't do lldb-mi tests")
-    X('+m', "Just do lldb-mi tests. Do not specify along with '-m'", dest='plus_m')
     group.add_argument('-p', metavar='pattern', help='Specify a regexp filename pattern for inclusion in the test suite')
     group.add_argument('-X', metavar='directory', help="Exclude a directory from consideration for test discovery. -X types => if 'types' appear in the pathname components of a potential testfile, it will be ignored")
     group.add_argument('-G', '--category', metavar='category', action='append', dest='categoriesList', help=textwrap.dedent('''Specify categories of test cases of interest. Can be specified more than once.'''))
@@ -186,6 +184,8 @@ def create_parser():
     # Deprecated on 23.10.2015. Remove completely after a grace period.
     D('-a')
     D('+a', dest='plus_a')
+    D('-m')
+    D('+m', dest='plus_m')
     del D
 
     group = parser.add_argument_group('Test directories')

lldb/test/lldbtest.py

@@ -481,20 +481,6 @@ def impl(func):
         return func
     return impl
 
-def lldbmi_test(func):
-    """Decorate the item as a lldb-mi only test."""
-    if isinstance(func, type) and issubclass(func, unittest2.TestCase):
-        raise Exception("@lldbmi_test can only be used to decorate a test method")
-    @wraps(func)
-    def wrapper(self, *args, **kwargs):
-        if lldb.dont_do_lldbmi_test:
-            self.skipTest("lldb-mi tests")
-        return func(self, *args, **kwargs)
-
-    # Mark this function as such to separate them from lldb command line tests.
-    wrapper.__lldbmi_test__ = True
-    return wrapper
-
 def benchmarks_test(func):
     """Decorate the item as a benchmarks test."""
     if isinstance(func, type) and issubclass(func, unittest2.TestCase):
@@ -1367,7 +1353,6 @@ def setUp(self):
             self.lldbMiExec = os.environ["LLDBMI_EXEC"]
         else:
             self.lldbMiExec = None
-            self.dont_do_lldbmi_test = True
 
         # If we spawn an lldb process for test (via pexpect), do not load the
         # init file unless told otherwise.
@@ -1384,19 +1369,6 @@ def setUp(self):
         # used for all the test cases.
         self.testMethodName = self._testMethodName
 
-        # lldb-mi only test is decorated with @lldbmi_test,
-        # which also sets the "__lldbmi_test__" attribute of the
-        # function object to True.
-        try:
-            if lldb.just_do_lldbmi_test:
-                testMethod = getattr(self, self._testMethodName)
-                if getattr(testMethod, "__lldbmi_test__", False):
-                    pass
-                else:
-                    self.skipTest("non lldb-mi test")
-        except AttributeError:
-            pass
-
         # Benchmarks test is decorated with @benchmarks_test,
         # which also sets the "__benchmarks_test__" attribute of the
         # function object to True.

lldb/test/test_categories.py

@@ -13,7 +13,8 @@
     'cmdline'       : 'Tests related to the LLDB command-line interface',
     'dyntype'       : 'Tests related to dynamic type support',
     'stresstest'    : 'Tests related to stressing lldb limits',
-    'flakey'        : 'Flakey test cases, i.e. tests that do not reliably pass at each execution'
+    'flakey'        : 'Flakey test cases, i.e. tests that do not reliably pass at each execution',
+    'lldb-mi'       : 'lldb-mi tests'
 }
 
 def unique_string_match(yourentry, list):

lldb/test/tools/lldb-mi/TestMiExit.py

@@ -13,7 +13,6 @@ class MiExitTestCase(lldbmi_testcase.MiTestCaseBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    @lldbmi_test
     @expectedFailureWindows("llvm.org/pr22274: need a pexpect replacement for windows")
     @skipIfFreeBSD # llvm.org/pr22411: Failure presumably due to known thread races
     def test_lldbmi_gdb_exit(self):
@@ -38,7 +37,6 @@ def test_lldbmi_gdb_exit(self):
         import pexpect
         self.expect(pexpect.EOF)
 
-    @lldbmi_test
     @expectedFailureWindows("llvm.org/pr22274: need a pexpect replacement for windows")
     @skipIfFreeBSD # llvm.org/pr22411: Failure presumably due to known thread races
     def test_lldbmi_quit(self):
@@ -62,7 +60,6 @@ def test_lldbmi_quit(self):
         import pexpect
         self.expect(pexpect.EOF)
 
-    @lldbmi_test
     @expectedFailureWindows("llvm.org/pr22274: need a pexpect replacement for windows")
     @skipIfFreeBSD # llvm.org/pr22411: Failure presumably due to known thread races
     def test_lldbmi_q(self):

lldb/test/tools/lldb-mi/TestMiFile.py

@@ -13,7 +13,6 @@ class MiFileTestCase(lldbmi_testcase.MiTestCaseBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    @lldbmi_test
     @skipIfWindows #llvm.org/pr24452: Get lldb-mi tests working on Windows
     @skipIfFreeBSD # llvm.org/pr22411: Failure presumably due to known thread races
     def test_lldbmi_file_exec_and_symbols_file(self):
@@ -30,7 +29,6 @@ def test_lldbmi_file_exec_and_symbols_file(self):
         self.expect("\^running")
         self.expect("\*stopped,reason=\"exited-normally\"")
 
-    @lldbmi_test
     @skipIfWindows #llvm.org/pr24452: Get lldb-mi tests working on Windows
     @skipIfFreeBSD # llvm.org/pr22411: Failure presumably due to known thread races
     def test_lldbmi_file_exec_and_symbols_absolute_path(self):
@@ -49,7 +47,6 @@ def test_lldbmi_file_exec_and_symbols_absolute_path(self):
         self.expect("\^running")
         self.expect("\*stopped,reason=\"exited-normally\"")
 
-    @lldbmi_test
     @skipIfWindows #llvm.org/pr24452: Get lldb-mi tests working on Windows
     @skipIfFreeBSD # llvm.org/pr22411: Failure presumably due to known thread races
     def test_lldbmi_file_exec_and_symbols_relative_path(self):
@@ -67,7 +64,6 @@ def test_lldbmi_file_exec_and_symbols_relative_path(self):
         self.expect("\^running")
         self.expect("\*stopped,reason=\"exited-normally\"")
 
-    @lldbmi_test
     @skipIfWindows #llvm.org/pr24452: Get lldb-mi tests working on Windows
     @skipIfFreeBSD # llvm.org/pr22411: Failure presumably due to known thread races
     def test_lldbmi_file_exec_and_symbols_unknown_path(self):

lldb/test/tools/lldb-mi/TestMiGdbSetShow.py

@@ -14,7 +14,6 @@ class MiGdbSetShowTestCase(lldbmi_testcase.MiTestCaseBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    @lldbmi_test
     @expectedFailureWindows("llvm.org/pr22274: need a pexpect replacement for windows")
     @skipIfFreeBSD # llvm.org/pr22411: Failure presumably due to known thread races
     def test_lldbmi_gdb_set_target_async_default(self):
@@ -34,7 +33,6 @@ def test_lldbmi_gdb_set_target_async_default(self):
         self.runCmd("-gdb-show target-async")
         self.expect("\^done,value=\"on\"")
 
-    @lldbmi_test
     @expectedFailureWindows("llvm.org/pr22274: need a pexpect replacement for windows")
     @skipIfFreeBSD # llvm.org/pr22411: Failure presumably due to known thread races
     def test_lldbmi_gdb_set_target_async_on(self):
@@ -63,7 +61,6 @@ def test_lldbmi_gdb_set_target_async_on(self):
         self.expect("\*running")
         self.expect("@\"argc=1")
 
-    @lldbmi_test
     @expectedFailureWindows("llvm.org/pr22274: need a pexpect replacement for windows")
     @skipIfFreeBSD # llvm.org/pr22411: Failure presumably due to known thread races
     @expectedFailureLinux # Failing in ~11/600 dosep runs (build 3120-3122)
@@ -89,7 +86,6 @@ def test_lldbmi_gdb_set_target_async_off(self):
         if it < len(unexpected):
             self.fail("unexpected found: %s" % unexpected[it])
 
-    @lldbmi_test
     @expectedFailureWindows("llvm.org/pr22274: need a pexpect replacement for windows")
     @skipIfFreeBSD # llvm.org/pr22411: Failure presumably due to known thread races
     def test_lldbmi_gdb_show_target_async(self):
@@ -101,7 +97,6 @@ def test_lldbmi_gdb_show_target_async(self):
         self.runCmd("-gdb-show target-async")
         self.expect("\^done,value=\"on\"")
 
-    @lldbmi_test
     @expectedFailureWindows("llvm.org/pr22274: need a pexpect replacement for windows")
     @skipIfFreeBSD # llvm.org/pr22411: Failure presumably due to known thread races
     def test_lldbmi_gdb_show_language(self):
@@ -124,7 +119,6 @@ def test_lldbmi_gdb_show_language(self):
         self.runCmd("-gdb-show language")
         self.expect("\^done,value=\"c\+\+\"")
 
-    @lldbmi_test
     @expectedFailureWindows("llvm.org/pr22274: need a pexpect replacement for windows")
     @unittest2.expectedFailure("-gdb-set ignores unknown properties")
     def test_lldbmi_gdb_set_unknown(self):
@@ -136,7 +130,6 @@ def test_lldbmi_gdb_set_unknown(self):
         self.runCmd("-gdb-set unknown some_value")
         self.expect("\^error")
 
-    @lldbmi_test
     @expectedFailureWindows("llvm.org/pr22274: need a pexpect replacement for windows")
     @unittest2.expectedFailure("-gdb-show ignores unknown properties")
     def test_lldbmi_gdb_show_unknown(self):
@@ -149,7 +142,6 @@ def test_lldbmi_gdb_show_unknown(self):
         self.expect("\^error")
 
 
-    @lldbmi_test
     @expectedFailureWindows("llvm.org/pr22274: need a pexpect replacement for windows")
     @skipIfFreeBSD # llvm.org/pr22411: Failure presumably due to known thread races
     @skipIfLinux # llvm.org/pr22841: lldb-mi tests fail on all Linux buildbots

lldb/test/tools/lldb-mi/TestMiLibraryLoaded.py

@@ -13,7 +13,6 @@ class MiLibraryLoadedTestCase(lldbmi_testcase.MiTestCaseBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    @lldbmi_test
     @skipIfWindows #llvm.org/pr24452: Get lldb-mi tests working on Windows
     @skipIfFreeBSD # llvm.org/pr22411: Failure presumably due to known thread races
     def test_lldbmi_library_loaded(self):

lldb/test/tools/lldb-mi/TestMiPrompt.py

@@ -13,7 +13,6 @@ class MiPromptTestCase(lldbmi_testcase.MiTestCaseBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    @lldbmi_test
     @skipIfWindows #llvm.org/pr24452: Get lldb-mi tests working on Windows
     @skipIfFreeBSD # llvm.org/pr22411: Failure presumably due to known thread races
     def test_lldbmi_prompt(self):