Improvements to testing blacklist

Summary:
This patch is necessary because individual test cases are not required
to have unique names. Therefore, test cases must now
be specified explicitly in the form <TestCase>.<TestMethod>.
Because it works by regex matching, passing just <TestCase> will
still disable an entire file.

This also allows for multiple exclusion files to be specified.

Reviewers: zturner, labath, jingham, tfiala

Subscribers: lldb-commits, sas

Differential Revision: https://reviews.llvm.org/D24988

llvm-svn: 283238

lldb/packages/Python/lldbsuite/test/configuration.py

@@ -102,10 +102,8 @@ def setupCrashInfoHook():
 regexp = None
 
 # Sets of tests which are excluded at runtime
-skip_files = None
-skip_methods = None
-xfail_files = None
-xfail_methods = None
+skip_tests = None
+xfail_tests = None
 
 # By default, recorded session info for errored/failed test are dumped into its
 # own file under a session directory named after the timestamp of the test suite

lldb/packages/Python/lldbsuite/test/dotest.py

@@ -216,33 +216,24 @@ def parseExclusion(exclusion_file):
        <method name>
     """
     excl_type = None
-    case_type = None
 
     with open(exclusion_file) as f:
         for line in f:
+            line = line.strip()
             if not excl_type:
-                [excl_type, case_type] = line.split()
+                excl_type = line
                 continue
 
-            line = line.strip()
             if not line:
                 excl_type = None
-            elif excl_type == 'skip' and case_type == 'files':
-                if not configuration.skip_files:
-                    configuration.skip_files = []
-                configuration.skip_files.append(line)
-            elif excl_type == 'skip' and case_type == 'methods':
-                if not configuration.skip_methods:
-                    configuration.skip_methods = []
-                configuration.skip_methods.append(line)
-            elif excl_type == 'xfail' and case_type == 'files':
-                if not configuration.xfail_files:
-                    configuration.xfail_files = []
-                configuration.xfail_files.append(line)
-            elif excl_type == 'xfail' and case_type == 'methods':
-                if not configuration.xfail_methods:
-                    configuration.xfail_methods = []
-                configuration.xfail_methods.append(line)
+            elif excl_type == 'skip':
+                if not configuration.skip_tests:
+                    configuration.skip_tests = []
+                configuration.skip_tests.append(line)
+            elif excl_type == 'xfail':
+                if not configuration.xfail_tests:
+                    configuration.xfail_tests = []
+                configuration.xfail_tests.append(line)
 
 
 def parseOptionsAndInitTestdirs():
@@ -375,7 +366,8 @@ def parseOptionsAndInitTestdirs():
         lldbtest_config.lldbExec = os.path.realpath(args.executable)
 
     if args.excluded:
-        parseExclusion(args.excluded)
+        for excl_file in args.excluded:
+            parseExclusion(excl_file)
 
     if args.p:
         if args.p.startswith('-'):
@@ -799,8 +791,8 @@ def visit_file(dir, name):
             # We didn't match the regex, we're done.
             return
 
-    if configuration.skip_files:
-        for file_regexp in configuration.skip_files:
+    if configuration.skip_tests:
+        for file_regexp in configuration.skip_tests:
             if re.search(file_regexp, name):
                 return
 

lldb/packages/Python/lldbsuite/test/dotest_args.py

@@ -96,7 +96,7 @@ def create_parser():
         '-p',
         metavar='pattern',
         help='Specify a regexp filename pattern for inclusion in the test suite')
-    group.add_argument('--excluded', metavar='exclusion-file', help=textwrap.dedent(
+    group.add_argument('--excluded', metavar='exclusion-file', action='append', help=textwrap.dedent(
         '''Specify a file for tests to exclude. File should contain lists of regular expressions for test files or methods,
                                 with each list under a matching header (xfail files, xfail methods, skip files, skip methods)'''))
     group.add_argument(

lldb/packages/Python/lldbsuite/test/test_result.py

@@ -18,8 +18,6 @@
 # Third-party modules
 import unittest2
 
-from unittest2.util import strclass
-
 # LLDB Modules
 from . import configuration
 from lldbsuite.test_event.event_builder import EventBuilder
@@ -139,8 +137,7 @@ def startTest(self, test):
                 self.getCategoriesForTest(test)):
             self.hardMarkAsSkipped(test)
         if self.checkExclusion(
-                configuration.skip_methods,
-                test._testMethodName):
+                configuration.skip_tests, test.id()):
             self.hardMarkAsSkipped(test)
 
         configuration.setCrashInfoHook(
@@ -161,11 +158,7 @@ def startTest(self, test):
 
     def addSuccess(self, test):
         if self.checkExclusion(
-                configuration.xfail_files,
-                strclass(
-                    test.__class__)) or self.checkExclusion(
-                configuration.xfail_methods,
-                test._testMethodName):
+                configuration.xfail_tests, test.id()):
             self.addUnexpectedSuccess(test, None)
             return
 
@@ -239,11 +232,7 @@ def addCleanupError(self, test, err):
 
     def addFailure(self, test, err):
         if self.checkExclusion(
-                configuration.xfail_files,
-                strclass(
-                    test.__class__)) or self.checkExclusion(
-                configuration.xfail_methods,
-                test._testMethodName):
+                configuration.xfail_tests, test.id()):
             self.addExpectedFailure(test, err, None)
             return