From f833f173fbca19275d3cb36b96d1f7461c0a8fee Mon Sep 17 00:00:00 2001 From: Francis Ricci Date: Tue, 4 Oct 2016 18:48:00 +0000 Subject: [PATCH] Improvements to testing blacklist Summary: This patch is necessary because individual test cases are not required to have unique names. Therefore, test cases must now be specified explicitly in the form .. Because it works by regex matching, passing just will still disable an entire file. This also allows for multiple exclusion files to be specified. Reviewers: zturner, labath, jingham, tfiala Subscribers: lldb-commits, sas Differential Revision: https://reviews.llvm.org/D24988 llvm-svn: 283238 --- .../Python/lldbsuite/test/configuration.py | 6 ++-- lldb/packages/Python/lldbsuite/test/dotest.py | 36 ++++++++----------- .../Python/lldbsuite/test/dotest_args.py | 2 +- .../Python/lldbsuite/test/test_result.py | 17 ++------- 4 files changed, 20 insertions(+), 41 deletions(-) diff --git a/lldb/packages/Python/lldbsuite/test/configuration.py b/lldb/packages/Python/lldbsuite/test/configuration.py index 37fa87563e7db..a0553a72c19d8 100644 --- a/lldb/packages/Python/lldbsuite/test/configuration.py +++ b/lldb/packages/Python/lldbsuite/test/configuration.py @@ -102,10 +102,8 @@ def setupCrashInfoHook(): regexp = None # Sets of tests which are excluded at runtime -skip_files = None -skip_methods = None -xfail_files = None -xfail_methods = None +skip_tests = None +xfail_tests = None # By default, recorded session info for errored/failed test are dumped into its # own file under a session directory named after the timestamp of the test suite diff --git a/lldb/packages/Python/lldbsuite/test/dotest.py b/lldb/packages/Python/lldbsuite/test/dotest.py index a37fff5ff1d2c..9991a33ae86f4 100644 --- a/lldb/packages/Python/lldbsuite/test/dotest.py +++ b/lldb/packages/Python/lldbsuite/test/dotest.py @@ -216,33 +216,24 @@ def parseExclusion(exclusion_file): """ excl_type = None - case_type = None with open(exclusion_file) as f: for line in f: + line = line.strip() if not excl_type: - [excl_type, case_type] = line.split() + excl_type = line continue - line = line.strip() if not line: excl_type = None - elif excl_type == 'skip' and case_type == 'files': - if not configuration.skip_files: - configuration.skip_files = [] - configuration.skip_files.append(line) - elif excl_type == 'skip' and case_type == 'methods': - if not configuration.skip_methods: - configuration.skip_methods = [] - configuration.skip_methods.append(line) - elif excl_type == 'xfail' and case_type == 'files': - if not configuration.xfail_files: - configuration.xfail_files = [] - configuration.xfail_files.append(line) - elif excl_type == 'xfail' and case_type == 'methods': - if not configuration.xfail_methods: - configuration.xfail_methods = [] - configuration.xfail_methods.append(line) + elif excl_type == 'skip': + if not configuration.skip_tests: + configuration.skip_tests = [] + configuration.skip_tests.append(line) + elif excl_type == 'xfail': + if not configuration.xfail_tests: + configuration.xfail_tests = [] + configuration.xfail_tests.append(line) def parseOptionsAndInitTestdirs(): @@ -375,7 +366,8 @@ def parseOptionsAndInitTestdirs(): lldbtest_config.lldbExec = os.path.realpath(args.executable) if args.excluded: - parseExclusion(args.excluded) + for excl_file in args.excluded: + parseExclusion(excl_file) if args.p: if args.p.startswith('-'): @@ -799,8 +791,8 @@ def visit_file(dir, name): # We didn't match the regex, we're done. return - if configuration.skip_files: - for file_regexp in configuration.skip_files: + if configuration.skip_tests: + for file_regexp in configuration.skip_tests: if re.search(file_regexp, name): return diff --git a/lldb/packages/Python/lldbsuite/test/dotest_args.py b/lldb/packages/Python/lldbsuite/test/dotest_args.py index 02d1047045ca0..0b8470bcaa817 100644 --- a/lldb/packages/Python/lldbsuite/test/dotest_args.py +++ b/lldb/packages/Python/lldbsuite/test/dotest_args.py @@ -96,7 +96,7 @@ def create_parser(): '-p', metavar='pattern', help='Specify a regexp filename pattern for inclusion in the test suite') - group.add_argument('--excluded', metavar='exclusion-file', help=textwrap.dedent( + group.add_argument('--excluded', metavar='exclusion-file', action='append', help=textwrap.dedent( '''Specify a file for tests to exclude. File should contain lists of regular expressions for test files or methods, with each list under a matching header (xfail files, xfail methods, skip files, skip methods)''')) group.add_argument( diff --git a/lldb/packages/Python/lldbsuite/test/test_result.py b/lldb/packages/Python/lldbsuite/test/test_result.py index 75e5fb80296ba..7bfe9708386c8 100644 --- a/lldb/packages/Python/lldbsuite/test/test_result.py +++ b/lldb/packages/Python/lldbsuite/test/test_result.py @@ -18,8 +18,6 @@ # Third-party modules import unittest2 -from unittest2.util import strclass - # LLDB Modules from . import configuration from lldbsuite.test_event.event_builder import EventBuilder @@ -139,8 +137,7 @@ def startTest(self, test): self.getCategoriesForTest(test)): self.hardMarkAsSkipped(test) if self.checkExclusion( - configuration.skip_methods, - test._testMethodName): + configuration.skip_tests, test.id()): self.hardMarkAsSkipped(test) configuration.setCrashInfoHook( @@ -161,11 +158,7 @@ def startTest(self, test): def addSuccess(self, test): if self.checkExclusion( - configuration.xfail_files, - strclass( - test.__class__)) or self.checkExclusion( - configuration.xfail_methods, - test._testMethodName): + configuration.xfail_tests, test.id()): self.addUnexpectedSuccess(test, None) return @@ -239,11 +232,7 @@ def addCleanupError(self, test, err): def addFailure(self, test, err): if self.checkExclusion( - configuration.xfail_files, - strclass( - test.__class__)) or self.checkExclusion( - configuration.xfail_methods, - test._testMethodName): + configuration.xfail_tests, test.id()): self.addExpectedFailure(test, err, None) return