diff --git a/llvm/unittests/Support/CrashRecoveryTest.cpp b/llvm/unittests/Support/CrashRecoveryTest.cpp index c7350452034a3..e95513eb28416 100644 --- a/llvm/unittests/Support/CrashRecoveryTest.cpp +++ b/llvm/unittests/Support/CrashRecoveryTest.cpp @@ -178,11 +178,6 @@ TEST(CrashRecoveryTest, UnixCRCReturnCode) { int Res = setenv("LLVM_CRC_UNIXCRCRETURNCODE", "1", 0); ASSERT_EQ(Res, 0); - Res = unsetenv("GTEST_SHARD_INDEX"); - ASSERT_EQ(Res, 0); - Res = unsetenv("GTEST_TOTAL_SHARDS"); - ASSERT_EQ(Res, 0); - std::string Error; bool ExecutionFailed; int RetCode = ExecuteAndWait(Executable, argv, {}, {}, 0, 0, &Error, diff --git a/llvm/unittests/Support/ProgramTest.cpp b/llvm/unittests/Support/ProgramTest.cpp index f93a77ab67057..fdd0478f70515 100644 --- a/llvm/unittests/Support/ProgramTest.cpp +++ b/llvm/unittests/Support/ProgramTest.cpp @@ -95,9 +95,7 @@ class ProgramEnvTest : public testing::Test { }; while (*EnvP != nullptr) { - auto S = prepareEnvVar(*EnvP); - if (!StringRef(S).startswith("GTEST_")) - EnvTable.emplace_back(S); + EnvTable.emplace_back(prepareEnvVar(*EnvP)); ++EnvP; } } diff --git a/llvm/utils/lit/lit/Test.py b/llvm/utils/lit/lit/Test.py index dc1c66e896c54..77b9c235e40d4 100644 --- a/llvm/utils/lit/lit/Test.py +++ b/llvm/utils/lit/lit/Test.py @@ -219,12 +219,11 @@ def getExecPath(self, components): class Test: """Test - Information on a single test instance.""" - def __init__(self, suite, path_in_suite, config, file_path = None, gtest_json_file = None): + def __init__(self, suite, path_in_suite, config, file_path = None): self.suite = suite self.path_in_suite = path_in_suite self.config = config self.file_path = file_path - self.gtest_json_file = gtest_json_file # A list of conditions under which this test is expected to fail. # Each condition is a boolean expression of features and target @@ -259,7 +258,7 @@ def __init__(self, suite, path_in_suite, config, file_path = None, gtest_json_fi # The previous test elapsed time, if applicable. self.previous_elapsed = 0.0 - if suite.test_times and '/'.join(path_in_suite) in suite.test_times: + if '/'.join(path_in_suite) in suite.test_times: time = suite.test_times['/'.join(path_in_suite)] self.previous_elapsed = abs(time) self.previous_failure = time < 0 diff --git a/llvm/utils/lit/lit/TestingConfig.py b/llvm/utils/lit/lit/TestingConfig.py index a9660b39c9cdb..e80369377857b 100644 --- a/llvm/utils/lit/lit/TestingConfig.py +++ b/llvm/utils/lit/lit/TestingConfig.py @@ -28,7 +28,7 @@ def fromdefaults(litConfig): 'TMPDIR', 'TMP', 'TEMP', 'TEMPDIR', 'AVRLIT_BOARD', 'AVRLIT_PORT', 'FILECHECK_OPTS', 'VCINSTALLDIR', 'VCToolsinstallDir', 'VSINSTALLDIR', 'WindowsSdkDir', - 'WindowsSDKLibVersion', 'SOURCE_DATE_EPOCH','GTEST_FILTER'] + 'WindowsSDKLibVersion', 'SOURCE_DATE_EPOCH'] if sys.platform == 'win32': pass_vars.append('COMSPEC') diff --git a/llvm/utils/lit/lit/formats/googletest.py b/llvm/utils/lit/lit/formats/googletest.py index 4ce14b78fe10d..5329f5773e54c 100644 --- a/llvm/utils/lit/lit/formats/googletest.py +++ b/llvm/utils/lit/lit/formats/googletest.py @@ -1,8 +1,8 @@ from __future__ import absolute_import -import json -import math import os +import re import shlex +import subprocess import sys import lit.Test @@ -25,19 +25,74 @@ def __init__(self, test_sub_dirs, test_suffix, run_under = []): self.test_suffixes = {exe_suffix, test_suffix + '.py'} self.run_under = run_under - def get_num_tests(self, path, localConfig): - cmd = [path, '--gtest_list_tests', '--gtest_filter=-*DISABLED_*'] - if cmd[0].endswith('.py'): - cmd = [sys.executable] + cmd - out, _, exitCode = lit.util.executeCommand(cmd, env=localConfig.environment) - if exitCode == 0: - return sum(map(lambda line: line.startswith(' '), out.splitlines())) - return None + def getGTestTests(self, path, litConfig, localConfig): + """getGTestTests(path) - [name] + + Return the tests available in gtest executable. + + Args: + path: String path to a gtest executable + litConfig: LitConfig instance + localConfig: TestingConfig instance""" + + list_test_cmd = self.prepareCmd([path, '--gtest_list_tests']) + + try: + output = subprocess.check_output(list_test_cmd, + env=localConfig.environment) + except subprocess.CalledProcessError as exc: + litConfig.warning( + "unable to discover google-tests in %r: %s. Process output: %s" + % (path, sys.exc_info()[1], exc.output)) + # This doesn't look like a valid gtest file. This can + # have a number of causes, none of them good. For + # instance, we could have created a broken executable. + # Alternatively, someone has cruft in their test + # directory. If we don't return a test here, then no + # failures will get reported, so return a dummy test name + # so that the failure is reported later. + yield 'failed_to_discover_tests_from_gtest' + return + + upstream_prefix = re.compile('Running main\(\) from .*gtest_main\.cc') + nested_tests = [] + for ln in output.splitlines(False): # Don't keep newlines. + ln = lit.util.to_string(ln) + + if upstream_prefix.fullmatch(ln): + # Upstream googletest prints this to stdout prior to running + # tests. LLVM removed that print statement in r61540, but we + # handle it here in case upstream googletest is being used. + continue + + # The test name list includes trailing comments beginning with + # a '#' on some lines, so skip those. We don't support test names + # that use escaping to embed '#' into their name as the names come + # from C++ class and method names where such things are hard and + # uninteresting to support. + ln = ln.split('#', 1)[0].rstrip() + if not ln.lstrip(): + continue + + index = 0 + while ln[index*2:index*2+2] == ' ': + index += 1 + while len(nested_tests) > index: + nested_tests.pop() + + ln = ln[index*2:] + if ln.endswith('.'): + nested_tests.append(ln) + elif any([name.startswith('DISABLED_') + for name in nested_tests + [ln]]): + # Gtest will internally skip these tests. No need to launch a + # child process for it. + continue + else: + yield ''.join(nested_tests) + ln def getTestsInDirectory(self, testSuite, path_in_suite, litConfig, localConfig): - init_shard_size = 512 # number of tests in a shard - core_count = lit.util.usable_core_count() source_path = testSuite.getSourcePath(path_in_suite) for subdir in self.test_sub_dirs: dir_path = os.path.join(source_path, subdir) @@ -47,40 +102,13 @@ def getTestsInDirectory(self, testSuite, path_in_suite, suffixes=self.test_suffixes): # Discover the tests in this executable. execpath = os.path.join(source_path, subdir, fn) - num_tests = self.get_num_tests(execpath, localConfig) - if num_tests is not None: - # Compute the number of shards. - shard_size = init_shard_size - nshard = int(math.ceil(num_tests/shard_size)) - while nshard < core_count and shard_size > 1: - shard_size = shard_size//2 - nshard = int(math.ceil(num_tests/shard_size)) - - # Create one lit test for each shard. - for idx in range(nshard): - testPath = path_in_suite + (subdir, fn, - str(idx), str(nshard)) - json_file = '-'.join([execpath, testSuite.config.name, - str(os.getpid()), str(idx), - str(nshard)]) + '.json' - yield lit.Test.Test(testSuite, testPath, localConfig, - file_path=execpath, - gtest_json_file=json_file) - else: - # This doesn't look like a valid gtest file. This can - # have a number of causes, none of them good. For - # instance, we could have created a broken executable. - # Alternatively, someone has cruft in their test - # directory. If we don't return a test here, then no - # failures will get reported, so return a dummy test name - # so that the failure is reported later. - testPath = path_in_suite + (subdir, fn, 'failed_to_discover_tests_from_gtest') - yield lit.Test.Test(testSuite, testPath, localConfig, file_path=execpath) + testnames = self.getGTestTests(execpath, litConfig, localConfig) + for testname in testnames: + testPath = path_in_suite + (subdir, fn, testname) + yield lit.Test.Test(testSuite, testPath, localConfig, + file_path=execpath) def execute(self, test, litConfig): - if test.gtest_json_file is None: - return lit.Test.FAIL, '' - testPath,testName = os.path.split(test.getSourcePath()) while not os.path.exists(testPath): # Handle GTest parametrized and typed tests, whose name includes @@ -88,12 +116,7 @@ def execute(self, test, litConfig): testPath, namePrefix = os.path.split(testPath) testName = namePrefix + '/' + testName - testName,total_shards = os.path.split(testName) - testName,shard_idx = os.path.split(testName) - shard_env = {'GTEST_COLOR':'no','GTEST_TOTAL_SHARDS':total_shards, 'GTEST_SHARD_INDEX':shard_idx, 'GTEST_OUTPUT':'json:'+test.gtest_json_file} - test.config.environment.update(shard_env) - - cmd = [testPath] + cmd = [testPath, '--gtest_filter=' + testName] cmd = self.prepareCmd(cmd) if litConfig.useValgrind: cmd = litConfig.valgrindArgs + cmd @@ -101,43 +124,30 @@ def execute(self, test, litConfig): if litConfig.noExecute: return lit.Test.PASS, '' - shard_envs= '\n'.join([k + '=' + v for k, v in shard_env.items()]) - shard_header = f"Script(shard):\n--\n{shard_envs}\n{' '.join(cmd)}\n--\n" + header = f"Script:\n--\n{' '.join(cmd)}\n--\n" try: - _, _, exitCode = lit.util.executeCommand( + out, err, exitCode = lit.util.executeCommand( cmd, env=test.config.environment, timeout=litConfig.maxIndividualTestTime) except lit.util.ExecuteCommandTimeoutException: return (lit.Test.TIMEOUT, - f'{shard_header}Reached timeout of ' + f'{header}Reached timeout of ' f'{litConfig.maxIndividualTestTime} seconds') - if not os.path.exists(test.gtest_json_file): - errmsg = f"shard JSON output does not exist: %s" % (test.gtest_json_file) - return lit.Test.FAIL, shard_header + errmsg - if exitCode: - output = shard_header + '\n' - with open(test.gtest_json_file, encoding='utf-8') as f: - testsuites = json.load(f)['testsuites'] - for testcase in testsuites: - for testinfo in testcase['testsuite']: - if testinfo['result'] == 'SUPPRESSED' or testinfo['result'] == 'SKIPPED': - continue - testname = testcase['name'] + '.' + testinfo['name'] - header = f"Script:\n--\n{' '.join(cmd)} --gtest_filter={testname}\n--\n" - if 'failures' in testinfo: - output += header - for fail in testinfo['failures']: - output += fail['failure'] + '\n' - output += '\n' - elif testinfo['result'] != 'COMPLETED': - output += header - output += 'unresolved test result\n' - return lit.Test.FAIL, output - else: - return lit.Test.PASS, '' + return lit.Test.FAIL, header + out + err + + if '[ SKIPPED ] 1 test,' in out: + return lit.Test.SKIPPED, '' + + passing_test_line = '[ PASSED ] 1 test.' + if passing_test_line not in out: + return (lit.Test.UNRESOLVED, + f'{header}Unable to find {passing_test_line} ' + f'in gtest output:\n\n{out}{err}') + + return lit.Test.PASS,'' def prepareCmd(self, cmd): """Insert interpreter if needed. @@ -156,61 +166,3 @@ def prepareCmd(self, cmd): else: cmd = shlex.split(self.run_under) + cmd return cmd - - @staticmethod - def post_process_shard_results(selected_tests, discovered_tests): - def remove_gtest(tests): - idxs = [] - for idx, t in enumerate(tests): - if t.gtest_json_file: - idxs.append(idx) - for i in range(len(idxs)): - del tests[idxs[i]-i] - - remove_gtest(discovered_tests) - gtests = [t for t in selected_tests if t.gtest_json_file] - remove_gtest(selected_tests) - for test in gtests: - # In case gtest has bugs such that no JSON file was emitted. - if not os.path.exists(test.gtest_json_file): - selected_tests.append(test) - discovered_tests.append(test) - continue - - # Load json file to retrieve results. - with open(test.gtest_json_file, encoding='utf-8') as f: - testsuites = json.load(f)['testsuites'] - for testcase in testsuites: - for testinfo in testcase['testsuite']: - # Ignore disabled tests. - if testinfo['result'] == 'SUPPRESSED': - continue - - testPath = test.path_in_suite[:-2] + (testcase['name'], testinfo['name']) - subtest = lit.Test.Test(test.suite, testPath, - test.config, test.file_path) - - testname = testcase['name'] + '.' + testinfo['name'] - header = f"Script:\n--\n{test.file_path} --gtest_filter={testname}\n--\n" - - output = '' - if testinfo['result'] == 'SKIPPED': - returnCode = lit.Test.SKIPPED - elif 'failures' in testinfo: - returnCode = lit.Test.FAIL - output = header - for fail in testinfo['failures']: - output += fail['failure'] + '\n' - elif testinfo['result'] == 'COMPLETED': - returnCode = lit.Test.PASS - else: - returnCode = lit.Test.UNRESOLVED - output = header + 'unresolved test result\n' - - subtest.setResult(lit.Test.Result(returnCode, output, float(testinfo['time'][:-1]))) - - selected_tests.append(subtest) - discovered_tests.append(subtest) - os.remove(test.gtest_json_file) - - return selected_tests, discovered_tests diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py index aacb579c8b03c..41f124a27ad7f 100755 --- a/llvm/utils/lit/lit/main.py +++ b/llvm/utils/lit/lit/main.py @@ -18,7 +18,6 @@ import lit.run import lit.Test import lit.util -from lit.formats.googletest import GoogleTest from lit.TestTimes import record_test_times @@ -109,9 +108,6 @@ def main(builtin_params={}): record_test_times(selected_tests, lit_config) - selected_tests, discovered_tests = GoogleTest.post_process_shard_results( - selected_tests, discovered_tests) - if opts.time_tests: print_histogram(discovered_tests) diff --git a/llvm/utils/lit/tests/Inputs/googletest-crash/DummySubDir/OneTest.py b/llvm/utils/lit/tests/Inputs/googletest-crash/DummySubDir/OneTest.py deleted file mode 100644 index bebedb6f0e12f..0000000000000 --- a/llvm/utils/lit/tests/Inputs/googletest-crash/DummySubDir/OneTest.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python - -import os -import sys - -if len(sys.argv) == 3 and sys.argv[1] == "--gtest_list_tests": - if sys.argv[2] != '--gtest_filter=-*DISABLED_*': - raise ValueError("unexpected argument: %s" % (sys.argv[2])) - print("""\ -FirstTest. - subTestA - subTestB - subTestC - subTestD -ParameterizedTest/0. - subTest -ParameterizedTest/1. - subTest""") - sys.exit(0) -elif len(sys.argv) != 1: - # sharding and json output are specified using environment variables - raise ValueError("unexpected argument: %r" % (' '.join(sys.argv[1:]))) - -for e in ['GTEST_TOTAL_SHARDS', 'GTEST_SHARD_INDEX', 'GTEST_OUTPUT']: - if e not in os.environ: - raise ValueError("missing environment variables: " + e) - -if not os.environ['GTEST_OUTPUT'].startswith('json:'): - raise ValueError("must emit json output: " + os.environ['GTEST_OUTPUT']) - -dummy_output = """\ -{ -"testsuites": [ -] -}""" - -if os.environ['GTEST_SHARD_INDEX'] == '0': - exit_code = 1 -else: - json_filename = os.environ['GTEST_OUTPUT'].split(':', 1)[1] - with open(json_filename, 'w') as f: - f.write(dummy_output) - exit_code = 0 - -sys.exit(exit_code) diff --git a/llvm/utils/lit/tests/Inputs/googletest-format/DummySubDir/OneTest.py b/llvm/utils/lit/tests/Inputs/googletest-format/DummySubDir/OneTest.py index c66696f77a519..7bff6b6252b25 100644 --- a/llvm/utils/lit/tests/Inputs/googletest-format/DummySubDir/OneTest.py +++ b/llvm/utils/lit/tests/Inputs/googletest-format/DummySubDir/OneTest.py @@ -1,11 +1,11 @@ #!/usr/bin/env python -import os import sys -if len(sys.argv) == 3 and sys.argv[1] == "--gtest_list_tests": - if sys.argv[2] != '--gtest_filter=-*DISABLED_*': - raise ValueError("unexpected argument: %s" % (sys.argv[2])) +if len(sys.argv) != 2: + raise ValueError("unexpected number of args") + +if sys.argv[1] == "--gtest_list_tests": print("""\ FirstTest. subTestA @@ -17,87 +17,31 @@ ParameterizedTest/1. subTest""") sys.exit(0) -elif len(sys.argv) != 1: - # sharding and json output are specified using environment variables - raise ValueError("unexpected argument: %r" % (' '.join(sys.argv[1:]))) - -for e in ['GTEST_TOTAL_SHARDS', 'GTEST_SHARD_INDEX', 'GTEST_OUTPUT']: - if e not in os.environ: - raise ValueError("missing environment variables: " + e) - -if not os.environ['GTEST_OUTPUT'].startswith('json:'): - raise ValueError("must emit json output: " + os.environ['GTEST_OUTPUT']) - -output = """\ -{ -"testsuites": [ - { - "name": "FirstTest", - "testsuite": [ - { - "name": "subTestA", - "result": "COMPLETED", - "time": "0.001s" - }, - { - "name": "subTestB", - "result": "COMPLETED", - "time": "0.001s", - "failures": [ - { - "failure": "I am subTest B, I FAIL\\nAnd I have two lines of output", - "type": "" - } - ] - }, - { - "name": "subTestC", - "result": "SKIPPED", - "time": "0.001s" - }, - { - "name": "subTestD", - "result": "UNRESOLVED", - "time": "0.001s" - } - ] - }, - { - "name": "ParameterizedTest/0", - "testsuite": [ - { - "name": "subTest", - "result": "COMPLETED", - "time": "0.001s" - } - ] - }, - { - "name": "ParameterizedTest/1", - "testsuite": [ - { - "name": "subTest", - "result": "COMPLETED", - "time": "0.001s" - } - ] - } -] -}""" - -dummy_output = """\ -{ -"testsuites": [ -] -}""" +elif not sys.argv[1].startswith("--gtest_filter="): + raise ValueError("unexpected argument: %r" % (sys.argv[1])) -json_filename = os.environ['GTEST_OUTPUT'].split(':', 1)[1] -with open(json_filename, 'w') as f: - if os.environ['GTEST_SHARD_INDEX'] == '0': - f.write(output) - exit_code = 1 - else: - f.write(dummy_output) - exit_code = 0 - -sys.exit(exit_code) +test_name = sys.argv[1].split('=',1)[1] +if test_name == 'FirstTest.subTestA': + print('I am subTest A, I PASS') + print('[ PASSED ] 1 test.') + sys.exit(0) +elif test_name == 'FirstTest.subTestB': + print('I am subTest B, I FAIL') + print('And I have two lines of output') + sys.exit(1) +elif test_name == 'FirstTest.subTestC': + print('I am subTest C, I am SKIPPED') + print('[ PASSED ] 0 tests.') + print('[ SKIPPED ] 1 test, listed below:') + print('[ SKIPPED ] FirstTest.subTestC') + sys.exit(0) +elif test_name == 'FirstTest.subTestD': + print('I am subTest D, I am UNRESOLVED') + sys.exit(0) +elif test_name in ('ParameterizedTest/0.subTest', + 'ParameterizedTest/1.subTest'): + print('I am a parameterized test, I also PASS') + print('[ PASSED ] 1 test.') + sys.exit(0) +else: + raise SystemExit("error: invalid test name: %r" % (test_name,)) diff --git a/llvm/utils/lit/tests/Inputs/googletest-upstream-format/DummySubDir/OneTest.py b/llvm/utils/lit/tests/Inputs/googletest-upstream-format/DummySubDir/OneTest.py new file mode 100644 index 0000000000000..aa79a22c0b106 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/googletest-upstream-format/DummySubDir/OneTest.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python + +import os +import sys + +if len(sys.argv) != 2: + raise ValueError("unexpected number of args") + +if sys.argv[1] == "--gtest_list_tests": + print(f"""\ +Running main() from {os.getcwd()}/gtest_main.cc +FirstTest. + subTestA + subTestB + subTestC + subTestD +ParameterizedTest/0. + subTest +ParameterizedTest/1. + subTest""") + sys.exit(0) +elif not sys.argv[1].startswith("--gtest_filter="): + raise ValueError("unexpected argument: %r" % (sys.argv[1])) + +test_name = sys.argv[1].split('=',1)[1] +print('Running main() from gtest_main.cc') +if test_name == 'FirstTest.subTestA': + print('I am subTest A, I PASS') + print('[ PASSED ] 1 test.') + sys.exit(0) +elif test_name == 'FirstTest.subTestB': + print('I am subTest B, I FAIL') + print('And I have two lines of output') + sys.exit(1) +elif test_name == 'FirstTest.subTestC': + print('I am subTest C, I am SKIPPED') + print('[ PASSED ] 0 tests.') + print('[ SKIPPED ] 1 test, listed below:') + print('[ SKIPPED ] FirstTest.subTestC') + sys.exit(0) +elif test_name == 'FirstTest.subTestD': + print('I am subTest D, I am UNRESOLVED') + sys.exit(0) +elif test_name in ('ParameterizedTest/0.subTest', + 'ParameterizedTest/1.subTest'): + print('I am a parameterized test, I also PASS') + print('[ PASSED ] 1 test.') + sys.exit(0) +else: + raise SystemExit("error: invalid test name: %r" % (test_name,)) diff --git a/llvm/utils/lit/tests/Inputs/googletest-crash/lit.cfg b/llvm/utils/lit/tests/Inputs/googletest-upstream-format/lit.cfg similarity index 66% rename from llvm/utils/lit/tests/Inputs/googletest-crash/lit.cfg rename to llvm/utils/lit/tests/Inputs/googletest-upstream-format/lit.cfg index 8048411a70882..9fb5d2b0247be 100644 --- a/llvm/utils/lit/tests/Inputs/googletest-crash/lit.cfg +++ b/llvm/utils/lit/tests/Inputs/googletest-upstream-format/lit.cfg @@ -1,3 +1,3 @@ import lit.formats -config.name = 'googletest-crash' +config.name = 'googletest-upstream-format' config.test_format = lit.formats.GoogleTest('DummySubDir', 'Test') diff --git a/llvm/utils/lit/tests/googletest-crash.py b/llvm/utils/lit/tests/googletest-crash.py deleted file mode 100644 index 1cbe05b4150ea..0000000000000 --- a/llvm/utils/lit/tests/googletest-crash.py +++ /dev/null @@ -1,20 +0,0 @@ -# Check GoogleTest shard test crashes are handled. - -# RUN: not %{lit} -v %{inputs}/googletest-crash | FileCheck %s - -# CHECK: -- Testing: -# CHECK: FAIL: googletest-crash :: [[PATH:[Dd]ummy[Ss]ub[Dd]ir/]][[FILE:OneTest\.py]]/0 -# CHECK: *** TEST 'googletest-crash :: [[PATH]][[FILE]]/0{{.*}} FAILED *** -# CHECK-NEXT: Script(shard): -# CHECK-NEXT: -- -# CHECK-NEXT: GTEST_COLOR=no -# CHECK-NEXT: GTEST_TOTAL_SHARDS=6 -# CHECK-NEXT: GTEST_SHARD_INDEX=0 -# CHECK-NEXT: GTEST_OUTPUT=json:[[JSON:.*\.json]] -# CHECK-NEXT: [[FILE]] -# CHECK-NEXT: -- -# CHECK-NEXT: shard JSON output does not exist: [[JSON]] -# CHECK-NEXT: *** -# CHECK: Failed Tests (1): -# CHECK-NEXT: googletest-crash :: [[PATH]][[FILE]]/0/6 -# CHECK: Failed{{ *}}: 1 \ No newline at end of file diff --git a/llvm/utils/lit/tests/googletest-format.py b/llvm/utils/lit/tests/googletest-format.py index 5d5f1f9e96f04..b960d0cdc64e8 100644 --- a/llvm/utils/lit/tests/googletest-format.py +++ b/llvm/utils/lit/tests/googletest-format.py @@ -9,35 +9,28 @@ # END. # CHECK: -- Testing: -# CHECK: FAIL: googletest-format :: [[PATH:[Dd]ummy[Ss]ub[Dd]ir/]][[FILE:OneTest\.py]]/0 -# CHECK: *** TEST 'googletest-format :: [[PATH]][[FILE]]/0{{.*}} FAILED *** -# CHECK-NEXT: Script(shard): -# CHECK-NEXT: -- -# CHECK-NEXT: GTEST_COLOR=no -# CHECK-NEXT: GTEST_TOTAL_SHARDS=6 -# CHECK-NEXT: GTEST_SHARD_INDEX=0 -# CHECK-NEXT: GTEST_OUTPUT=json:{{.*\.json}} -# CHECK-NEXT: [[FILE]] -# CHECK-NEXT: -- -# CHECK-EMPTY: +# CHECK: PASS: googletest-format :: [[PATH:[Dd]ummy[Ss]ub[Dd]ir/]][[FILE:OneTest\.py]]/FirstTest.subTestA +# CHECK: FAIL: googletest-format :: [[PATH]][[FILE]]/[[TEST:FirstTest\.subTestB]] +# CHECK-NEXT: *** TEST 'googletest-format :: [[PATH]][[FILE]]/[[TEST]]' FAILED *** # CHECK-NEXT: Script: # CHECK-NEXT: -- -# CHECK-NEXT: [[FILE]] --gtest_filter=FirstTest.subTestB +# CHECK-NEXT: [[FILE]] --gtest_filter=[[TEST]] # CHECK-NEXT: -- # CHECK-NEXT: I am subTest B, I FAIL # CHECK-NEXT: And I have two lines of output -# CHECK-EMPTY: -# CHECK: Script: +# CHECK: *** +# CHECK: SKIPPED: googletest-format :: [[PATH]][[FILE]]/FirstTest.subTestC +# CHECK: UNRESOLVED: googletest-format :: [[PATH]][[FILE]]/[[TEST:FirstTest\.subTestD]] +# CHECK-NEXT: *** TEST 'googletest-format :: [[PATH]][[FILE]]/[[TEST]]' FAILED *** +# CHECK-NEXT: Script: # CHECK-NEXT: -- -# CHECK-NEXT: [[FILE]] --gtest_filter=FirstTest.subTestD +# CHECK-NEXT: [[FILE]] --gtest_filter=[[TEST]] # CHECK-NEXT: -- -# CHECK-NEXT: unresolved test result -# CHECK: *** -# CHECK: Unresolved Tests (1): -# CHECK-NEXT: googletest-format :: [[PATH]][[FILE]]/FirstTest/subTestD -# CHECK: *** -# CHECK-NEXT: Failed Tests (1): -# CHECK-NEXT: googletest-format :: [[PATH]][[FILE]]/FirstTest/subTestB +# CHECK-NEXT: Unable to find [ PASSED ] 1 test. in gtest output +# CHECK: I am subTest D, I am UNRESOLVED +# CHECK: PASS: googletest-format :: [[PATH]][[FILE]]/ParameterizedTest/0.subTest +# CHECK: PASS: googletest-format :: [[PATH]][[FILE]]/ParameterizedTest/1.subTest +# CHECK: Failed Tests (1) # CHECK: Skipped{{ *}}: 1 # CHECK: Passed{{ *}}: 3 # CHECK: Unresolved{{ *}}: 1 diff --git a/llvm/utils/lit/tests/googletest-upstream-format.py b/llvm/utils/lit/tests/googletest-upstream-format.py new file mode 100644 index 0000000000000..cf0f73e2d31ed --- /dev/null +++ b/llvm/utils/lit/tests/googletest-upstream-format.py @@ -0,0 +1,35 @@ +# Check the various features of the GoogleTest format. + +# RUN: not %{lit} -v %{inputs}/googletest-upstream-format > %t.out +# RUN: FileCheck < %t.out %s +# +# END. + +# CHECK: -- Testing: +# CHECK: PASS: googletest-upstream-format :: [[PATH:[Dd]ummy[Ss]ub[Dd]ir/]][[FILE:OneTest\.py]]/FirstTest.subTestA +# CHECK: FAIL: googletest-upstream-format :: [[PATH]][[FILE]]/[[TEST:FirstTest\.subTestB]] +# CHECK-NEXT: *** TEST 'googletest-upstream-format :: [[PATH]][[FILE]]/[[TEST]]' FAILED *** +# CHECK-NEXT: Script: +# CHECK-NEXT: -- +# CHECK-NEXT: [[FILE]] --gtest_filter=[[TEST]] +# CHECK-NEXT: -- +# CHECK-NEXT: Running main() from gtest_main.cc +# CHECK-NEXT: I am subTest B, I FAIL +# CHECK-NEXT: And I have two lines of output +# CHECK: SKIPPED: googletest-upstream-format :: [[PATH]][[FILE]]/FirstTest.subTestC +# CHECK: UNRESOLVED: googletest-upstream-format :: [[PATH]][[FILE]]/[[TEST:FirstTest\.subTestD]] +# CHECK-NEXT: *** TEST 'googletest-upstream-format :: [[PATH]][[FILE]]/[[TEST]]' FAILED *** +# CHECK-NEXT: Script: +# CHECK-NEXT: -- +# CHECK-NEXT: [[FILE]] --gtest_filter=[[TEST]] +# CHECK-NEXT: -- +# CHECK-NEXT: Unable to find [ PASSED ] 1 test. in gtest output +# CHECK: I am subTest D, I am UNRESOLVED +# CHECK: *** +# CHECK: PASS: googletest-upstream-format :: [[PATH]][[FILE]]/ParameterizedTest/0.subTest +# CHECK: PASS: googletest-upstream-format :: [[PATH]][[FILE]]/ParameterizedTest/1.subTest +# CHECK: Failed Tests (1) +# CHECK: Skipped{{ *}}: 1 +# CHECK: Passed{{ *}}: 3 +# CHECK: Unresolved{{ *}}: 1 +# CHECK: Failed{{ *}}: 1