Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
test infra: catch bad decorators and import-time errors
Summary: This change enhances the LLDB test infrastructure to convert load-time exceptions in a given Python test module into errors. Before this change, specifying a non-existent test decorator, or otherwise having some load-time error in a python test module, would not get flagged as an error. With this change, typos and other load-time errors in a python test file get converted to errors and reported by the test runner. This change also includes test infrastructure tests that include covering the new work here. I'm going to wait until we have these infrastructure tests runnable on the main platforms before I try to work that into all the normal testing workflows. The test infrastructure tests can be run by using the standard python module testing practice of doing the following: cd packages/Python/lldbsuite/test_event python -m unittest discover -s test/src -p 'Test*.py' Those tests run the dotest inferior with a known broken test and verify that the errors are caught. These tests did not pass until I modified dotest.py to capture them properly. @zturner, if you have the chance, if you could try those steps above (the python -m unittest ... line) on Windows, that would be great if we can address any python2/3/Windows bits there. I don't think there's anything fancy, but I didn't want to hook it into test flow until I know it works there. I'll be slowly adding more tests that cover some of the other breakage I've occasionally seen that didn't get collected as part of the summarization. This is the biggest one I'm aware of. Reviewers: zturner, labath Subscribers: zturner, lldb-commits Differential Revision: http://reviews.llvm.org/D20193 llvm-svn: 269489
- Loading branch information
Showing
11 changed files
with
307 additions
and
67 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
13 changes: 13 additions & 0 deletions
13
lldb/packages/Python/lldbsuite/test/issue_verification/TestInvalidDecorator.py.park
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
from __future__ import print_function | ||
from lldbsuite.test import lldbtest | ||
from lldbsuite.test import decorators | ||
|
||
|
||
class NonExistentDecoratorTestCase(lldbtest.TestBase): | ||
|
||
mydir = lldbtest.TestBase.compute_mydir(__file__) | ||
|
||
@decorators.nonExistentDecorator(bugnumber="yt/1300") | ||
def test(self): | ||
"""Verify non-existent decorators are picked up by test runner.""" | ||
pass |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
13 changes: 13 additions & 0 deletions
13
...ages/Python/lldbsuite/test_event/test/resources/invalid_decorator/TestInvalidDecorator.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
from __future__ import print_function | ||
from lldbsuite.test import lldbtest | ||
from lldbsuite.test import decorators | ||
|
||
|
||
class NonExistentDecoratorTestCase(lldbtest.TestBase): | ||
|
||
mydir = lldbtest.TestBase.compute_mydir(__file__) | ||
|
||
@decorators.nonExistentDecorator(bugnumber="yt/1300") | ||
def test(self): | ||
"""Verify non-existent decorators are picked up by test runner.""" | ||
pass |
70 changes: 70 additions & 0 deletions
70
lldb/packages/Python/lldbsuite/test_event/test/src/TestCatchInvalidDecorator.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,70 @@ | ||
#!/usr/bin/env python | ||
""" | ||
Tests that the event system reports issues during decorator | ||
handling as errors. | ||
""" | ||
# System-provided imports | ||
import os | ||
import unittest | ||
|
||
# Local-provided imports | ||
import event_collector | ||
|
||
|
||
class TestCatchInvalidDecorator(unittest.TestCase): | ||
|
||
TEST_DIR = os.path.join( | ||
os.path.dirname(__file__), | ||
os.path.pardir, | ||
"resources", | ||
"invalid_decorator") | ||
|
||
def test_with_whole_file(self): | ||
""" | ||
Test that a non-existent decorator generates a test-event error | ||
when running all tests in the file. | ||
""" | ||
# Determine the test case file we're using. | ||
test_file = os.path.join(self.TEST_DIR, "TestInvalidDecorator.py") | ||
|
||
# Collect all test events generated for this file. | ||
error_results = _filter_error_results( | ||
event_collector.collect_events_whole_file(test_file)) | ||
|
||
self.assertGreater( | ||
len(error_results), | ||
0, | ||
"At least one job or test error result should have been returned") | ||
|
||
def test_with_function_filter(self): | ||
""" | ||
Test that a non-existent decorator generates a test-event error | ||
when running a filtered test. | ||
""" | ||
# Collect all test events generated during running of tests | ||
# in a given directory using a test name filter. Internally, | ||
# this runs through a different code path that needs to be | ||
# set up to catch exceptions. | ||
error_results = _filter_error_results( | ||
event_collector.collect_events_for_directory_with_filter( | ||
self.TEST_DIR, | ||
"NonExistentDecoratorTestCase.test")) | ||
|
||
self.assertGreater( | ||
len(error_results), | ||
0, | ||
"At least one job or test error result should have been returned") | ||
|
||
|
||
def _filter_error_results(events): | ||
# Filter out job result events. | ||
return [ | ||
event | ||
for event in events | ||
if event.get("event", None) in ["job_result", "test_result"] and | ||
event.get("status", None) == "error" | ||
] | ||
|
||
|
||
if __name__ == "__main__": | ||
unittest.main() |
Oops, something went wrong.