Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Do expected error checking in (Check|Infer)WithErrors.
Now that we have comment-based error checking, I really wanted to be able to insert expected errors into test code and have them checked without needing to call a separate assert method afterwards. This CL: * Fixes a couple cases where errors were reported in a non-deterministic order, so that a mark can always be associated with the right error. * Has CheckWithErrors and InferWithErrors call assert_errors_match_expected() on the errorlog. * Forbids adding expected error comments to Check, Infer, and InferFromFile, so that we don't accidentally use one of these methods and have our expected errors silently ignored. * Replaces assertErrorLogIs and assertErrorsMatch with an assertErrorRegexes method that only has to be called when we want to check marks. * Makes some simplifications to the line number handling and error matching code that are possible because of the removal of assertErrorLogIs. On its own, this CL does not pass tests; a follow-up will migrate all uses of assertErrorLogIs over to assertErrorRegexes. I'll wait until both CLs have been reviewed, then submit them in quick succession. PiperOrigin-RevId: 303863481
- Loading branch information
Showing
6 changed files
with
202 additions
and
117 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -18,6 +18,7 @@ py_test( | |
| test_base_test.py | ||
| DEPS | ||
| .test_base | ||
| pytype.utils | ||
| ) | ||
|
|
||
| py_test( | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,31 +1,123 @@ | ||
| """Tests for our test framework.""" | ||
|
|
||
| from pytype import file_utils | ||
| from pytype import utils | ||
| from pytype.tests import test_base | ||
| import six | ||
| from pytype.tests import test_utils | ||
|
|
||
|
|
||
| class ErrorLogTest(test_base.TargetIndependentTest): | ||
|
|
||
| def _lineno(self, line): | ||
| if self.options.python_version == (2, 7) and utils.USE_ANNOTATIONS_BACKPORT: | ||
| return line + 1 | ||
| return line | ||
|
|
||
| def test_error_comments(self): | ||
| err = self.CheckWithErrors("""\ | ||
| a = 10 # a random comment | ||
| b = "hello" # .mark | ||
| c = a + b # some-error | ||
| d = a + b # .another_mark | ||
| b = "hello" + 3 # unsupported-operands[.mark] | ||
| c = (10).foo # attribute-error | ||
| d = int(int) # wrong-arg-types[.another_mark] | ||
| """) | ||
| self.assertEqual( | ||
| {mark: (e.lineno, e.name) for mark, e in err.marks.items()}, | ||
| {".mark": (self._lineno(2), "unsupported-operands"), | ||
| ".another_mark": (self._lineno(4), "wrong-arg-types")}) | ||
| self.assertEqual(err.expected, { | ||
| self._lineno(2): [("unsupported-operands", ".mark")], | ||
| self._lineno(3): [("attribute-error", None)], | ||
| self._lineno(4): [("wrong-arg-types", ".another_mark")]}) | ||
|
|
||
| def test_multiple_errors_one_line(self): | ||
| err = self.CheckWithErrors("""\ | ||
| x = (10).foo, "hello".foo # attribute-error[e1] # attribute-error[e2] | ||
| """) | ||
| six.assertCountEqual(self, err.marks.keys(), [".mark", ".another_mark"]) | ||
| self.assertEqual(err.marks[".mark"], 2) | ||
| six.assertCountEqual(self, err.expected.keys(), [3]) | ||
| six.assertCountEqual(self, err.expected[3], "some-error") | ||
| line = self._lineno(1) | ||
| self.assertEqual(err.expected, {line: [("attribute-error", "e1"), | ||
| ("attribute-error", "e2")]}) | ||
| self.assertCountEqual(err.marks, ["e1", "e2"]) | ||
| self.assertIn("on int", err.marks["e1"].message) | ||
| self.assertIn("on str", err.marks["e2"].message) | ||
|
|
||
| def test_populate_marks(self): | ||
| # Test that assert_error_regexes populates self.marks if not already done. | ||
| errorlog = test_utils.TestErrorLog("x = 0") | ||
| self.assertIsNone(errorlog.marks) | ||
| self.assertErrorRegexes(errorlog, {}) | ||
| self.assertIsNotNone(errorlog.marks) | ||
|
|
||
| def test_duplicate_mark(self): | ||
| with self.assertRaises(AssertionError) as ctx: | ||
| self.CheckWithErrors("x = 0 # attribute-error[e] # attribute-error[e]") | ||
| self.assertEqual(str(ctx.exception), "Mark e already used") | ||
|
|
||
| def test_error_matching(self): | ||
| err = self.CheckWithErrors("""\ | ||
| a = 10 | ||
| b = "hello" | ||
| c = a + b # unsupported-operands | ||
| d = a.foo() # .mark | ||
| d = a.foo() # attribute-error[.mark] | ||
| """) | ||
| self.assertErrorsMatch(err, [(".mark", "attribute-error", ".*foo.*")]) | ||
| self.assertErrorRegexes(err, {".mark": ".*foo.*"}) | ||
|
|
||
| def test_mismatched_error(self): | ||
| with self.assertRaises(AssertionError) as ctx: | ||
| self.CheckWithErrors("(10).foo # wrong-arg-types") | ||
| self.assertIn("Error does not match", str(ctx.exception)) | ||
|
|
||
| def test_unexpected_error(self): | ||
| with self.assertRaises(AssertionError) as ctx: | ||
| self.CheckWithErrors("""\ | ||
| (10).foo # attribute-error | ||
| "hello".foo | ||
| """) | ||
| self.assertIn("Unexpected error", str(ctx.exception)) | ||
|
|
||
| def test_leftover_error(self): | ||
| with self.assertRaises(AssertionError) as ctx: | ||
| self.CheckWithErrors("x = 0 # attribute-error") | ||
| self.assertIn("Errors not found", str(ctx.exception)) | ||
|
|
||
| def test_misspelled_leftover_error(self): | ||
| with self.assertRaises(AssertionError) as ctx: | ||
| self.CheckWithErrors("x = 0 # misspelled-error") | ||
| self.assertIn("Errors not found", str(ctx.exception)) | ||
|
|
||
| def test_mismatched_regex(self): | ||
| err = self.CheckWithErrors("(10).foo # attribute-error[e]") | ||
| with self.assertRaises(AssertionError) as ctx: | ||
| self.assertErrorRegexes(err, {"e": r"does not match error message"}) | ||
| self.assertIn("Bad error message", str(ctx.exception)) | ||
|
|
||
| def test_missing_regex(self): | ||
| err = self.CheckWithErrors("(10).foo # attribute-error[e]") | ||
| with self.assertRaises(AssertionError) as ctx: | ||
| self.assertErrorRegexes(err, {}) | ||
| self.assertEqual(str(ctx.exception), "No regex for mark e") | ||
|
|
||
| def test_leftover_regex(self): | ||
| err = self.CheckWithErrors("x = 0") | ||
| with self.assertRaises(AssertionError) as ctx: | ||
| self.assertErrorRegexes(err, {"e": ""}) | ||
| self.assertEqual(str(ctx.exception), "Marks not found in code: e") | ||
|
|
||
| def test_bad_check(self): | ||
| with self.assertRaises(AssertionError) as ctx: | ||
| self.Check("name_error # name-error") | ||
| self.assertIn("Cannot assert errors", str(ctx.exception)) | ||
|
|
||
| def test_bad_infer(self): | ||
| with self.assertRaises(AssertionError) as ctx: | ||
| self.Infer("name_error # name-error") | ||
| self.assertIn("Cannot assert errors", str(ctx.exception)) | ||
|
|
||
| def test_bad_infer_from_file(self): | ||
| with file_utils.Tempdir() as d: | ||
| d.create_file("some_file.py", "name_error # name-error") | ||
| with self.assertRaises(AssertionError) as ctx: | ||
| self.InferFromFile(filename=d["some_file.py"], pythonpath=[]) | ||
| self.assertIn("Cannot assert errors", str(ctx.exception)) | ||
|
|
||
|
|
||
| test_base.main(globals(), __name__ == "__main__") |
Oops, something went wrong.