From ff609565055713c1539b53f178cf38230d1a4722 Mon Sep 17 00:00:00 2001 From: Enrico Minack Date: Fri, 15 Dec 2023 14:40:36 +0100 Subject: [PATCH 1/3] Add example file with flaky tests --- python/test/files/junit-xml/flaky.xml | 57 +++++++++++++++++++++++++++ python/test/test_action_script.py | 34 ++++++++-------- 2 files changed, 74 insertions(+), 17 deletions(-) create mode 100644 python/test/files/junit-xml/flaky.xml diff --git a/python/test/files/junit-xml/flaky.xml b/python/test/files/junit-xml/flaky.xml new file mode 100644 index 00000000..0d9b65b7 --- /dev/null +++ b/python/test/files/junit-xml/flaky.xml @@ -0,0 +1,57 @@ + + + + + + + + + + + + + + + + expected: <1> but was: <2> + at example.ExampleTests.failsAssertionAlways(ExampleTests.java:40) +]]> + + expected: <1> but was: <2> + at example.ExampleTests.failsAssertionAlways(ExampleTests.java:40) +]]> + + + expected: <1> but was: <2> + at example.ExampleTests.failsAssertionAlways(ExampleTests.java:40) +]]> + + + expected: <1> but was: <2> + at example.ExampleTests.failsAssertionAlways(ExampleTests.java:40) +]]> + + + + + expected: <1> but was: <2> + at example.ExampleTests.failsAssertionOnlyFirstAttempt(ExampleTests.java:32) +]]> + + + + + + + + diff --git a/python/test/test_action_script.py b/python/test/test_action_script.py index dae244d9..8db4839f 100644 --- a/python/test/test_action_script.py +++ b/python/test/test_action_script.py @@ -982,12 +982,12 @@ def test_parse_files(self): print(call.args[0]) self.assertEqual(17, len(l.info.call_args_list)) - self.assertTrue(any([call.args[0].startswith(f"Reading files {prettify_glob_pattern(settings.files_glob)} (76 files, ") for call in l.info.call_args_list])) - self.assertTrue(any([call.args[0].startswith(f'Reading JUnit XML files {prettify_glob_pattern(settings.junit_files_glob)} (28 files, ') for call in l.info.call_args_list])) + self.assertTrue(any([call.args[0].startswith(f"Reading files {prettify_glob_pattern(settings.files_glob)} (77 files, ") for call in l.info.call_args_list])) + self.assertTrue(any([call.args[0].startswith(f'Reading JUnit XML files {prettify_glob_pattern(settings.junit_files_glob)} (29 files, ') for call in l.info.call_args_list])) self.assertTrue(any([call.args[0].startswith(f'Reading NUnit XML files {prettify_glob_pattern(settings.nunit_files_glob)} (24 files, ') for call in l.info.call_args_list])) self.assertTrue(any([call.args[0].startswith(f'Reading XUnit XML files {prettify_glob_pattern(settings.xunit_files_glob)} (8 files, ') for call in l.info.call_args_list])) self.assertTrue(any([call.args[0].startswith(f'Reading TRX files {prettify_glob_pattern(settings.trx_files_glob)} (9 files, ') for call in l.info.call_args_list])) - self.assertTrue(any([call.args[0].startswith(f'Detected 27 JUnit XML files (') for call in l.info.call_args_list])) + self.assertTrue(any([call.args[0].startswith(f'Detected 28 JUnit XML files (') for call in l.info.call_args_list])) self.assertTrue(any([call.args[0].startswith(f'Detected 24 NUnit XML files (') for call in l.info.call_args_list])) self.assertTrue(any([call.args[0].startswith(f'Detected 8 XUnit XML files (') for call in l.info.call_args_list])) self.assertTrue(any([call.args[0].startswith(f'Detected 9 TRX files (') for call in l.info.call_args_list])) @@ -999,7 +999,7 @@ def test_parse_files(self): self.assertTrue(any([call.args[0].endswith(f'python{os.sep}test{os.sep}files{os.sep}junit-xml{os.sep}non-junit.xml') for call in l.info.call_args_list])) self.assertTrue(any([call.args[0].endswith(f'python{os.sep}test{os.sep}files{os.sep}json{os.sep}non-json.json') for call in l.info.call_args_list])) self.assertTrue(any([call.args[0].endswith(f'python{os.sep}test{os.sep}files{os.sep}json{os.sep}malformed-json.json') for call in l.info.call_args_list])) - self.assertTrue(any([call.args[0].startswith(f'Finished reading 145 files in ') for call in l.info.call_args_list])) + self.assertTrue(any([call.args[0].startswith(f'Finished reading 147 files in ') for call in l.info.call_args_list])) for call in l.debug.call_args_list: print(call.args[0]) @@ -1019,7 +1019,7 @@ def test_parse_files(self): self.assertEqual([], gha.method_calls) - self.assertEqual(145, actual.files) + self.assertEqual(147, actual.files) if Version(sys.version.split(' ')[0]) < Version('3.9.0') and sys.platform.startswith('darwin') and \ (platform.mac_ver()[0].startswith("11.") or platform.mac_ver()[0].startswith("12.")): # on macOS and below Python 3.9 we see one particular error @@ -1034,14 +1034,14 @@ def test_parse_files(self): self.assertEqual(4085, len(actual.cases)) else: self.assertEqual(13, len(actual.errors)) - self.assertEqual(735, actual.suites) - self.assertEqual(4117, actual.suite_tests) + self.assertEqual(737, actual.suites) + self.assertEqual(4121, actual.suite_tests) self.assertEqual(214, actual.suite_skipped) - self.assertEqual(454, actual.suite_failures) - self.assertEqual(21, actual.suite_errors) + self.assertEqual(456, actual.suite_failures) + self.assertEqual(23, actual.suite_errors) self.assertEqual(7957, actual.suite_time) self.assertEqual(0, len(actual.suite_details)) - self.assertEqual(4093, len(actual.cases)) + self.assertEqual(4101, len(actual.cases)) self.assertEqual('commit', actual.commit) with io.StringIO() as string: @@ -1195,17 +1195,17 @@ def test_main(self): # Publisher.publish is expected to have been called with these arguments results, cases, conclusion = m.call_args_list[0].args - self.assertEqual(145, results.files) + self.assertEqual(147, results.files) if Version(sys.version.split(' ')[0]) < Version('3.9.0') and sys.platform.startswith('darwin') and \ (platform.mac_ver()[0].startswith("11.") or platform.mac_ver()[0].startswith("12.")): # on macOS and below Python 3.9 we see one particular error - self.assertEqual(731, results.suites) - self.assertEqual(731, len(results.suite_details)) - self.assertEqual(1811, len(cases)) + self.assertEqual(733, results.suites) + self.assertEqual(733, len(results.suite_details)) + self.assertEqual(1815, len(cases)) else: - self.assertEqual(735, results.suites) - self.assertEqual(735, len(results.suite_details)) - self.assertEqual(1811, len(cases)) + self.assertEqual(737, results.suites) + self.assertEqual(737, len(results.suite_details)) + self.assertEqual(1815, len(cases)) self.assertEqual('failure', conclusion) def test_main_fork_pr_check_wo_summary(self): From 173dcc680e8deb3a809535c7349b632f3d076b6b Mon Sep 17 00:00:00 2001 From: Enrico Minack Date: Fri, 15 Dec 2023 18:38:15 +0100 Subject: [PATCH 2/3] Add expectation files --- python/test/files/junit-xml/flaky.annotations | 79 +++++++++++++++++++ python/test/files/junit-xml/flaky.junit-xml | 57 +++++++++++++ python/test/files/junit-xml/flaky.results | 78 ++++++++++++++++++ 3 files changed, 214 insertions(+) create mode 100644 python/test/files/junit-xml/flaky.annotations create mode 100644 python/test/files/junit-xml/flaky.junit-xml create mode 100644 python/test/files/junit-xml/flaky.results diff --git a/python/test/files/junit-xml/flaky.annotations b/python/test/files/junit-xml/flaky.annotations new file mode 100644 index 00000000..4c883a67 --- /dev/null +++ b/python/test/files/junit-xml/flaky.annotations @@ -0,0 +1,79 @@ +[ + { + 'name': 'Test Results', + 'head_sha': 'commit sha', + 'status': 'completed', + 'conclusion': 'failure', + 'output': { + 'title': '1 errors, 1 fail, 2 pass in 0s', + 'summary': + '1 files\u2004\u20031 suites\u2004\u2003\u20020s ' + '[:stopwatch:](https://github.com/EnricoMi/publish-unit-test-result-ac' + 'tion/blob/VERSION/README.md#the-symbols "duration of all tests")\n4 ' + 'tests\u20032 ' + '[:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-re' + 'sult-action/blob/VERSION/README.md#the-symbols "passed tests")\u2003' + '0 ' + '[:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/b' + 'lob/VERSION/README.md#the-symbols "skipped / disabled tests")\u20031 ' + '[:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blo' + 'b/VERSION/README.md#the-symbols "failed tests")\u20031 ' + '[:fire:](https://github.com/EnricoMi/publish-unit-test-result-action/' + 'blob/VERSION/README.md#the-symbols "test errors")\n2 runs\u2006\u2003' + '0 ' + '[:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-re' + 'sult-action/blob/VERSION/README.md#the-symbols "passed tests")\u2003' + '0 ' + '[:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/b' + 'lob/VERSION/README.md#the-symbols "skipped / disabled tests")\u20031 ' + '[:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blo' + 'b/VERSION/README.md#the-symbols "failed tests")\u20031 ' + '[:fire:](https://github.com/EnricoMi/publish-unit-test-result-action/' + 'blob/VERSION/README.md#the-symbols "test errors")\n\nResults for ' + 'commit commit s.\n\n' + '[test-results]:data:application/gzip;base64,H4sIAAAAAAAC/02MQQ6AIAwEv' + '0I4e1Djyc8QghobBUwLJ+PfrYjIbWe3nVMusM8kR9E1QlKEUGCKqAN4x9gy8hCeafiyom' + 'gMF/1fbHDU12rRsGfbW8yIHnOD0VF+f+Knawv/tsSVLHHtMt5aCAw5CVq1vG6wyJ3P3QA' + 'AAA==\n', + 'annotations': [ + { + 'path': 'example.ExampleTests', + 'start_line': 0, + 'end_line': 0, + 'annotation_level': 'failure', + 'message': 'flaky.xml\u2003[took 0s]', + 'title': 'throwsAlways (example.ExampleTests) with error', + 'raw_details': + 'Always throws exception\njava.lang.RuntimeException: Always throws ' + 'exception\n\tat ' + 'example.ExampleTests.throwsAlways(ExampleTests.java:26)' + }, + { + 'path': 'example.ExampleTests', + 'start_line': 0, + 'end_line': 0, + 'annotation_level': 'warning', + 'message': 'flaky.xml\u2003[took 0s]', + 'title': 'failsAssertionAlways (example.ExampleTests) failed', + 'raw_details': + 'Always fails assertion ==> expected: <1> but was: <2>\n' + 'org.opentest4j.AssertionFailedError: Always fails assertion ==> ' + 'expected: <1> but was: <2>\n\tat ' + 'example.ExampleTests.failsAssertionAlways(ExampleTests.java:40)' + }, + { + 'path': '.github', + 'start_line': 0, + 'end_line': 0, + 'annotation_level': 'notice', + 'message': 'There are 4 tests, see "Raw output" for the full list of tests.', + 'title': '4 tests found', + 'raw_details': + 'example.ExampleTests ‑ failsAssertionAlways\nexample.ExampleTests ' + '‑ failsAssertionOnlyFirstAttempt\nexample.ExampleTests ‑ ' + 'throwsAlways\nexample.ExampleTests ‑ throwsOnlyFirstAttempt' + } + ] + } + } +] \ No newline at end of file diff --git a/python/test/files/junit-xml/flaky.junit-xml b/python/test/files/junit-xml/flaky.junit-xml new file mode 100644 index 00000000..e01b48e0 --- /dev/null +++ b/python/test/files/junit-xml/flaky.junit-xml @@ -0,0 +1,57 @@ + + + + java.lang.RuntimeException: Always throws exception + at example.ExampleTests.throwsAlways(ExampleTests.java:26) + + + java.lang.RuntimeException: Always throws exception + at example.ExampleTests.throwsAlways(ExampleTests.java:26) + + + + java.lang.RuntimeException: Always throws exception + at example.ExampleTests.throwsAlways(ExampleTests.java:26) + + + + java.lang.RuntimeException: Always throws exception + at example.ExampleTests.throwsAlways(ExampleTests.java:26) + + + + + org.opentest4j.AssertionFailedError: Always fails assertion ==> expected: <1> but was: <2> + at example.ExampleTests.failsAssertionAlways(ExampleTests.java:40) + + + org.opentest4j.AssertionFailedError: Always fails assertion ==> expected: <1> but was: <2> + at example.ExampleTests.failsAssertionAlways(ExampleTests.java:40) + + + + org.opentest4j.AssertionFailedError: Always fails assertion ==> expected: <1> but was: <2> + at example.ExampleTests.failsAssertionAlways(ExampleTests.java:40) + + + + org.opentest4j.AssertionFailedError: Always fails assertion ==> expected: <1> but was: <2> + at example.ExampleTests.failsAssertionAlways(ExampleTests.java:40) + + + + + + org.opentest4j.AssertionFailedError: First attempt fails assertion ==> expected: <1> but was: <2> + at example.ExampleTests.failsAssertionOnlyFirstAttempt(ExampleTests.java:32) + + + + + + java.lang.RuntimeException: First attempt throws exception + at example.ExampleTests.throwsOnlyFirstAttempt(ExampleTests.java:18) + + + + diff --git a/python/test/files/junit-xml/flaky.results b/python/test/files/junit-xml/flaky.results new file mode 100644 index 00000000..35d60ed1 --- /dev/null +++ b/python/test/files/junit-xml/flaky.results @@ -0,0 +1,78 @@ +publish.unittestresults.ParsedUnitTestResults( + files=1, + errors=[], + suites=1, + suite_tests=2, + suite_skipped=0, + suite_failures=1, + suite_errors=1, + suite_time=0, + suite_details=[ + publish.unittestresults.UnitTestSuite( + name='example.ExampleTests', + tests=2, + skipped=0, + failures=1, + errors=1, + stdout=None, + stderr=None + ) + ], + cases=[ + publish.unittestresults.UnitTestCase( + result_file='flaky.xml', + test_file=None, + line=None, + class_name='example.ExampleTests', + test_name='throwsAlways', + result='error', + message='Always throws exception', + content='java.lang.RuntimeException: Always throws exception\n\tat ' + 'example.ExampleTests.throwsAlways(ExampleTests.java:26)\n', + stdout=None, + stderr=None, + time=0.01 + ), + publish.unittestresults.UnitTestCase( + result_file='flaky.xml', + test_file=None, + line=None, + class_name='example.ExampleTests', + test_name='failsAssertionAlways', + result='failure', + message='Always fails assertion ==> expected: <1> but was: <2>', + content='org.opentest4j.AssertionFailedError: Always fails assertion ==> ' + 'expected: <1> but was: <2>\n\tat ' + 'example.ExampleTests.failsAssertionAlways(ExampleTests.java:40)\n', + stdout=None, + stderr=None, + time=0.003 + ), + publish.unittestresults.UnitTestCase( + result_file='flaky.xml', + test_file=None, + line=None, + class_name='example.ExampleTests', + test_name='failsAssertionOnlyFirstAttempt', + result='success', + message=None, + content=None, + stdout=None, + stderr=None, + time=0.0 + ), + publish.unittestresults.UnitTestCase( + result_file='flaky.xml', + test_file=None, + line=None, + class_name='example.ExampleTests', + test_name='throwsOnlyFirstAttempt', + result='success', + message=None, + content=None, + stdout=None, + stderr=None, + time=0.001 + ) + ] +) \ No newline at end of file From 907a02f2e72679a6866712f19843606f61cad09d Mon Sep 17 00:00:00 2001 From: Enrico Minack Date: Fri, 15 Dec 2023 18:45:50 +0100 Subject: [PATCH 3/3] Fix expectations --- python/test/test_action_script.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/python/test/test_action_script.py b/python/test/test_action_script.py index 8db4839f..04344251 100644 --- a/python/test/test_action_script.py +++ b/python/test/test_action_script.py @@ -1024,14 +1024,14 @@ def test_parse_files(self): (platform.mac_ver()[0].startswith("11.") or platform.mac_ver()[0].startswith("12.")): # on macOS and below Python 3.9 we see one particular error self.assertEqual(17, len(actual.errors)) - self.assertEqual(731, actual.suites) - self.assertEqual(4109, actual.suite_tests) + self.assertEqual(733, actual.suites) + self.assertEqual(4113, actual.suite_tests) self.assertEqual(214, actual.suite_skipped) - self.assertEqual(450, actual.suite_failures) - self.assertEqual(21, actual.suite_errors) + self.assertEqual(452, actual.suite_failures) + self.assertEqual(23, actual.suite_errors) self.assertEqual(7956, actual.suite_time) self.assertEqual(0, len(actual.suite_details)) - self.assertEqual(4085, len(actual.cases)) + self.assertEqual(4093, len(actual.cases)) else: self.assertEqual(13, len(actual.errors)) self.assertEqual(737, actual.suites) @@ -1107,9 +1107,9 @@ def test_parse_files_with_suite_details(self): if Version(sys.version.split(' ')[0]) < Version('3.9.0') and sys.platform.startswith('darwin') and \ (platform.mac_ver()[0].startswith("11.") or platform.mac_ver()[0].startswith("12.")): # on macOS (below macOS 13) and Python below 3.9 we see one particular error - self.assertEqual(363, len(actual.suite_details)) + self.assertEqual(364, len(actual.suite_details)) else: - self.assertEqual(365, len(actual.suite_details)) + self.assertEqual(366, len(actual.suite_details)) def test_parse_files_no_matches(self): gha = mock.MagicMock()