From 6e3638a3f0c1591694f6f2cb6e5df7d48467bd7a Mon Sep 17 00:00:00 2001 From: Patrick Lannigan Date: Wed, 29 Nov 2023 08:00:03 -0500 Subject: [PATCH 01/11] Output verbosity for test cases --- src/_pytest/config/__init__.py | 2 + src/_pytest/terminal.py | 26 +++++-- testing/test_terminal.py | 120 +++++++++++++++++++++++++++++++++ 3 files changed, 141 insertions(+), 7 deletions(-) diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py index ea23c77421a..01c705f34e5 100644 --- a/src/_pytest/config/__init__.py +++ b/src/_pytest/config/__init__.py @@ -1653,6 +1653,8 @@ def getvalueorskip(self, name: str, path=None): #: Verbosity type for failed assertions (see :confval:`verbosity_assertions`). VERBOSITY_ASSERTIONS: Final = "assertions" + #: Verbosity type for test case execution (see :confval:`verbosity_test_cases`). + VERBOSITY_TEST_CASES: Final = "test_cases" _VERBOSITY_INI_DEFAULT: Final = "auto" def get_verbosity(self, verbosity_type: Optional[str] = None) -> int: diff --git a/src/_pytest/terminal.py b/src/_pytest/terminal.py index ea26d9368dc..b690b6be4df 100644 --- a/src/_pytest/terminal.py +++ b/src/_pytest/terminal.py @@ -253,6 +253,14 @@ def pytest_addoption(parser: Parser) -> None: "progress even when capture=no)", default="progress", ) + Config._add_verbosity_ini( + parser, + Config.VERBOSITY_TEST_CASES, + help=( + "Specify a verbosity level for test case execution, overriding the main level. " + "Higher levels will provide more detailed information about each test case executed." + ), + ) def pytest_configure(config: Config) -> None: @@ -415,7 +423,7 @@ def showfspath(self, value: Optional[bool]) -> None: @property def showlongtestinfo(self) -> bool: - return self.verbosity > 0 + return self.config.get_verbosity(Config.VERBOSITY_TEST_CASES) > 0 def hasopt(self, char: str) -> bool: char = {"xfailed": "x", "skipped": "s"}.get(char, char) @@ -593,7 +601,7 @@ def pytest_runtest_logreport(self, report: TestReport) -> None: markup = {"yellow": True} else: markup = {} - if self.verbosity <= 0: + if self.config.get_verbosity(Config.VERBOSITY_TEST_CASES) <= 0: self._tw.write(letter, **markup) else: self._progress_nodeids_reported.add(rep.nodeid) @@ -602,7 +610,7 @@ def pytest_runtest_logreport(self, report: TestReport) -> None: self.write_ensure_prefix(line, word, **markup) if rep.skipped or hasattr(report, "wasxfail"): reason = _get_raw_skip_reason(rep) - if self.config.option.verbose < 2: + if self.config.get_verbosity(Config.VERBOSITY_TEST_CASES) < 2: available_width = ( (self._tw.fullwidth - self._tw.width_of_current_line) - len(" [100%]") @@ -639,7 +647,10 @@ def _is_last_item(self) -> bool: def pytest_runtest_logfinish(self, nodeid: str) -> None: assert self._session - if self.verbosity <= 0 and self._show_progress_info: + if ( + self.config.get_verbosity(Config.VERBOSITY_TEST_CASES) <= 0 + and self._show_progress_info + ): if self._show_progress_info == "count": num_tests = self._session.testscollected progress_length = len(f" [{num_tests}/{num_tests}]") @@ -819,8 +830,9 @@ def pytest_collection_finish(self, session: "Session") -> None: rep.toterminal(self._tw) def _printcollecteditems(self, items: Sequence[Item]) -> None: - if self.config.option.verbose < 0: - if self.config.option.verbose < -1: + test_cases_verbosity = self.config.get_verbosity(Config.VERBOSITY_TEST_CASES) + if test_cases_verbosity < 0: + if test_cases_verbosity < -1: counts = Counter(item.nodeid.split("::", 1)[0] for item in items) for name, count in sorted(counts.items()): self._tw.line("%s: %d" % (name, count)) @@ -840,7 +852,7 @@ def _printcollecteditems(self, items: Sequence[Item]) -> None: stack.append(col) indent = (len(stack) - 1) * " " self._tw.line(f"{indent}{col}") - if self.config.option.verbose >= 1: + if test_cases_verbosity >= 1: obj = getattr(col, "obj", None) doc = inspect.getdoc(obj) if obj else None if doc: diff --git a/testing/test_terminal.py b/testing/test_terminal.py index 264ab96d8d0..5d64de3418d 100644 --- a/testing/test_terminal.py +++ b/testing/test_terminal.py @@ -2614,3 +2614,123 @@ def test_format_trimmed() -> None: assert _format_trimmed(" ({}) ", msg, len(msg) + 4) == " (unconditional skip) " assert _format_trimmed(" ({}) ", msg, len(msg) + 3) == " (unconditional ...) " + + +def test_fine_grained_test_case_verbosity(pytester: Pytester): + p = pytester.makepyfile(_fine_grained_verbosity_file_contents()) + pytester.makeini( + """ + [pytest] + verbosity_test_cases = 2 + """ + ) + result = pytester.runpytest(p) + + result.stdout.fnmatch_lines( + [ + f"{p.name}::test_ok PASSED [ 14%]", + f"{p.name}::test_words_fail FAILED [ 28%]", + f"{p.name}::test_numbers_fail FAILED [ 42%]", + f"{p.name}::test_long_text_fail FAILED [ 57%]", + f"{p.name}::test_parametrize_fail[hello-1] FAILED [ 71%]", + f"{p.name}::test_parametrize_fail[world-987654321] FAILED [ 85%]", + f"{p.name}::test_sample_skip SKIPPED (some", + "long skip reason that will not fit on a single line with other content", + "that goes on and on and on and on and on) [100%]", + ], + consecutive=True, + ) + + +def test_fine_grained_test_case_verbosity_collect_only_negative_2(pytester: Pytester): + p = pytester.makepyfile(_fine_grained_verbosity_file_contents()) + pytester.makeini( + """ + [pytest] + verbosity_test_cases = -2 + """ + ) + result = pytester.runpytest("--collect-only", p) + + result.stdout.fnmatch_lines( + [ + "collected 7 items", + "", + f"{p.name}: 7", + ], + consecutive=True, + ) + + +def test_fine_grained_test_case_verbosity_collect_only_positive_2(pytester: Pytester): + p = pytester.makepyfile(_fine_grained_verbosity_file_contents()) + pytester.makeini( + """ + [pytest] + verbosity_test_cases = 2 + """ + ) + result = pytester.runpytest("--collect-only", p) + + result.stdout.fnmatch_lines( + [ + "collected 7 items", + "", + f"", + " ", + " some docstring", + " ", + " ", + " ", + " ", + " ", + " ", + ], + consecutive=True, + ) + + +def _fine_grained_verbosity_file_contents() -> str: + long_text = "Lorem ipsum dolor sit amet " * 10 + return f""" + import pytest + def test_ok(): + ''' + some docstring + ''' + pass + + + def test_words_fail(): + fruits1 = ["banana", "apple", "grapes", "melon", "kiwi"] + fruits2 = ["banana", "apple", "orange", "melon", "kiwi"] + assert fruits1 == fruits2 + + + def test_numbers_fail(): + number_to_text1 = {{str(x): x for x in range(5)}} + number_to_text2 = {{str(x * 10): x * 10 for x in range(5)}} + assert number_to_text1 == number_to_text2 + + + def test_long_text_fail(): + long_text = "{long_text}" + assert "hello world" in long_text + + + @pytest.mark.parametrize(["foo", "bar"], [ + ("hello", 1), + ("world", 987654321), + ]) + def test_parametrize_fail(foo, bar): + long_text = f"{{foo}} {{bar}}" + assert "hello world" in long_text + + + @pytest.mark.skip( + "some long skip reason that will not fit on a single line with other content that goes" + " on and on and on and on and on" + ) + def test_sample_skip(): + pass + """ From 1181384493aa0dc8875bd8ef5aa53158afded699 Mon Sep 17 00:00:00 2001 From: Patrick Lannigan Date: Sat, 25 Nov 2023 15:22:57 -0500 Subject: [PATCH 02/11] Add documentaiton & changelog entry --- changelog/11639.feature.rst | 5 +++++ doc/en/how-to/output.rst | 4 +++- doc/en/reference/reference.rst | 13 +++++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 changelog/11639.feature.rst diff --git a/changelog/11639.feature.rst b/changelog/11639.feature.rst new file mode 100644 index 00000000000..2c83f798bda --- /dev/null +++ b/changelog/11639.feature.rst @@ -0,0 +1,5 @@ +Added the new :confval:`verbosity_test_case` configuration option for fine-grained control of failed assertions verbosity. + +See :ref:`Fine-grained verbosity ` for more details. + +For plugin authors, :attr:`config.get_verbosity ` can be used to retrieve the verbosity level for a specific verbosity type. diff --git a/doc/en/how-to/output.rst b/doc/en/how-to/output.rst index 8af9a38b768..d24e98f9be1 100644 --- a/doc/en/how-to/output.rst +++ b/doc/en/how-to/output.rst @@ -298,7 +298,9 @@ This is done by setting a verbosity level in the configuration file for the spec ``pytest --no-header`` with a value of ``2`` would have the same output as the previous example, but each test inside the file is shown by a single character in the output. -(Note: currently this is the only option available, but more might be added in the future). +:confval:`verbosity_test_case`: Controls how verbose the test execution output should be when pytest is executed. +Running ``pytest --no-header`` with a value of ``2`` would have the same output as the first verbosity example, but each +test inside the file gets its own line in the output. .. _`pytest.detailed_failed_tests_usage`: diff --git a/doc/en/reference/reference.rst b/doc/en/reference/reference.rst index 254973709af..4d5e1b3815c 100644 --- a/doc/en/reference/reference.rst +++ b/doc/en/reference/reference.rst @@ -1835,6 +1835,19 @@ passed multiple times. The expected format is ``name=value``. For example:: "auto" can be used to explicitly use the global verbosity level. +.. confval:: verbosity_test_case + + Set a verbosity level specifically for test case execution related output, overriding the application wide level. + + .. code-block:: ini + + [pytest] + verbosity_test_case = 2 + + Defaults to application wide verbosity level (via the ``-v`` command-line option). A special value of + "auto" can be used to explicitly use the global verbosity level. + + .. confval:: xfail_strict If set to ``True``, tests marked with ``@pytest.mark.xfail`` that actually succeed will by default fail the From 018b290faa73eba66abfa93b84c95530724abd5f Mon Sep 17 00:00:00 2001 From: Patrick Lannigan Date: Sat, 2 Dec 2023 16:57:30 -0500 Subject: [PATCH 03/11] Consolidate changelog and correct typo --- changelog/11387.feature.rst | 1 + changelog/11639.feature.rst | 5 ----- doc/en/how-to/output.rst | 2 +- doc/en/reference/reference.rst | 4 ++-- 4 files changed, 4 insertions(+), 8 deletions(-) delete mode 100644 changelog/11639.feature.rst diff --git a/changelog/11387.feature.rst b/changelog/11387.feature.rst index 90f20885b0a..6fe1256f9ea 100644 --- a/changelog/11387.feature.rst +++ b/changelog/11387.feature.rst @@ -1,4 +1,5 @@ Added the new :confval:`verbosity_assertions` configuration option for fine-grained control of failed assertions verbosity. +Added the new :confval:`verbosity_test_cases` configuration option for fine-grained control of test execution verbosity. See :ref:`Fine-grained verbosity ` for more details. diff --git a/changelog/11639.feature.rst b/changelog/11639.feature.rst deleted file mode 100644 index 2c83f798bda..00000000000 --- a/changelog/11639.feature.rst +++ /dev/null @@ -1,5 +0,0 @@ -Added the new :confval:`verbosity_test_case` configuration option for fine-grained control of failed assertions verbosity. - -See :ref:`Fine-grained verbosity ` for more details. - -For plugin authors, :attr:`config.get_verbosity ` can be used to retrieve the verbosity level for a specific verbosity type. diff --git a/doc/en/how-to/output.rst b/doc/en/how-to/output.rst index d24e98f9be1..c5061ed288e 100644 --- a/doc/en/how-to/output.rst +++ b/doc/en/how-to/output.rst @@ -298,7 +298,7 @@ This is done by setting a verbosity level in the configuration file for the spec ``pytest --no-header`` with a value of ``2`` would have the same output as the previous example, but each test inside the file is shown by a single character in the output. -:confval:`verbosity_test_case`: Controls how verbose the test execution output should be when pytest is executed. +:confval:`verbosity_test_cases`: Controls how verbose the test execution output should be when pytest is executed. Running ``pytest --no-header`` with a value of ``2`` would have the same output as the first verbosity example, but each test inside the file gets its own line in the output. diff --git a/doc/en/reference/reference.rst b/doc/en/reference/reference.rst index 4d5e1b3815c..bcbf9901895 100644 --- a/doc/en/reference/reference.rst +++ b/doc/en/reference/reference.rst @@ -1835,14 +1835,14 @@ passed multiple times. The expected format is ``name=value``. For example:: "auto" can be used to explicitly use the global verbosity level. -.. confval:: verbosity_test_case +.. confval:: verbosity_test_cases Set a verbosity level specifically for test case execution related output, overriding the application wide level. .. code-block:: ini [pytest] - verbosity_test_case = 2 + verbosity_test_cases = 2 Defaults to application wide verbosity level (via the ``-v`` command-line option). A special value of "auto" can be used to explicitly use the global verbosity level. From dc52beadc428f18cf267dc67163c51fdc015ec51 Mon Sep 17 00:00:00 2001 From: Patrick Lannigan Date: Sat, 2 Dec 2023 17:39:45 -0500 Subject: [PATCH 04/11] Add more type annotations to test function --- testing/test_terminal.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/test_terminal.py b/testing/test_terminal.py index 5d64de3418d..03c7d410121 100644 --- a/testing/test_terminal.py +++ b/testing/test_terminal.py @@ -2616,7 +2616,7 @@ def test_format_trimmed() -> None: assert _format_trimmed(" ({}) ", msg, len(msg) + 3) == " (unconditional ...) " -def test_fine_grained_test_case_verbosity(pytester: Pytester): +def test_fine_grained_test_case_verbosity(pytester: Pytester) -> None: p = pytester.makepyfile(_fine_grained_verbosity_file_contents()) pytester.makeini( """ @@ -2642,7 +2642,7 @@ def test_fine_grained_test_case_verbosity(pytester: Pytester): ) -def test_fine_grained_test_case_verbosity_collect_only_negative_2(pytester: Pytester): +def test_fine_grained_test_case_verbosity_collect_only_negative_2(pytester: Pytester) -> None: p = pytester.makepyfile(_fine_grained_verbosity_file_contents()) pytester.makeini( """ @@ -2662,7 +2662,7 @@ def test_fine_grained_test_case_verbosity_collect_only_negative_2(pytester: Pyte ) -def test_fine_grained_test_case_verbosity_collect_only_positive_2(pytester: Pytester): +def test_fine_grained_test_case_verbosity_collect_only_positive_2(pytester: Pytester) -> None: p = pytester.makepyfile(_fine_grained_verbosity_file_contents()) pytester.makeini( """ From 61cc290f3e556c95ebf5a6dec5798ff2f2cf842f Mon Sep 17 00:00:00 2001 From: Patrick Lannigan Date: Sat, 2 Dec 2023 19:52:53 -0500 Subject: [PATCH 05/11] Organize tests into a class --- testing/test_terminal.py | 194 +++++++++++++++++++-------------------- 1 file changed, 93 insertions(+), 101 deletions(-) diff --git a/testing/test_terminal.py b/testing/test_terminal.py index 03c7d410121..25234b19ed3 100644 --- a/testing/test_terminal.py +++ b/testing/test_terminal.py @@ -2616,121 +2616,113 @@ def test_format_trimmed() -> None: assert _format_trimmed(" ({}) ", msg, len(msg) + 3) == " (unconditional ...) " -def test_fine_grained_test_case_verbosity(pytester: Pytester) -> None: - p = pytester.makepyfile(_fine_grained_verbosity_file_contents()) - pytester.makeini( - """ - [pytest] - verbosity_test_cases = 2 - """ - ) - result = pytester.runpytest(p) - - result.stdout.fnmatch_lines( - [ - f"{p.name}::test_ok PASSED [ 14%]", - f"{p.name}::test_words_fail FAILED [ 28%]", - f"{p.name}::test_numbers_fail FAILED [ 42%]", - f"{p.name}::test_long_text_fail FAILED [ 57%]", - f"{p.name}::test_parametrize_fail[hello-1] FAILED [ 71%]", - f"{p.name}::test_parametrize_fail[world-987654321] FAILED [ 85%]", - f"{p.name}::test_sample_skip SKIPPED (some", - "long skip reason that will not fit on a single line with other content", - "that goes on and on and on and on and on) [100%]", - ], - consecutive=True, - ) +class TestFineGrainedTestCase: + def test_max_verbosity(self, pytester: Pytester) -> None: + p = TestFineGrainedTestCase._initialize_files(pytester, verbosity=2) + result = pytester.runpytest(p) + result.stdout.fnmatch_lines( + [ + f"{p.name}::test_ok PASSED [ 14%]", + f"{p.name}::test_words_fail FAILED [ 28%]", + f"{p.name}::test_numbers_fail FAILED [ 42%]", + f"{p.name}::test_long_text_fail FAILED [ 57%]", + f"{p.name}::test_parametrize_fail[hello-1] FAILED [ 71%]", + f"{p.name}::test_parametrize_fail[world-987654321] FAILED [ 85%]", + f"{p.name}::test_sample_skip SKIPPED (some long skip reason", + "that will not fit on a single line with other content that goes on and", + "on and on and on and on) [100%]", + ], + consecutive=True, + ) -def test_fine_grained_test_case_verbosity_collect_only_negative_2(pytester: Pytester) -> None: - p = pytester.makepyfile(_fine_grained_verbosity_file_contents()) - pytester.makeini( - """ - [pytest] - verbosity_test_cases = -2 - """ - ) - result = pytester.runpytest("--collect-only", p) - - result.stdout.fnmatch_lines( - [ - "collected 7 items", - "", - f"{p.name}: 7", - ], - consecutive=True, - ) + def test_collect_only_negative_2(self, pytester: Pytester) -> None: + p = TestFineGrainedTestCase._initialize_files(pytester, verbosity=-2) + result = pytester.runpytest("--collect-only", p) + result.stdout.fnmatch_lines( + [ + "collected 7 items", + "", + f"{p.name}: 7", + ], + consecutive=True, + ) -def test_fine_grained_test_case_verbosity_collect_only_positive_2(pytester: Pytester) -> None: - p = pytester.makepyfile(_fine_grained_verbosity_file_contents()) - pytester.makeini( - """ - [pytest] - verbosity_test_cases = 2 - """ - ) - result = pytester.runpytest("--collect-only", p) + def test__collect_only_positive_2(self, pytester: Pytester) -> None: + p = TestFineGrainedTestCase._initialize_files(pytester, verbosity=2) + result = pytester.runpytest("--collect-only", p) - result.stdout.fnmatch_lines( - [ - "collected 7 items", - "", - f"", - " ", - " some docstring", - " ", - " ", - " ", - " ", - " ", - " ", - ], - consecutive=True, - ) + result.stdout.fnmatch_lines( + [ + "collected 7 items", + "", + f"", + " ", + " some docstring", + " ", + " ", + " ", + " ", + " ", + " ", + ], + consecutive=True, + ) + @staticmethod + def _initialize_files(pytester: Pytester, verbosity: int) -> Path: + p = pytester.makepyfile(TestFineGrainedTestCase.file_contents()) + pytester.makeini( + f""" + [pytest] + verbosity_test_cases = {verbosity} + """ + ) + return p -def _fine_grained_verbosity_file_contents() -> str: - long_text = "Lorem ipsum dolor sit amet " * 10 - return f""" - import pytest - def test_ok(): - ''' - some docstring - ''' - pass + @staticmethod + def file_contents() -> str: + long_text = "Lorem ipsum dolor sit amet " * 10 + return f""" + import pytest + def test_ok(): + ''' + some docstring + ''' + pass - def test_words_fail(): - fruits1 = ["banana", "apple", "grapes", "melon", "kiwi"] - fruits2 = ["banana", "apple", "orange", "melon", "kiwi"] - assert fruits1 == fruits2 + def test_words_fail(): + fruits1 = ["banana", "apple", "grapes", "melon", "kiwi"] + fruits2 = ["banana", "apple", "orange", "melon", "kiwi"] + assert fruits1 == fruits2 - def test_numbers_fail(): - number_to_text1 = {{str(x): x for x in range(5)}} - number_to_text2 = {{str(x * 10): x * 10 for x in range(5)}} - assert number_to_text1 == number_to_text2 + def test_numbers_fail(): + number_to_text1 = {{str(x): x for x in range(5)}} + number_to_text2 = {{str(x * 10): x * 10 for x in range(5)}} + assert number_to_text1 == number_to_text2 - def test_long_text_fail(): - long_text = "{long_text}" - assert "hello world" in long_text + def test_long_text_fail(): + long_text = "{long_text}" + assert "hello world" in long_text - @pytest.mark.parametrize(["foo", "bar"], [ - ("hello", 1), - ("world", 987654321), - ]) - def test_parametrize_fail(foo, bar): - long_text = f"{{foo}} {{bar}}" - assert "hello world" in long_text + @pytest.mark.parametrize(["foo", "bar"], [ + ("hello", 1), + ("world", 987654321), + ]) + def test_parametrize_fail(foo, bar): + long_text = f"{{foo}} {{bar}}" + assert "hello world" in long_text - @pytest.mark.skip( - "some long skip reason that will not fit on a single line with other content that goes" - " on and on and on and on and on" - ) - def test_sample_skip(): - pass - """ + @pytest.mark.skip( + "some long skip reason that will not fit on a single line with other content that goes" + " on and on and on and on and on" + ) + def test_sample_skip(): + pass + """ From 629f165045b1843c4051898d5b627e56d05fe5d5 Mon Sep 17 00:00:00 2001 From: Patrick Lannigan Date: Sun, 3 Dec 2023 07:33:23 -0500 Subject: [PATCH 06/11] Restructure tests to use a less complicated sample file --- testing/test_terminal.py | 256 ++++++++++++++++++++++++++++----------- 1 file changed, 184 insertions(+), 72 deletions(-) diff --git a/testing/test_terminal.py b/testing/test_terminal.py index 25234b19ed3..11c1bd49ea0 100644 --- a/testing/test_terminal.py +++ b/testing/test_terminal.py @@ -2617,112 +2617,224 @@ def test_format_trimmed() -> None: class TestFineGrainedTestCase: - def test_max_verbosity(self, pytester: Pytester) -> None: - p = TestFineGrainedTestCase._initialize_files(pytester, verbosity=2) + DEFAULT_FILE_CONTENTS = """ + import pytest + + @pytest.mark.parametrize("i", range(4)) + def test_ok(i): + ''' + some docstring + ''' + pass + """ + LONG_SKIP_FILE_CONTENTS = """ + import pytest + + @pytest.mark.skip( + "some long skip reason that will not fit on a single line with other content that goes" + " on and on and on and on and on" + ) + def test_skip(): + pass + """ + + @pytest.mark.parametrize("verbosity", [1, 2]) + def test_execute_positive(self, verbosity, pytester: Pytester) -> None: + # expected: one test case per line (with file name), word describing result + p = TestFineGrainedTestCase._initialize_files(pytester, verbosity=verbosity) result = pytester.runpytest(p) result.stdout.fnmatch_lines( [ - f"{p.name}::test_ok PASSED [ 14%]", - f"{p.name}::test_words_fail FAILED [ 28%]", - f"{p.name}::test_numbers_fail FAILED [ 42%]", - f"{p.name}::test_long_text_fail FAILED [ 57%]", - f"{p.name}::test_parametrize_fail[hello-1] FAILED [ 71%]", - f"{p.name}::test_parametrize_fail[world-987654321] FAILED [ 85%]", - f"{p.name}::test_sample_skip SKIPPED (some long skip reason", - "that will not fit on a single line with other content that goes on and", - "on and on and on and on) [100%]", + "collected 4 items", + "", + f"{p.name}::test_ok[0] PASSED [ 25%]", + f"{p.name}::test_ok[1] PASSED [ 50%]", + f"{p.name}::test_ok[2] PASSED [ 75%]", + f"{p.name}::test_ok[3] PASSED [100%]", ], consecutive=True, ) - def test_collect_only_negative_2(self, pytester: Pytester) -> None: - p = TestFineGrainedTestCase._initialize_files(pytester, verbosity=-2) - result = pytester.runpytest("--collect-only", p) + def test_execute_0_global_1(self, pytester: Pytester) -> None: + # expected: one file name per line, single character describing result + p = TestFineGrainedTestCase._initialize_files(pytester, verbosity=0) + result = pytester.runpytest("-v", p) result.stdout.fnmatch_lines( [ - "collected 7 items", + "collecting ... collected 4 items", "", - f"{p.name}: 7", + f"{p.name} .... [100%]", ], consecutive=True, ) - def test__collect_only_positive_2(self, pytester: Pytester) -> None: - p = TestFineGrainedTestCase._initialize_files(pytester, verbosity=2) - result = pytester.runpytest("--collect-only", p) + @pytest.mark.parametrize("verbosity", [-1, -2]) + def test_execute_negative(self, verbosity, pytester: Pytester) -> None: + # expected: single character describing result + p = TestFineGrainedTestCase._initialize_files(pytester, verbosity=verbosity) + result = pytester.runpytest(p) result.stdout.fnmatch_lines( [ - "collected 7 items", + "collected 4 items", + ".... [100%]", + ], + consecutive=True, + ) + + def test_execute_skipped_positive_2(self, pytester: Pytester) -> None: + # expected: one test case per line (with file name), word describing result, full reason + p = TestFineGrainedTestCase._initialize_files( + pytester, + verbosity=2, + file_contents=TestFineGrainedTestCase.LONG_SKIP_FILE_CONTENTS, + ) + result = pytester.runpytest(p) + + result.stdout.fnmatch_lines( + [ + "collected 1 item", "", - f"", - " ", - " some docstring", - " ", - " ", - " ", - " ", - " ", - " ", + f"{p.name}::test_skip SKIPPED (some long skip", + "reason that will not fit on a single line with other content that goes", + "on and on and on and on and on) [100%]", ], consecutive=True, ) - @staticmethod - def _initialize_files(pytester: Pytester, verbosity: int) -> Path: - p = pytester.makepyfile(TestFineGrainedTestCase.file_contents()) - pytester.makeini( - f""" - [pytest] - verbosity_test_cases = {verbosity} - """ + def test_execute_skipped_positive_1(self, pytester: Pytester) -> None: + # expected: one test case per line (with file name), word describing result, reason truncated + p = TestFineGrainedTestCase._initialize_files( + pytester, + verbosity=1, + file_contents=TestFineGrainedTestCase.LONG_SKIP_FILE_CONTENTS, ) - return p + result = pytester.runpytest(p) - @staticmethod - def file_contents() -> str: - long_text = "Lorem ipsum dolor sit amet " * 10 - return f""" - import pytest - def test_ok(): - ''' - some docstring - ''' - pass + result.stdout.fnmatch_lines( + [ + "collected 1 item", + "", + f"{p.name}::test_skip SKIPPED (some long ski...) [100%]", + ], + consecutive=True, + ) + + def test_execute_skipped__0_global_1(self, pytester: Pytester) -> None: + # expected: one file name per line, single character describing result (no reason) + p = TestFineGrainedTestCase._initialize_files( + pytester, + verbosity=0, + file_contents=TestFineGrainedTestCase.LONG_SKIP_FILE_CONTENTS, + ) + result = pytester.runpytest("-v", p) + + result.stdout.fnmatch_lines( + [ + "collecting ... collected 1 item", + "", + f"{p.name} s [100%]", + ], + consecutive=True, + ) + @pytest.mark.parametrize("verbosity", [-1, -2]) + def test_execute_skipped_negative(self, verbosity, pytester: Pytester) -> None: + # expected: single character describing result (no reason) + p = TestFineGrainedTestCase._initialize_files( + pytester, + verbosity=verbosity, + file_contents=TestFineGrainedTestCase.LONG_SKIP_FILE_CONTENTS, + ) + result = pytester.runpytest(p) - def test_words_fail(): - fruits1 = ["banana", "apple", "grapes", "melon", "kiwi"] - fruits2 = ["banana", "apple", "orange", "melon", "kiwi"] - assert fruits1 == fruits2 + result.stdout.fnmatch_lines( + [ + "collected 1 item", + "s [100%]", + ], + consecutive=True, + ) + @pytest.mark.parametrize("verbosity", [1, 2]) + def test__collect_only_positive(self, verbosity, pytester: Pytester) -> None: + p = TestFineGrainedTestCase._initialize_files(pytester, verbosity=verbosity) + result = pytester.runpytest("--collect-only", p) - def test_numbers_fail(): - number_to_text1 = {{str(x): x for x in range(5)}} - number_to_text2 = {{str(x * 10): x * 10 for x in range(5)}} - assert number_to_text1 == number_to_text2 + result.stdout.fnmatch_lines( + [ + "collected 4 items", + "", + f"", + " ", + " some docstring", + " ", + " some docstring", + " ", + " some docstring", + " ", + " some docstring", + ], + consecutive=True, + ) + def test_collect_only_0_global_1(self, pytester: Pytester) -> None: + p = TestFineGrainedTestCase._initialize_files(pytester, verbosity=0) + result = pytester.runpytest("-v", "--collect-only", p) - def test_long_text_fail(): - long_text = "{long_text}" - assert "hello world" in long_text + result.stdout.fnmatch_lines( + [ + "collecting ... collected 4 items", + "", + f"", + " ", + " ", + " ", + " ", + ], + consecutive=True, + ) + def test_collect_only_negative_1(self, pytester: Pytester) -> None: + p = TestFineGrainedTestCase._initialize_files(pytester, verbosity=-1) + result = pytester.runpytest("--collect-only", p) - @pytest.mark.parametrize(["foo", "bar"], [ - ("hello", 1), - ("world", 987654321), - ]) - def test_parametrize_fail(foo, bar): - long_text = f"{{foo}} {{bar}}" - assert "hello world" in long_text + result.stdout.fnmatch_lines( + [ + "collected 4 items", + "", + f"{p.name}::test_ok[0]", + f"{p.name}::test_ok[1]", + f"{p.name}::test_ok[2]", + f"{p.name}::test_ok[3]", + ], + consecutive=True, + ) + def test_collect_only_negative_2(self, pytester: Pytester) -> None: + p = TestFineGrainedTestCase._initialize_files(pytester, verbosity=-2) + result = pytester.runpytest("--collect-only", p) - @pytest.mark.skip( - "some long skip reason that will not fit on a single line with other content that goes" - " on and on and on and on and on" - ) - def test_sample_skip(): - pass + result.stdout.fnmatch_lines( + [ + "collected 4 items", + "", + f"{p.name}: 4", + ], + consecutive=True, + ) + + @staticmethod + def _initialize_files( + pytester: Pytester, verbosity: int, file_contents: str = DEFAULT_FILE_CONTENTS + ) -> Path: + p = pytester.makepyfile(file_contents) + pytester.makeini( + f""" + [pytest] + verbosity_test_cases = {verbosity} """ + ) + return p From 44bab5e0bc767a537e8799302f326dd3134ee1f6 Mon Sep 17 00:00:00 2001 From: Patrick Lannigan Date: Sun, 3 Dec 2023 08:12:19 -0500 Subject: [PATCH 07/11] Showing file path also needs to respect verbosity --- src/_pytest/terminal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_pytest/terminal.py b/src/_pytest/terminal.py index b690b6be4df..9416952ba1d 100644 --- a/src/_pytest/terminal.py +++ b/src/_pytest/terminal.py @@ -414,7 +414,7 @@ def no_summary(self) -> bool: @property def showfspath(self) -> bool: if self._showfspath is None: - return self.verbosity >= 0 + return self.config.get_verbosity(Config.VERBOSITY_TEST_CASES) >= 0 return self._showfspath @showfspath.setter From 7e17b5972d1b1f5b8d48136c70962939f2fea9ef Mon Sep 17 00:00:00 2001 From: Patrick Lannigan Date: Sun, 3 Dec 2023 08:17:54 -0500 Subject: [PATCH 08/11] Add a single failing test to sample --- testing/test_terminal.py | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/testing/test_terminal.py b/testing/test_terminal.py index 11c1bd49ea0..30e28585c39 100644 --- a/testing/test_terminal.py +++ b/testing/test_terminal.py @@ -2626,6 +2626,9 @@ def test_ok(i): some docstring ''' pass + + def test_fail(): + assert False """ LONG_SKIP_FILE_CONTENTS = """ import pytest @@ -2646,12 +2649,13 @@ def test_execute_positive(self, verbosity, pytester: Pytester) -> None: result.stdout.fnmatch_lines( [ - "collected 4 items", + "collected 5 items", "", - f"{p.name}::test_ok[0] PASSED [ 25%]", - f"{p.name}::test_ok[1] PASSED [ 50%]", - f"{p.name}::test_ok[2] PASSED [ 75%]", - f"{p.name}::test_ok[3] PASSED [100%]", + f"{p.name}::test_ok[0] PASSED [ 20%]", + f"{p.name}::test_ok[1] PASSED [ 40%]", + f"{p.name}::test_ok[2] PASSED [ 60%]", + f"{p.name}::test_ok[3] PASSED [ 80%]", + f"{p.name}::test_fail FAILED [100%]", ], consecutive=True, ) @@ -2663,9 +2667,9 @@ def test_execute_0_global_1(self, pytester: Pytester) -> None: result.stdout.fnmatch_lines( [ - "collecting ... collected 4 items", + "collecting ... collected 5 items", "", - f"{p.name} .... [100%]", + f"{p.name} ....F [100%]", ], consecutive=True, ) @@ -2678,8 +2682,8 @@ def test_execute_negative(self, verbosity, pytester: Pytester) -> None: result.stdout.fnmatch_lines( [ - "collected 4 items", - ".... [100%]", + "collected 5 items", + "....F [100%]", ], consecutive=True, ) @@ -2765,7 +2769,7 @@ def test__collect_only_positive(self, verbosity, pytester: Pytester) -> None: result.stdout.fnmatch_lines( [ - "collected 4 items", + "collected 5 items", "", f"", " ", @@ -2776,6 +2780,7 @@ def test__collect_only_positive(self, verbosity, pytester: Pytester) -> None: " some docstring", " ", " some docstring", + " ", ], consecutive=True, ) @@ -2786,13 +2791,14 @@ def test_collect_only_0_global_1(self, pytester: Pytester) -> None: result.stdout.fnmatch_lines( [ - "collecting ... collected 4 items", + "collecting ... collected 5 items", "", f"", " ", " ", " ", " ", + " ", ], consecutive=True, ) @@ -2803,12 +2809,13 @@ def test_collect_only_negative_1(self, pytester: Pytester) -> None: result.stdout.fnmatch_lines( [ - "collected 4 items", + "collected 5 items", "", f"{p.name}::test_ok[0]", f"{p.name}::test_ok[1]", f"{p.name}::test_ok[2]", f"{p.name}::test_ok[3]", + f"{p.name}::test_fail", ], consecutive=True, ) @@ -2819,9 +2826,9 @@ def test_collect_only_negative_2(self, pytester: Pytester) -> None: result.stdout.fnmatch_lines( [ - "collected 4 items", + "collected 5 items", "", - f"{p.name}: 4", + f"{p.name}: 5", ], consecutive=True, ) From 09f091d69865689c07aa98ef9052c9f10d87607a Mon Sep 17 00:00:00 2001 From: Patrick Lannigan Date: Sat, 13 Jan 2024 11:36:04 -0500 Subject: [PATCH 09/11] Correct link name --- doc/en/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/en/changelog.rst b/doc/en/changelog.rst index d59f8d97423..641e4723d38 100644 --- a/doc/en/changelog.rst +++ b/doc/en/changelog.rst @@ -267,7 +267,7 @@ Separate Control For Assertion Verbosity ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - `#11387 `_: Added the new :confval:`verbosity_assertions` configuration option for fine-grained control of failed assertions verbosity. -- `#11387 `_: Added the new :confval:`verbosity_test_cases` configuration option for fine-grained control of test execution verbosity. +- `#11653 `_: Added the new :confval:`verbosity_test_cases` configuration option for fine-grained control of test execution verbosity. If you've ever wished that pytest always show you full diffs, but without making everything else verbose, this is for you. From 2d253baedcf0389be8ff7c47b39626db39269cff Mon Sep 17 00:00:00 2001 From: Patrick Lannigan Date: Sat, 24 Feb 2024 10:59:16 -0500 Subject: [PATCH 10/11] Move changelog back to individual entry --- changelog/11653.feature.rst | 2 ++ doc/en/changelog.rst | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog/11653.feature.rst diff --git a/changelog/11653.feature.rst b/changelog/11653.feature.rst new file mode 100644 index 00000000000..9657ac04679 --- /dev/null +++ b/changelog/11653.feature.rst @@ -0,0 +1,2 @@ +Added the new :confval:`verbosity_test_cases` configuration option for fine-grained control of test execution verbosity. +See :ref:`Fine-grained verbosity ` for more details. \ No newline at end of file diff --git a/doc/en/changelog.rst b/doc/en/changelog.rst index f5c2dc5aa80..88f67635565 100644 --- a/doc/en/changelog.rst +++ b/doc/en/changelog.rst @@ -340,7 +340,6 @@ Separate Control For Assertion Verbosity ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - `#11387 `_: Added the new :confval:`verbosity_assertions` configuration option for fine-grained control of failed assertions verbosity. -- `#11653 `_: Added the new :confval:`verbosity_test_cases` configuration option for fine-grained control of test execution verbosity. If you've ever wished that pytest always show you full diffs, but without making everything else verbose, this is for you. From 8c37cb89e1e9b2cb7679ed11c0c9097d14ab8dc2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 24 Feb 2024 16:04:29 +0000 Subject: [PATCH 11/11] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- changelog/11653.feature.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/11653.feature.rst b/changelog/11653.feature.rst index 9657ac04679..f165c3f8e20 100644 --- a/changelog/11653.feature.rst +++ b/changelog/11653.feature.rst @@ -1,2 +1,2 @@ Added the new :confval:`verbosity_test_cases` configuration option for fine-grained control of test execution verbosity. -See :ref:`Fine-grained verbosity ` for more details. \ No newline at end of file +See :ref:`Fine-grained verbosity ` for more details.