From ce9a797229b6f6ecba1e920718084895676059e6 Mon Sep 17 00:00:00 2001 From: Zac-HD Date: Wed, 9 Jun 2021 22:25:12 +1000 Subject: [PATCH] Move tests to nocover --- .../tests/cover/test_scrutineer.py | 51 +------------ .../tests/nocover/test_scrutineer.py | 72 +++++++++++++++++++ 2 files changed, 75 insertions(+), 48 deletions(-) create mode 100644 hypothesis-python/tests/nocover/test_scrutineer.py diff --git a/hypothesis-python/tests/cover/test_scrutineer.py b/hypothesis-python/tests/cover/test_scrutineer.py index 3ba9035e9c..c4e2570a32 100644 --- a/hypothesis-python/tests/cover/test_scrutineer.py +++ b/hypothesis-python/tests/cover/test_scrutineer.py @@ -19,60 +19,17 @@ import pytest from hypothesis.internal.compat import PYPY -from hypothesis.internal.scrutineer import HAD_TRACE, make_report +from hypothesis.internal.scrutineer import HAD_TRACE -BUG_MARKER = "# BUG" -PRELUDE = """ +CODE_SAMPLE = """ from hypothesis import Phase, given, settings, strategies as st @settings(phases=tuple(Phase), derandomize=True) -""" - - -def expected_reports(file_contents, fname): - # Takes the source code string with "# BUG" comments, and returns a list of - # multi-line report strings which we expect to see in explain-mode output. - # The list length is the number of explainable bugs, usually one. - explanations = { - i: {(fname, i)} - for i, line in enumerate(file_contents.splitlines()) - if line.endswith(BUG_MARKER) - } - return ["\n".join(r) for k, r in make_report(explanations).items()] - - -TRIVIAL = """ @given(st.integers()) def test_reports_branch_in_test(x): if x > 10: raise AssertionError # BUG """ -MULTIPLE_BUGS = """ -@given(st.integers(), st.integers()) -def test_reports_branch_in_test(x, y): - if x > 10: - raise (AssertionError if x % 2 else Exception) # BUG -""" - - -# We skip tracing for explanations under PyPy, where it has a large performance -# impact, or if there is already a trace function (e.g. coverage or a debugger) -@pytest.mark.skipif(PYPY or sys.gettrace(), reason="See comment") -@pytest.mark.parametrize( - "code", - [ - pytest.param(TRIVIAL, id="trivial"), - pytest.param(MULTIPLE_BUGS, id="multiple-bugs"), - ], -) -def test_explanations(code, testdir): - code = PRELUDE + code - test_file = str(testdir.makepyfile(code)) - pytest_stdout = str(testdir.runpytest_inprocess(test_file, "--tb=native").stdout) - expected = expected_reports(code, fname=test_file) - assert len(expected) == code.count(BUG_MARKER) - for report in expected: - assert report in pytest_stdout @pytest.mark.skipif(PYPY, reason="Tracing is slow under PyPy") @@ -85,9 +42,7 @@ def test_cannot_explain_message(testdir): try: if no_tracer: sys.settrace(lambda frame, event, arg: None) - - code = PRELUDE + TRIVIAL - test_file = testdir.makepyfile(code) + test_file = testdir.makepyfile(CODE_SAMPLE) testdir.runpytest_inprocess(test_file, "--tb=native").stdout.re_match_lines( [r"Explanation:", re.escape(HAD_TRACE)], consecutive=True ) diff --git a/hypothesis-python/tests/nocover/test_scrutineer.py b/hypothesis-python/tests/nocover/test_scrutineer.py new file mode 100644 index 0000000000..ec5f41814e --- /dev/null +++ b/hypothesis-python/tests/nocover/test_scrutineer.py @@ -0,0 +1,72 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Most of this work is copyright (C) 2013-2021 David R. MacIver +# (david@drmaciver.com), but it contains contributions by others. See +# CONTRIBUTING.rst for a full list of people who may hold copyright, and +# consult the git log if you need to determine who owns an individual +# contribution. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. +# +# END HEADER + +import sys + +import pytest + +from hypothesis.internal.compat import PYPY +from hypothesis.internal.scrutineer import make_report + +# We skip tracing for explanations under PyPy, where it has a large performance +# impact, or if there is already a trace function (e.g. coverage or a debugger) +pytestmark = pytest.mark.skipif(PYPY or sys.gettrace(), reason="See comment") + +BUG_MARKER = "# BUG" +PRELUDE = """ +from hypothesis import Phase, given, settings, strategies as st + +@settings(phases=tuple(Phase), derandomize=True) +""" +TRIVIAL = """ +@given(st.integers()) +def test_reports_branch_in_test(x): + if x > 10: + raise AssertionError # BUG +""" +MULTIPLE_BUGS = """ +@given(st.integers(), st.integers()) +def test_reports_branch_in_test(x, y): + if x > 10: + raise (AssertionError if x % 2 else Exception) # BUG +""" +FRAGMENTS = ( + pytest.param(TRIVIAL, id="trivial"), + pytest.param(MULTIPLE_BUGS, id="multiple-bugs"), +) + + +def get_reports(file_contents, *, testdir): + # Takes the source code string with "# BUG" comments, and returns a list of + # multi-line report strings which we expect to see in explain-mode output. + # The list length is the number of explainable bugs, usually one. + test_file = str(testdir.makepyfile(file_contents)) + pytest_stdout = str(testdir.runpytest_inprocess(test_file, "--tb=native").stdout) + + explanations = { + i: {(test_file, i)} + for i, line in enumerate(file_contents.splitlines()) + if line.endswith(BUG_MARKER) + } + expected = ["\n".join(r) for k, r in make_report(explanations).items()] + return pytest_stdout, expected + + +@pytest.mark.parametrize("code", FRAGMENTS) +def test_explanations(code, testdir): + pytest_stdout, expected = get_reports(PRELUDE + code, testdir=testdir) + assert len(expected) == code.count(BUG_MARKER) + for report in expected: + assert report in pytest_stdout