From 59e001bbd2bf2f12853f9ff0483c92cd4eee9db0 Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Wed, 15 Jun 2022 01:29:59 -0700 Subject: [PATCH 1/5] Improve raises() helper --- hypothesis-python/tests/common/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hypothesis-python/tests/common/utils.py b/hypothesis-python/tests/common/utils.py index 6a2fb1deda..ca463ec59a 100644 --- a/hypothesis-python/tests/common/utils.py +++ b/hypothesis-python/tests/common/utils.py @@ -11,6 +11,7 @@ import contextlib import sys from io import StringIO +from types import SimpleNamespace from hypothesis import Phase, settings from hypothesis.errors import HypothesisDeprecationWarning @@ -29,9 +30,11 @@ @contextlib.contextmanager def raises(expected_exception, match=None): + err = SimpleNamespace(value=None) try: - yield + yield err except expected_exception as e: + err.value = e if match is not None: import re From 1639c81cb4c87b07962a372eee96887d682b8a65 Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Wed, 15 Jun 2022 01:29:59 -0700 Subject: [PATCH 2/5] Check notes in assertion helper --- hypothesis-python/tests/common/utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/hypothesis-python/tests/common/utils.py b/hypothesis-python/tests/common/utils.py index ca463ec59a..ce8e1a4737 100644 --- a/hypothesis-python/tests/common/utils.py +++ b/hypothesis-python/tests/common/utils.py @@ -178,11 +178,14 @@ def assert_falsifying_output( if expected_exception is None: # Some tests want to check the output of non-failing runs. test() + msg = "" else: - with raises(expected_exception): + with raises(expected_exception) as exc_info: test() + notes = "\n".join(getattr(exc_info.value, "__notes__", [])) + msg = str(exc_info.value) + "\n" + notes - output = out.getvalue() + output = out.getvalue() + msg assert f"{example_type} example:" in output assert_output_contains_failure(output, test, **kwargs) From c10559a792cd0124a3a04d85e8e27e26d0aea18b Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Wed, 15 Jun 2022 01:29:59 -0700 Subject: [PATCH 3/5] Refactor tests --- hypothesis-python/tests/cover/test_control.py | 14 +++++++------- .../tests/nocover/test_interesting_origin.py | 6 +----- hypothesis-python/tests/nocover/test_targeting.py | 2 +- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/hypothesis-python/tests/cover/test_control.py b/hypothesis-python/tests/cover/test_control.py index e2bb120d08..41c1eb1c55 100644 --- a/hypothesis-python/tests/cover/test_control.py +++ b/hypothesis-python/tests/cover/test_control.py @@ -74,7 +74,7 @@ def test_does_not_suppress_exceptions(): def test_suppresses_exceptions_in_teardown(): - with pytest.raises(ValueError) as err: + with pytest.raises(ValueError) as exc_info: with bc(): def foo(): @@ -83,12 +83,12 @@ def foo(): cleanup(foo) raise AssertionError - assert isinstance(err.value, ValueError) - assert isinstance(err.value.__cause__, AssertionError) + assert isinstance(exc_info.value, ValueError) + assert isinstance(exc_info.value.__cause__, AssertionError) def test_runs_multiple_cleanup_with_teardown(): - with pytest.raises(ExceptionGroup) as err: + with pytest.raises(ExceptionGroup) as exc_info: with bc(): def foo(): @@ -101,9 +101,9 @@ def bar(): cleanup(bar) raise AssertionError - assert isinstance(err.value, ExceptionGroup) - assert isinstance(err.value.__cause__, AssertionError) - assert {type(e) for e in err.value.exceptions} == {ValueError, TypeError} + assert isinstance(exc_info.value, ExceptionGroup) + assert isinstance(exc_info.value.__cause__, AssertionError) + assert {type(e) for e in exc_info.value.exceptions} == {ValueError, TypeError} assert _current_build_context.value is None diff --git a/hypothesis-python/tests/nocover/test_interesting_origin.py b/hypothesis-python/tests/nocover/test_interesting_origin.py index 972f817753..d0d1194055 100644 --- a/hypothesis-python/tests/nocover/test_interesting_origin.py +++ b/hypothesis-python/tests/nocover/test_interesting_origin.py @@ -58,9 +58,5 @@ def test_fn(x, y): # Indirection to fix https://github.com/HypothesisWorks/hypothesis/issues/2888 return function(x, y) - try: + with pytest.raises(MultipleFailures): test_fn() - except MultipleFailures: - pass - else: - raise AssertionError("Expected MultipleFailures") diff --git a/hypothesis-python/tests/nocover/test_targeting.py b/hypothesis-python/tests/nocover/test_targeting.py index 31221a4c32..58fb6be6f8 100644 --- a/hypothesis-python/tests/nocover/test_targeting.py +++ b/hypothesis-python/tests/nocover/test_targeting.py @@ -29,7 +29,7 @@ def test_threshold_problem(x): @pytest.mark.parametrize("multiple", [False, True]) def test_reports_target_results(testdir, multiple): script = testdir.makepyfile(TESTSUITE.format("" if multiple else "# ")) - result = testdir.runpytest(script) + result = testdir.runpytest(script, "--tb=native") out = "\n".join(result.stdout.lines) assert "Falsifying example" in out assert "x=101" in out From b64826432784da4290ee31531617c346d39c3a9d Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Wed, 15 Jun 2022 01:29:59 -0700 Subject: [PATCH 4/5] Improve too_slow healthcheck --- hypothesis-python/RELEASE.rst | 4 ++++ .../src/hypothesis/internal/conjecture/engine.py | 6 +++++- hypothesis-python/tests/cover/test_health_checks.py | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 hypothesis-python/RELEASE.rst diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 0000000000..97c314a830 --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,4 @@ +RELEASE_TYPE: patch + +This patch makes the :obj:`~hypothesis.HealthCheck.too_slow` health check more +consistent with long :obj:`~hypothesis.settings.deadline` tests (:issue:`3367`) diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py index a1fb21041c..b3fad161ee 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py @@ -13,6 +13,7 @@ import time from collections import defaultdict from contextlib import contextmanager +from datetime import timedelta from enum import Enum from random import Random, getrandbits from weakref import WeakKeyDictionary @@ -386,7 +387,10 @@ def record_for_health_check(self, data): draw_time = sum(state.draw_times) - if draw_time > 1.0: + # Allow at least the greater of one second or 5x the deadline. If deadline + # is None, allow 30s - the user can disable the healthcheck too if desired. + draw_time_limit = 5 * (self.settings.deadline or timedelta(seconds=6)) + if draw_time > max(1.0, draw_time_limit.total_seconds()): fail_health_check( self.settings, "Data generation is extremely slow: Only produced " diff --git a/hypothesis-python/tests/cover/test_health_checks.py b/hypothesis-python/tests/cover/test_health_checks.py index 50092c185c..5fd1f0d31b 100644 --- a/hypothesis-python/tests/cover/test_health_checks.py +++ b/hypothesis-python/tests/cover/test_health_checks.py @@ -45,7 +45,7 @@ def test(x): def test_slow_generation_inline_fails_a_health_check(): - @settings(HEALTH_CHECK_SETTINGS, deadline=None) + @HEALTH_CHECK_SETTINGS @given(st.data()) def test(data): data.draw(st.integers().map(lambda x: time.sleep(0.2))) From 9632a903ab09f15d75f69bc3a6c675961c473e04 Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Wed, 15 Jun 2022 01:29:59 -0700 Subject: [PATCH 5/5] Fix pipenv install issue --- hypothesis-python/RELEASE.rst | 2 ++ hypothesis-python/setup.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst index 97c314a830..4a777042d9 100644 --- a/hypothesis-python/RELEASE.rst +++ b/hypothesis-python/RELEASE.rst @@ -2,3 +2,5 @@ RELEASE_TYPE: patch This patch makes the :obj:`~hypothesis.HealthCheck.too_slow` health check more consistent with long :obj:`~hypothesis.settings.deadline` tests (:issue:`3367`) +and fixes an install issue under :pypi:`pipenv` which was introduced in +:ref:`Hypothesis 6.47.2 ` (:issue:`3374`). diff --git a/hypothesis-python/setup.py b/hypothesis-python/setup.py index 40c7146da2..0c15eca835 100644 --- a/hypothesis-python/setup.py +++ b/hypothesis-python/setup.py @@ -102,7 +102,7 @@ def local_file(name): extras_require=extras, install_requires=[ "attrs>=19.2.0", - "exceptiongroup>=1.0.0rc8 ; python_version<'3.11.0b1'", + "exceptiongroup>=1.0.0rc8 ; python_version<'3.11'", "sortedcontainers>=2.1.0,<3.0.0", ], python_requires=">=3.7",