Merge pull request #3376 from Zac-HD/tune-healthcheck

Fix pipenv install issue and sometimes-too-short draw time healthcheck

hypothesis-python/RELEASE.rst

@@ -0,0 +1,6 @@
+RELEASE_TYPE: patch
+
+This patch makes the :obj:`~hypothesis.HealthCheck.too_slow` health check more
+consistent with long :obj:`~hypothesis.settings.deadline` tests (:issue:`3367`)
+and fixes an install issue under :pypi:`pipenv` which was introduced in
+:ref:`Hypothesis 6.47.2 <v6.47.2>` (:issue:`3374`).

hypothesis-python/setup.py

@@ -102,7 +102,7 @@ def local_file(name):
     extras_require=extras,
     install_requires=[
         "attrs>=19.2.0",
-        "exceptiongroup>=1.0.0rc8 ; python_version<'3.11.0b1'",
+        "exceptiongroup>=1.0.0rc8 ; python_version<'3.11'",
         "sortedcontainers>=2.1.0,<3.0.0",
     ],
     python_requires=">=3.7",

hypothesis-python/src/hypothesis/internal/conjecture/engine.py

@@ -13,6 +13,7 @@
 import time
 from collections import defaultdict
 from contextlib import contextmanager
+from datetime import timedelta
 from enum import Enum
 from random import Random, getrandbits
 from weakref import WeakKeyDictionary
@@ -386,7 +387,10 @@ def record_for_health_check(self, data):
 
         draw_time = sum(state.draw_times)
 
-        if draw_time > 1.0:
+        # Allow at least the greater of one second or 5x the deadline.  If deadline
+        # is None, allow 30s - the user can disable the healthcheck too if desired.
+        draw_time_limit = 5 * (self.settings.deadline or timedelta(seconds=6))
+        if draw_time > max(1.0, draw_time_limit.total_seconds()):
             fail_health_check(
                 self.settings,
                 "Data generation is extremely slow: Only produced "

hypothesis-python/tests/common/utils.py

@@ -11,6 +11,7 @@
 import contextlib
 import sys
 from io import StringIO
+from types import SimpleNamespace
 
 from hypothesis import Phase, settings
 from hypothesis.errors import HypothesisDeprecationWarning
@@ -29,9 +30,11 @@
 
     @contextlib.contextmanager
     def raises(expected_exception, match=None):
+        err = SimpleNamespace(value=None)
         try:
-            yield
+            yield err
         except expected_exception as e:
+            err.value = e
             if match is not None:
                 import re
 
@@ -175,11 +178,14 @@ def assert_falsifying_output(
         if expected_exception is None:
             # Some tests want to check the output of non-failing runs.
             test()
+            msg = ""
         else:
-            with raises(expected_exception):
+            with raises(expected_exception) as exc_info:
                 test()
+            notes = "\n".join(getattr(exc_info.value, "__notes__", []))
+            msg = str(exc_info.value) + "\n" + notes
 
-    output = out.getvalue()
+    output = out.getvalue() + msg
     assert f"{example_type} example:" in output
     assert_output_contains_failure(output, test, **kwargs)
 

hypothesis-python/tests/cover/test_control.py

@@ -74,7 +74,7 @@ def test_does_not_suppress_exceptions():
 
 
 def test_suppresses_exceptions_in_teardown():
-    with pytest.raises(ValueError) as err:
+    with pytest.raises(ValueError) as exc_info:
         with bc():
 
             def foo():
@@ -83,12 +83,12 @@ def foo():
             cleanup(foo)
             raise AssertionError
 
-    assert isinstance(err.value, ValueError)
-    assert isinstance(err.value.__cause__, AssertionError)
+    assert isinstance(exc_info.value, ValueError)
+    assert isinstance(exc_info.value.__cause__, AssertionError)
 
 
 def test_runs_multiple_cleanup_with_teardown():
-    with pytest.raises(ExceptionGroup) as err:
+    with pytest.raises(ExceptionGroup) as exc_info:
         with bc():
 
             def foo():
@@ -101,9 +101,9 @@ def bar():
             cleanup(bar)
             raise AssertionError
 
-    assert isinstance(err.value, ExceptionGroup)
-    assert isinstance(err.value.__cause__, AssertionError)
-    assert {type(e) for e in err.value.exceptions} == {ValueError, TypeError}
+    assert isinstance(exc_info.value, ExceptionGroup)
+    assert isinstance(exc_info.value.__cause__, AssertionError)
+    assert {type(e) for e in exc_info.value.exceptions} == {ValueError, TypeError}
     assert _current_build_context.value is None
 
 

hypothesis-python/tests/cover/test_health_checks.py

@@ -45,7 +45,7 @@ def test(x):
 
 
 def test_slow_generation_inline_fails_a_health_check():
-    @settings(HEALTH_CHECK_SETTINGS, deadline=None)
+    @HEALTH_CHECK_SETTINGS
     @given(st.data())
     def test(data):
         data.draw(st.integers().map(lambda x: time.sleep(0.2)))

hypothesis-python/tests/nocover/test_interesting_origin.py

@@ -58,9 +58,5 @@ def test_fn(x, y):
         # Indirection to fix https://github.com/HypothesisWorks/hypothesis/issues/2888
         return function(x, y)
 
-    try:
+    with pytest.raises(MultipleFailures):
         test_fn()
-    except MultipleFailures:
-        pass
-    else:
-        raise AssertionError("Expected MultipleFailures")

hypothesis-python/tests/nocover/test_targeting.py

@@ -29,7 +29,7 @@ def test_threshold_problem(x):
 @pytest.mark.parametrize("multiple", [False, True])
 def test_reports_target_results(testdir, multiple):
     script = testdir.makepyfile(TESTSUITE.format("" if multiple else "# "))
-    result = testdir.runpytest(script)
+    result = testdir.runpytest(script, "--tb=native")
     out = "\n".join(result.stdout.lines)
     assert "Falsifying example" in out
     assert "x=101" in out