Merge pull request #2104 from Zac-HD/warn-on-example

Warn on non-interactive `s.example()`

hypothesis-python/RELEASE.rst

@@ -0,0 +1,8 @@
+RELEASE_TYPE: minor
+
+This release emits a warning if you use the ``.example()`` method of
+a strategy in a non-interactive context.
+
+:func:`~hypothesis.given` is a much better choice for writing tests,
+whether you care about performance, minimal examples, reproducing
+failures, or even just the variety of inputs that will be tested!

hypothesis-python/src/hypothesis/errors.py

@@ -112,6 +112,12 @@ def __init__(self, message, check):
         self.health_check = check
 
 
+class NonInteractiveExampleWarning(HypothesisWarning):
+    """SearchStrategy.example() is designed for interactive use,
+    but should never be used in the body of a test.
+    """
+
+
 class HypothesisDeprecationWarning(HypothesisWarning, FutureWarning):
     """A deprecation warning issued by Hypothesis.
 

hypothesis-python/src/hypothesis/searchstrategy/strategies.py

@@ -17,6 +17,8 @@
 
 from __future__ import absolute_import, division, print_function
 
+import sys
+import warnings
 from collections import defaultdict
 from random import choice as random_choice
 
@@ -30,7 +32,11 @@
     settings,
 )
 from hypothesis.control import _current_build_context, assume
-from hypothesis.errors import HypothesisException, UnsatisfiedAssumption
+from hypothesis.errors import (
+    HypothesisException,
+    NonInteractiveExampleWarning,
+    UnsatisfiedAssumption,
+)
 from hypothesis.internal.compat import bit_length, hrange
 from hypothesis.internal.conjecture.utils import (
     LABEL_MASK,
@@ -268,6 +274,17 @@ def example(self, random=not_set):
         if random is not not_set:
             note_deprecation("The random argument does nothing", since="2019-07-08")
 
+        if getattr(sys, "ps1", None) is None:  # pragma: no branch
+            # The other branch *is* covered in cover/test_examples.py; but as that
+            # uses `pexpect` for an interactive session `coverage` doesn't see it.
+            warnings.warn(
+                "The `.example()` method is good for exploring strategies, but should "
+                "only be used interactively.  We recommend using `@given` for tests - "
+                "it performs better, saves and replays failures to avoid flakiness, "
+                "and reports minimal examples. (strategy: %r)" % (self,),
+                NonInteractiveExampleWarning,
+            )
+
         context = _current_build_context.value
         if context is not None:
             if context.data is not None and context.data.depth > 0:

hypothesis-python/tests/common/setup.py

@@ -24,6 +24,7 @@
 from hypothesis import Verbosity, settings
 from hypothesis._settings import not_set
 from hypothesis.configuration import set_hypothesis_home_dir
+from hypothesis.errors import NonInteractiveExampleWarning
 from hypothesis.internal.charmap import charmap, charmap_file
 from hypothesis.internal.coverage import IN_COVERAGE_TESTS
 
@@ -58,6 +59,9 @@ def run():
         "ignore", message="Importing from numpy.testing.nosetester is deprecated"
     )
 
+    # User-facing warning which does not apply to our own tests
+    filterwarnings("ignore", category=NonInteractiveExampleWarning)
+
     new_home = mkdtemp()
     set_hypothesis_home_dir(new_home)
     assert settings.default.database.path.startswith(new_home)

hypothesis-python/tests/cover/test_example.py

@@ -17,14 +17,22 @@
 
 from __future__ import absolute_import, division, print_function
 
+import sys
+import warnings
 from decimal import Decimal
 from random import Random
 
+import pexpect
 import pytest
 
 import hypothesis.strategies as st
 from hypothesis import example, find, given
-from hypothesis.errors import HypothesisException, Unsatisfiable
+from hypothesis.errors import (
+    HypothesisException,
+    NonInteractiveExampleWarning,
+    Unsatisfiable,
+)
+from hypothesis.internal.compat import WINDOWS
 from tests.common.utils import checks_deprecated_behaviour, fails_with
 
 
@@ -66,3 +74,21 @@ def test_example_inside_find():
 @fails_with(HypothesisException)
 def test_example_inside_strategy():
     st.booleans().map(lambda x: st.integers().example()).example()
+
+
+def test_non_interactive_example_emits_warning():
+    # We have this warning disabled for most of our tests, because self-testing
+    # Hypothesis means `.example()` can be a good idea when it never is for users.
+    with warnings.catch_warnings():
+        warnings.simplefilter("always")
+        with pytest.warns(NonInteractiveExampleWarning):
+            st.text().example()
+
+
+@pytest.mark.skipif(WINDOWS, reason="pexpect.spawn not supported on Windows")
+def test_interactive_example_does_not_emit_warning():
+    child = pexpect.spawn("%s -Werror" % (sys.executable,))
+    child.expect(">>> ", timeout=1)
+    child.sendline("from hypothesis.strategies import none")
+    child.sendline("none().example()")
+    child.sendline("quit(code=0)")

requirements/py2.txt

@@ -12,7 +12,9 @@ mock==3.0.5
 more-itertools==5.0.0
 numpy==1.16.4             # last compatible with Python 2
 packaging==19.0           # via pytest
+pexpect==4.7.0
 pluggy==0.12.0            # via pytest
+ptyprocess==0.6.0         # via pexpect
 py==1.8.0                 # via pytest
 pyparsing==2.4.0          # via packaging
 pytest-forked==1.0.2      # via pytest-xdist

requirements/test.in

@@ -1,3 +1,4 @@
 attrs
+pexpect
 pytest
 pytest-xdist

requirements/test.txt

@@ -11,7 +11,9 @@ execnet==1.7.1            # via pytest-xdist
 importlib-metadata==0.23  # via pluggy, pytest
 more-itertools==7.2.0     # via pytest, zipp
 packaging==19.2           # via pytest
+pexpect==4.7.0
 pluggy==0.13.0            # via pytest
+ptyprocess==0.6.0         # via pexpect
 py==1.8.0                 # via pytest
 pyparsing==2.4.2          # via packaging
 pytest-forked==1.0.2      # via pytest-xdist