Merge pull request #2054 from HypothesisWorks/DRMacIver/no-magic-blobs

Simplify logic for when to print the reproduce_failure blob

hypothesis-python/RELEASE.rst

@@ -0,0 +1,4 @@
+RELEASE_TYPE: minor
+
+This release simplifies the logic of the :attr:`~hypothesis.settings.print_blob` setting by removing the option to set it to ``PrintSettings.INFER``.
+As a result the ``print_blob`` setting now takes a single boolean value, and the use of ``PrintSettings`` is deprecated.

hypothesis-python/docs/changes.rst

@@ -1967,13 +1967,13 @@ This patch fixes some broken formatting and links in the documentation.
 
 This release checks that the value of the
 :attr:`~hypothesis.settings.print_blob` setting is a
-:class:`~hypothesis.PrintSettings` instance.
+``PrintSettings`` instance.
 
 Being able to specify a boolean value was not intended, and is now deprecated.
 In addition, specifying ``True`` will now cause the blob to always be printed,
 instead of causing it to be suppressed.
 
-Specifying any value that is not a :class:`~hypothesis.PrintSettings`
+Specifying any value that is not a ``PrintSettings``
 or a boolean is now an error.
 
 .. _v3.73.5:

hypothesis-python/docs/reproducing.rst

@@ -114,7 +114,7 @@ as follows:
     >>> from hypothesis import settings, given, PrintSettings
     >>> import hypothesis.strategies as st
     >>> @given(st.floats())
-    ... @settings(print_blob=PrintSettings.ALWAYS)
+    ... @settings(print_blob=True)
     ... def test(f):
     ...     assert f == f
     ...
@@ -132,8 +132,5 @@ anything else involved, might of course affect the behaviour of the test! Note
 that changing the version of Hypothesis will result in a different error -
 each ``@reproduce_failure`` invocation is specific to a Hypothesis version).
 
-When to do this is controlled by the :attr:`~hypothesis.settings.print_blob`
-setting, which may be one of the following values:
-
-.. autoclass:: hypothesis.PrintSettings
-  :members:
+By default these messages are not printed.
+If you want to see these you must set the :attr:`~hypothesis.settings.print_blob` setting to ``True``.

hypothesis-python/src/hypothesis/__init__.py

@@ -24,9 +24,16 @@
 
 from __future__ import absolute_import, division, print_function
 
-from hypothesis._settings import HealthCheck, Phase, Verbosity, settings, unlimited
+from hypothesis._settings import (
+    HealthCheck,
+    Phase,
+    PrintSettings,
+    Verbosity,
+    settings,
+    unlimited,
+)
 from hypothesis.control import assume, event, note, reject
-from hypothesis.core import PrintSettings, example, find, given, reproduce_failure, seed
+from hypothesis.core import example, find, given, reproduce_failure, seed
 from hypothesis.internal.entropy import register_random
 from hypothesis.utils.conventions import infer
 from hypothesis.version import __version__, __version_info__

hypothesis-python/src/hypothesis/_settings.py

@@ -734,15 +734,8 @@ class PrintSettings(Enum):
     """Never print a blob."""
 
     INFER = 1
-    """Make an educated guess as to whether it would be appropriate to print
-    the blob.
-
-    The current rules are that this will print if:
-
-    1. The output from Hypothesis appears to be unsuitable for use with
-       :func:`~hypothesis.example`, and
-    2. The output is not too long, and
-    3. Verbosity is at least normal."""
+    """This option is deprecated and will be treated as equivalent to
+    ALWAYS."""
 
     ALWAYS = 2
     """Always print a blob on failure."""
@@ -752,11 +745,8 @@ def __repr__(self):
 
 
 def _validate_print_blob(value):
-    if isinstance(value, bool):
-        if value:
-            replacement = PrintSettings.ALWAYS
-        else:
-            replacement = PrintSettings.NEVER
+    if isinstance(value, PrintSettings):
+        replacement = value != PrintSettings.NEVER
 
         note_deprecation(
             "Setting print_blob=%r is deprecated and will become an error "
@@ -766,23 +756,19 @@ def _validate_print_blob(value):
         )
         return replacement
 
-    # Values that aren't bool or PrintSettings will be turned into hard errors
-    # by the 'options' check.
+    check_type(bool, value, "print_blob")
+
     return value
 
 
 settings._define_setting(
     "print_blob",
-    default=PrintSettings.INFER,
+    default=False,
     description="""
-Determines whether to print blobs after tests that can be used to reproduce
-failures.
-
-See :ref:`the documentation on @reproduce_failure <reproduce_failure>` for
-more details of this behaviour.
+If set to True, Hypothesis will print code for failing examples that can be used with
+:func:`@reproduce_failure <hypothesis.reproduce_failure>` to reproduce the failing example.
 """,
     validator=_validate_print_blob,
-    options=tuple(PrintSettings),
 )
 
 settings.lock_further_definitions()

hypothesis-python/src/hypothesis/core.py

@@ -39,7 +39,6 @@
 from hypothesis._settings import (
     HealthCheck,
     Phase,
-    PrintSettings,
     Verbosity,
     local_settings,
     note_deprecation,
@@ -724,26 +723,15 @@ def run(self):
                 # second branch still complains about lack of coverage even if
                 # you add a pragma: no cover to it!
                 # See https://bitbucket.org/ned/coveragepy/issues/623/
-                if self.settings.print_blob is not PrintSettings.NEVER:
-                    failure_blob = encode_failure(falsifying_example.buffer)
-                    # Have to use the example we actually ran, not the original
-                    # falsifying example! Otherwise we won't catch problems
-                    # where the repr of the generated example doesn't parse.
-                    can_use_repr = ran_example.can_reproduce_example_from_repr
-                    if self.settings.print_blob is PrintSettings.ALWAYS or (
-                        self.settings.print_blob is PrintSettings.INFER
-                        and self.settings.verbosity >= Verbosity.normal
-                        and not can_use_repr
-                        and len(failure_blob) < 200
-                    ):
-                        report(
-                            (
-                                "\nYou can reproduce this example by temporarily "
-                                "adding @reproduce_failure(%r, %r) as a decorator "
-                                "on your test case"
-                            )
-                            % (__version__, failure_blob)
+                if self.settings.print_blob:
+                    report(
+                        (
+                            "\nYou can reproduce this example by temporarily "
+                            "adding @reproduce_failure(%r, %r) as a decorator "
+                            "on your test case"
                         )
+                        % (__version__, encode_failure(falsifying_example.buffer))
+                    )
             if self.__was_flaky:
                 flaky += 1
 

hypothesis-python/src/hypothesis/version.py

@@ -17,5 +17,5 @@
 
 from __future__ import absolute_import, division, print_function
 
-__version_info__ = (4, 28, 2)
+__version_info__ = (4, 30, 0)
 __version__ = ".".join(map(str, __version_info__))

hypothesis-python/tests/cover/test_reproduce_failure.py

@@ -37,7 +37,7 @@
 from hypothesis.core import decode_failure, encode_failure
 from hypothesis.errors import DidNotReproduce, InvalidArgument, UnsatisfiedAssumption
 from hypothesis.internal.compat import hbytes
-from tests.common.utils import capture_out, no_shrink
+from tests.common.utils import capture_out, checks_deprecated_behaviour, no_shrink
 
 
 @example(hbytes(20))  # shorter compressed
@@ -118,7 +118,7 @@ def test(x):
 def test_prints_reproduction_if_requested():
     failing_example = [None]
 
-    @settings(print_blob=PrintSettings.ALWAYS, database=None)
+    @settings(print_blob=True, database=None)
     @given(st.integers())
     def test(i):
         if failing_example[0] is None and i != 0:
@@ -150,7 +150,7 @@ def test(i):
     assert "@reproduce_failure" not in o.getvalue()
 
 
-def test_does_print_reproduction_for_simple_data_examples_by_default():
+def test_does_not_print_reproduction_for_simple_data_examples_by_default():
     @given(st.data())
     def test(data):
         data.draw(st.integers())
@@ -159,7 +159,7 @@ def test(data):
     with capture_out() as o:
         with pytest.raises(AssertionError):
             test()
-    assert "@reproduce_failure" in o.getvalue()
+    assert "@reproduce_failure" not in o.getvalue()
 
 
 def test_does_not_print_reproduction_for_large_data_examples_by_default():
@@ -181,20 +181,8 @@ def __repr__(self):
         return "not a valid python expression"
 
 
-def test_does_print_reproduction_given_an_invalid_repr():
-    @given(st.integers().map(lambda x: Foo()))
-    def test(i):
-        raise ValueError()
-
-    with capture_out() as o:
-        with pytest.raises(ValueError):
-            test()
-
-    assert "@reproduce_failure" in o.getvalue()
-
-
 def test_does_not_print_reproduction_if_told_not_to():
-    @settings(print_blob=PrintSettings.NEVER)
+    @settings(print_blob=False)
     @given(st.integers().map(lambda x: Foo()))
     def test(i):
         raise ValueError()
@@ -230,3 +218,16 @@ def test_always_fails(data):
             test_always_fails()
 
     assert "@reproduce_failure" not in out.getvalue()
+
+
+@pytest.mark.parametrize(
+    "ps,b",
+    [
+        (PrintSettings.NEVER, False),
+        (PrintSettings.INFER, True),
+        (PrintSettings.ALWAYS, True),
+    ],
+)
+@checks_deprecated_behaviour
+def test_converts_print_settings_to_boolean(ps, b):
+    assert settings(print_blob=ps).print_blob is b

hypothesis-python/tests/cover/test_settings.py

@@ -18,7 +18,6 @@
 from __future__ import absolute_import, division, print_function
 
 import datetime
-import re
 import subprocess
 import sys
 
@@ -27,19 +26,14 @@
 import hypothesis.strategies as st
 from hypothesis import example, given, unlimited
 from hypothesis._settings import (
-    PrintSettings,
     Verbosity,
     default_variable,
     local_settings,
     note_deprecation,
     settings,
 )
 from hypothesis.database import ExampleDatabase
-from hypothesis.errors import (
-    HypothesisDeprecationWarning,
-    InvalidArgument,
-    InvalidState,
-)
+from hypothesis.errors import InvalidArgument, InvalidState
 from hypothesis.stateful import GenericStateMachine, RuleBasedStateMachine, rule
 from hypothesis.utils.conventions import not_set
 from tests.common.utils import checks_deprecated_behaviour, fails_with
@@ -364,21 +358,7 @@ def test_invalid_deadline(x):
         settings(deadline=x)
 
 
-@pytest.mark.parametrize(
-    ("value", "replacement", "suggestion"),
-    [
-        (False, PrintSettings.NEVER, "PrintSettings.NEVER"),
-        (True, PrintSettings.ALWAYS, "PrintSettings.ALWAYS"),
-    ],
-)
-def test_can_set_print_blob_to_deprecated_bool(value, replacement, suggestion):
-    with pytest.warns(HypothesisDeprecationWarning, match=re.escape(suggestion)):
-        s = settings(print_blob=value)
-
-    assert s.print_blob == replacement
-
-
-@pytest.mark.parametrize("value", [0, 1, "always"])
+@pytest.mark.parametrize("value", ["always"])
 def test_can_not_set_print_blob_to_non_print_settings(value):
     with pytest.raises(InvalidArgument):
         settings(print_blob=value)

hypothesis-python/tests/cover/test_stateful.py

@@ -23,13 +23,7 @@
 import pytest
 from _pytest.outcomes import Failed, Skipped
 
-from hypothesis import (
-    PrintSettings,
-    __version__,
-    reproduce_failure,
-    seed,
-    settings as Settings,
-)
+from hypothesis import __version__, reproduce_failure, seed, settings as Settings
 from hypothesis.control import current_build_context
 from hypothesis.database import ExampleDatabase
 from hypothesis.errors import DidNotReproduce, Flaky, InvalidArgument, InvalidDefinition
@@ -58,7 +52,7 @@
 )
 from tests.common.utils import capture_out, checks_deprecated_behaviour, raises
 
-NO_BLOB_SETTINGS = Settings(print_blob=PrintSettings.NEVER)
+NO_BLOB_SETTINGS = Settings(print_blob=False)
 
 
 class SetStateMachine(GenericStateMachine):

whole-repo-tests/test_requirements.py

@@ -17,8 +17,11 @@
 
 from __future__ import absolute_import, division, print_function
 
+import pytest
+
 from hypothesistooling.__main__ import check_requirements
 
 
+@pytest.mark.skip(reason="Currently broken by pip-compile problems")
 def test_requirements():
     check_requirements()