Merge pull request #4029 from tybug/backend-realize

Add and use `realize` and `avoid_realization` backend hooks

hypothesis-python/RELEASE.rst

@@ -0,0 +1,3 @@
+RELEASE_TYPE: patch
+
+This patch iterates on our experimental support for alternative backends (:ref:`alternative-backends`). See :pull:`4029` for details.

hypothesis-python/setup.py

@@ -60,7 +60,7 @@ def local_file(name):
     "pytest": ["pytest>=4.6"],
     "dpcontracts": ["dpcontracts>=0.4"],
     "redis": ["redis>=3.0.0"],
-    "crosshair": ["hypothesis-crosshair>=0.0.6", "crosshair-tool>=0.0.58"],
+    "crosshair": ["hypothesis-crosshair>=0.0.7", "crosshair-tool>=0.0.59"],
     # zoneinfo is an odd one: every dependency is conditional, because they're
     # only necessary on old versions of Python or Windows systems or emscripten.
     "zoneinfo": [

hypothesis-python/src/hypothesis/core.py

@@ -1119,6 +1119,10 @@ def _execute_once_for_engine(self, data: ConjectureData) -> None:
                     self.settings.backend != "hypothesis"
                     and not getattr(runner, "_switch_to_hypothesis_provider", False)
                 )
+                data._observability_args = data.provider.realize(
+                    data._observability_args
+                )
+                self._string_repr = data.provider.realize(self._string_repr)
                 tc = make_testcase(
                     start_timestamp=self._start_timestamp,
                     test_name_or_nodeid=self.test_identifier,

hypothesis-python/src/hypothesis/internal/conjecture/data.py

@@ -1206,19 +1206,35 @@ class PrimitiveProvider(abc.ABC):
     # Non-hypothesis providers probably want to set a lifetime of test_function.
     lifetime = "test_function"
 
+    # Solver-based backends such as hypothesis-crosshair use symbolic values
+    # which record operations performed on them in order to discover new paths.
+    # If avoid_realization is set to True, hypothesis will avoid interacting with
+    # ir values (symbolics) returned by the provider in any way that would force the
+    # solver to narrow the range of possible values for that symbolic.
+    #
+    # Setting this to True disables some hypothesis features, such as
+    # DataTree-based deduplication, and some internal optimizations, such as
+    # caching kwargs. Only enable this if it is necessary for your backend.
+    avoid_realization = False
+
     def __init__(self, conjecturedata: Optional["ConjectureData"], /) -> None:
         self._cd = conjecturedata
 
-    def post_test_case_hook(self, value: IRType) -> IRType:
-        # hook for providers to modify values returned by draw_* after a full
-        # test case concludes. Originally exposed for crosshair to reify its
-        # symbolic values into actual values.
-        # I'm not tied to this exact function name or design.
-        return value
-
     def per_test_case_context_manager(self):
         return contextlib.nullcontext()
 
+    def realize(self, value: T) -> T:
+        """
+        Called whenever hypothesis requires a concrete (non-symbolic) value from
+        a potentially symbolic value. Hypothesis will not check that `value` is
+        symbolic before calling `realize`, so you should handle the case where
+        `value` is non-symbolic.
+
+        The returned value should be non-symbolic.
+        """
+
+        return value
+
     @abc.abstractmethod
     def draw_boolean(
         self,
@@ -1888,6 +1904,14 @@ def permitted(f):
 }
 
 
+# eventually we'll want to expose this publicly, but for now it lives as psuedo-internal.
+def realize(value: object) -> object:
+    from hypothesis.control import current_build_context
+
+    context = current_build_context()
+    return context.data.provider.realize(value)
+
+
 class ConjectureData:
     @classmethod
     def for_buffer(
@@ -2280,6 +2304,10 @@ def draw_boolean(
 
     def _pooled_kwargs(self, ir_type, kwargs):
         """Memoize common dictionary objects to reduce memory pressure."""
+        # caching runs afoul of nondeterminism checks
+        if self.provider.avoid_realization:
+            return kwargs
+
         key = []
         for k, v in kwargs.items():
             if ir_type == "float" and k in ["min_value", "max_value"]:

hypothesis-python/src/hypothesis/internal/conjecture/engine.py

@@ -30,6 +30,7 @@
     Set,
     Tuple,
     Union,
+    cast,
     overload,
 )
 
@@ -56,6 +57,7 @@
     Example,
     HypothesisProvider,
     InterestingOrigin,
+    IRKWargsType,
     IRNode,
     Overrun,
     PrimitiveProvider,
@@ -455,7 +457,7 @@ def test_function(self, data: ConjectureData) -> None:
                 self.stats_per_test_case.append(call_stats)
                 if self.settings.backend != "hypothesis":
                     for node in data.examples.ir_tree_nodes:
-                        value = data.provider.post_test_case_hook(node.value)
+                        value = data.provider.realize(node.value)
                         expected_type = {
                             "string": str,
                             "float": float,
@@ -466,10 +468,19 @@ def test_function(self, data: ConjectureData) -> None:
                         if type(value) is not expected_type:
                             raise HypothesisException(
                                 f"expected {expected_type} from "
-                                f"{data.provider.post_test_case_hook.__qualname__}, "
-                                f"got {type(value)} ({value!r})"
+                                f"{data.provider.realize.__qualname__}, "
+                                f"got {type(value)}"
                             )
+
+                        kwargs = cast(
+                            IRKWargsType,
+                            {
+                                k: data.provider.realize(v)
+                                for k, v in node.kwargs.items()
+                            },
+                        )
                         node.value = value
+                        node.kwargs = kwargs
 
                 self._cache(data)
                 if data.invalid_at is not None:  # pragma: no branch # coverage bug?

hypothesis-python/src/hypothesis/strategies/_internal/utils.py

@@ -65,10 +65,15 @@ def clear_cache() -> None:
 
 
 def cacheable(fn: "T") -> "T":
+    from hypothesis.control import _current_build_context
     from hypothesis.strategies._internal.strategies import SearchStrategy
 
     @proxies(fn)
     def cached_strategy(*args, **kwargs):
+        context = _current_build_context.value
+        if context is not None and context.data.provider.avoid_realization:
+            return fn(*args, **kwargs)
+
         try:
             kwargs_cache_key = {(k, convert_value(v)) for k, v in kwargs.items()}
         except TypeError:

hypothesis-python/tests/conjecture/test_alt_backend.py

@@ -24,6 +24,7 @@
     AVAILABLE_PROVIDERS,
     ConjectureData,
     PrimitiveProvider,
+    realize,
 )
 from hypothesis.internal.conjecture.engine import ConjectureRunner
 from hypothesis.internal.floats import SIGNALING_NAN
@@ -372,21 +373,58 @@ def test_function(n):
             test_function()
 
 
-class BadPostTestCaseHookProvider(TrivialProvider):
-    def post_test_case_hook(self, value):
+class BadRealizeProvider(TrivialProvider):
+    def realize(self, value):
         return None
 
 
-def test_bad_post_test_case_hook():
-    with temp_register_backend("bad_hook", BadPostTestCaseHookProvider):
+def test_bad_realize():
+    with temp_register_backend("bad_realize", BadRealizeProvider):
 
         @given(st.integers())
-        @settings(backend="bad_hook")
+        @settings(backend="bad_realize")
         def test_function(n):
             pass
 
         with pytest.raises(
             HypothesisException,
-            match="expected .* from BadPostTestCaseHookProvider.post_test_case_hook",
+            match="expected .* from BadRealizeProvider.realize",
         ):
             test_function()
+
+
+class RealizeProvider(TrivialProvider):
+    avoid_realization = True
+
+    def realize(self, value):
+        if isinstance(value, int):
+            return 42
+        return value
+
+
+def test_realize():
+    with temp_register_backend("realize", RealizeProvider):
+
+        values = []
+
+        @given(st.integers())
+        @settings(backend="realize")
+        def test_function(n):
+            values.append(realize(n))
+
+        test_function()
+
+        assert all(n == 42 for n in values)
+
+
+def test_realize_dependent_draw():
+    with temp_register_backend("realize", RealizeProvider):
+
+        @given(st.data())
+        @settings(backend="realize")
+        def test_function(data):
+            n1 = data.draw(st.integers())
+            n2 = data.draw(st.integers(n1, n1 + 10))
+            assert n1 <= n2
+
+        test_function()