Merge pull request #3806 from Zac-HD/provider-plugins

WIP: support for `crosshair` backend

hypothesis-python/RELEASE.rst

@@ -0,0 +1,4 @@
+RELEASE_TYPE: minor
+
+This release adds the **experimental and unstable** :obj:`~hypothesis.settings.backend`
+setting.  See :ref:`alternative-backends` for details.

hypothesis-python/docs/strategies.rst

@@ -204,3 +204,36 @@ loading our pytest plugin from your ``conftest.py`` instead::
 
     echo "pytest_plugins = ['hypothesis.extra.pytestplugin']\n" > tests/conftest.py
     pytest -p "no:hypothesispytest" ...
+
+
+.. _alternative-backends:
+
+-----------------------------------
+Alternative backends for Hypothesis
+-----------------------------------
+
+.. warning::
+
+   EXPERIMENTAL AND UNSTABLE.
+
+The importable name of a backend which Hypothesis should use to generate primitive
+types.  We aim to support heuristic-random, solver-based, and fuzzing-based backends.
+
+See :issue:`3086` for details, e.g. if you're interested in writing your own backend.
+(note that there is *no stable interface* for this; you'd be helping us work out
+what that should eventually look like, and we're likely to make regular breaking
+changes for some time to come)
+
+Using the prototype :pypi:`crosshair-tool` backend `via this plugin
+<https://github.com/pschanely/hypothesis-crosshair>`__,
+a solver-backed test might look something like:
+
+.. code-block:: python
+
+    from hypothesis import given, settings, strategies as st
+
+
+    @settings(backend="crosshair")
+    @given(st.integers())
+    def test_needs_solver(x):
+        assert x != 123456789

hypothesis-python/setup.py

@@ -60,6 +60,7 @@ def local_file(name):
     "pytest": ["pytest>=4.6"],
     "dpcontracts": ["dpcontracts>=0.4"],
     "redis": ["redis>=3.0.0"],
+    "crosshair": ["hypothesis-crosshair>=0.0.1", "crosshair-tool>=0.0.50"],
     # zoneinfo is an odd one: every dependency is conditional, because they're
     # only necessary on old versions of Python or Windows systems or emscripten.
     "zoneinfo": [

hypothesis-python/src/hypothesis/_settings.py

@@ -165,6 +165,7 @@ def __init__(
         suppress_health_check: Collection["HealthCheck"] = not_set,  # type: ignore
         deadline: Union[int, float, datetime.timedelta, None] = not_set,  # type: ignore
         print_blob: bool = not_set,  # type: ignore
+        backend: str = not_set,  # type: ignore
     ) -> None:
         if parent is not None:
             check_type(settings, parent, "parent")
@@ -289,7 +290,13 @@ def __setattr__(self, name, value):
         raise AttributeError("settings objects are immutable")
 
     def __repr__(self):
-        bits = sorted(f"{name}={getattr(self, name)!r}" for name in all_settings)
+        from hypothesis.internal.conjecture.data import AVAILABLE_PROVIDERS
+
+        bits = sorted(
+            f"{name}={getattr(self, name)!r}"
+            for name in all_settings
+            if (name != "backend" or len(AVAILABLE_PROVIDERS) > 1)  # experimental
+        )
         return "settings({})".format(", ".join(bits))
 
     def show_changed(self):
@@ -706,6 +713,33 @@ def is_in_ci() -> bool:
 """,
 )
 
+
+def _backend_validator(value):
+    from hypothesis.internal.conjecture.data import AVAILABLE_PROVIDERS
+
+    if value not in AVAILABLE_PROVIDERS:
+        if value == "crosshair":  # pragma: no cover
+            install = '`pip install "hypothesis[crosshair]"` and try again.'
+            raise InvalidArgument(f"backend={value!r} is not available.  {install}")
+        raise InvalidArgument(
+            f"backend={value!r} is not available - maybe you need to install a plugin?"
+            f"\n    Installed backends: {sorted(AVAILABLE_PROVIDERS)!r}"
+        )
+    return value
+
+
+settings._define_setting(
+    "backend",
+    default="hypothesis",
+    show_default=False,
+    validator=_backend_validator,
+    description="""
+EXPERIMENTAL AND UNSTABLE - see :ref:`alternative-backends`.
+The importable name of a backend which Hypothesis should use to generate primitive
+types.  We aim to support heuristic-random, solver-based, and fuzzing-based backends.
+""",
+)
+
 settings.lock_further_definitions()
 
 

hypothesis-python/src/hypothesis/core.py

@@ -938,9 +938,13 @@ def run(data):
         with local_settings(self.settings):
             with deterministic_PRNG():
                 with BuildContext(data, is_final=is_final) as context:
-                    # Run the test function once, via the executor hook.
-                    # In most cases this will delegate straight to `run(data)`.
-                    result = self.test_runner(data, run)
+                    # providers may throw in per_case_context_fn, and we'd like
+                    # `result` to still be set in these cases.
+                    result = None
+                    with data.provider.per_test_case_context_manager():
+                        # Run the test function once, via the executor hook.
+                        # In most cases this will delegate straight to `run(data)`.
+                        result = self.test_runner(data, run)
 
         # If a failure was expected, it should have been raised already, so
         # instead raise an appropriate diagnostic error.