diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 0000000000..8f22e9b5d1 --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,4 @@ +RELEASE_TYPE: minor + +This release adds the **experimental and unstable** :obj:`~hypothesis.settings.backend` +setting. See :ref:`alternative-backends` for details. diff --git a/hypothesis-python/docs/strategies.rst b/hypothesis-python/docs/strategies.rst index e9006d37ca..8670b08625 100644 --- a/hypothesis-python/docs/strategies.rst +++ b/hypothesis-python/docs/strategies.rst @@ -204,3 +204,36 @@ loading our pytest plugin from your ``conftest.py`` instead:: echo "pytest_plugins = ['hypothesis.extra.pytestplugin']\n" > tests/conftest.py pytest -p "no:hypothesispytest" ... + + +.. _alternative-backends: + +----------------------------------- +Alternative backends for Hypothesis +----------------------------------- + +.. warning:: + + EXPERIMENTAL AND UNSTABLE. + +The importable name of a backend which Hypothesis should use to generate primitive +types. We aim to support heuristic-random, solver-based, and fuzzing-based backends. + +See :issue:`3086` for details, e.g. if you're interested in writing your own backend. +(note that there is *no stable interface* for this; you'd be helping us work out +what that should eventually look like, and we're likely to make regular breaking +changes for some time to come) + +Using the prototype :pypi:`crosshair-tool` backend `via this plugin +`__, +a solver-backed test might look something like: + +.. code-block:: python + + from hypothesis import given, settings, strategies as st + + + @settings(backend="crosshair") + @given(st.integers()) + def test_needs_solver(x): + assert x != 123456789 diff --git a/hypothesis-python/setup.py b/hypothesis-python/setup.py index bb132c5f4d..494a305321 100644 --- a/hypothesis-python/setup.py +++ b/hypothesis-python/setup.py @@ -60,6 +60,7 @@ def local_file(name): "pytest": ["pytest>=4.6"], "dpcontracts": ["dpcontracts>=0.4"], "redis": ["redis>=3.0.0"], + "crosshair": ["hypothesis-crosshair>=0.0.1", "crosshair-tool>=0.0.50"], # zoneinfo is an odd one: every dependency is conditional, because they're # only necessary on old versions of Python or Windows systems or emscripten. "zoneinfo": [ diff --git a/hypothesis-python/src/hypothesis/_settings.py b/hypothesis-python/src/hypothesis/_settings.py index b11ffc54a4..061292e9bd 100644 --- a/hypothesis-python/src/hypothesis/_settings.py +++ b/hypothesis-python/src/hypothesis/_settings.py @@ -165,6 +165,7 @@ def __init__( suppress_health_check: Collection["HealthCheck"] = not_set, # type: ignore deadline: Union[int, float, datetime.timedelta, None] = not_set, # type: ignore print_blob: bool = not_set, # type: ignore + backend: str = not_set, # type: ignore ) -> None: if parent is not None: check_type(settings, parent, "parent") @@ -289,7 +290,13 @@ def __setattr__(self, name, value): raise AttributeError("settings objects are immutable") def __repr__(self): - bits = sorted(f"{name}={getattr(self, name)!r}" for name in all_settings) + from hypothesis.internal.conjecture.data import AVAILABLE_PROVIDERS + + bits = sorted( + f"{name}={getattr(self, name)!r}" + for name in all_settings + if (name != "backend" or len(AVAILABLE_PROVIDERS) > 1) # experimental + ) return "settings({})".format(", ".join(bits)) def show_changed(self): @@ -706,6 +713,33 @@ def is_in_ci() -> bool: """, ) + +def _backend_validator(value): + from hypothesis.internal.conjecture.data import AVAILABLE_PROVIDERS + + if value not in AVAILABLE_PROVIDERS: + if value == "crosshair": # pragma: no cover + install = '`pip install "hypothesis[crosshair]"` and try again.' + raise InvalidArgument(f"backend={value!r} is not available. {install}") + raise InvalidArgument( + f"backend={value!r} is not available - maybe you need to install a plugin?" + f"\n Installed backends: {sorted(AVAILABLE_PROVIDERS)!r}" + ) + return value + + +settings._define_setting( + "backend", + default="hypothesis", + show_default=False, + validator=_backend_validator, + description=""" +EXPERIMENTAL AND UNSTABLE - see :ref:`alternative-backends`. +The importable name of a backend which Hypothesis should use to generate primitive +types. We aim to support heuristic-random, solver-based, and fuzzing-based backends. +""", +) + settings.lock_further_definitions() diff --git a/hypothesis-python/src/hypothesis/core.py b/hypothesis-python/src/hypothesis/core.py index 4e4411fadf..402382c6aa 100644 --- a/hypothesis-python/src/hypothesis/core.py +++ b/hypothesis-python/src/hypothesis/core.py @@ -938,9 +938,13 @@ def run(data): with local_settings(self.settings): with deterministic_PRNG(): with BuildContext(data, is_final=is_final) as context: - # Run the test function once, via the executor hook. - # In most cases this will delegate straight to `run(data)`. - result = self.test_runner(data, run) + # providers may throw in per_case_context_fn, and we'd like + # `result` to still be set in these cases. + result = None + with data.provider.per_test_case_context_manager(): + # Run the test function once, via the executor hook. + # In most cases this will delegate straight to `run(data)`. + result = self.test_runner(data, run) # If a failure was expected, it should have been raised already, so # instead raise an appropriate diagnostic error. diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/data.py b/hypothesis-python/src/hypothesis/internal/conjecture/data.py index 10c4e17faf..486709edbb 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/data.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/data.py @@ -8,6 +8,8 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. +import abc +import contextlib import math import time from collections import defaultdict @@ -956,7 +958,7 @@ def as_result(self) -> "ConjectureResult": BYTE_MASKS[0] = 255 -class PrimitiveProvider: +class PrimitiveProvider(abc.ABC): # This is the low-level interface which would also be implemented # by e.g. CrossHair, by an Atheris-hypothesis integration, etc. # We'd then build the structured tree handling, database and replay @@ -964,16 +966,119 @@ class PrimitiveProvider: # # See https://github.com/HypothesisWorks/hypothesis/issues/3086 - def __init__(self, conjecturedata: "ConjectureData", /) -> None: + # How long a provider instance is used for. One of test_function or + # test_case. Defaults to test_function. + # + # If test_function, a single provider instance will be instantiated and used + # for the entirety of each test function. I.e., roughly one provider per + # @given annotation. This can be useful if you need to track state over many + # executions to a test function. + # + # This lifetime will cause None to be passed for the ConjectureData object + # in PrimitiveProvider.__init__, because that object is instantiated per + # test case. + # + # If test_case, a new provider instance will be instantiated and used each + # time hypothesis tries to generate a new input to the test function. This + # lifetime can access the passed ConjectureData object. + # + # Non-hypothesis providers probably want to set a lifetime of test_function. + lifetime = "test_function" + + def __init__(self, conjecturedata: Optional["ConjectureData"], /) -> None: self._cd = conjecturedata - def draw_boolean(self, p: float = 0.5, *, forced: Optional[bool] = None) -> bool: + def post_test_case_hook(self, value): + # hook for providers to modify values returned by draw_* after a full + # test case concludes. Originally exposed for crosshair to reify its + # symbolic values into actual values. + # I'm not tied to this exact function name or design. + return value + + def per_test_case_context_manager(self): + return contextlib.nullcontext() + + @abc.abstractmethod + def draw_boolean( + self, + p: float = 0.5, + *, + forced: Optional[bool] = None, + fake_forced: bool = False, + ) -> bool: + raise NotImplementedError + + @abc.abstractmethod + def draw_integer( + self, + min_value: Optional[int] = None, + max_value: Optional[int] = None, + *, + # weights are for choosing an element index from a bounded range + weights: Optional[Sequence[float]] = None, + shrink_towards: int = 0, + forced: Optional[int] = None, + fake_forced: bool = False, + ) -> int: + raise NotImplementedError + + @abc.abstractmethod + def draw_float( + self, + *, + min_value: float = -math.inf, + max_value: float = math.inf, + allow_nan: bool = True, + smallest_nonzero_magnitude: float, + # TODO: consider supporting these float widths at the IR level in the + # future. + # width: Literal[16, 32, 64] = 64, + # exclude_min and exclude_max handled higher up, + forced: Optional[float] = None, + fake_forced: bool = False, + ) -> float: + raise NotImplementedError + + @abc.abstractmethod + def draw_string( + self, + intervals: IntervalSet, + *, + min_size: int = 0, + max_size: Optional[int] = None, + forced: Optional[str] = None, + fake_forced: bool = False, + ) -> str: + raise NotImplementedError + + @abc.abstractmethod + def draw_bytes( + self, size: int, *, forced: Optional[bytes] = None, fake_forced: bool = False + ) -> bytes: + raise NotImplementedError + + +class HypothesisProvider(PrimitiveProvider): + lifetime = "test_case" + + def __init__(self, conjecturedata: Optional["ConjectureData"], /): + assert conjecturedata is not None + super().__init__(conjecturedata) + + def draw_boolean( + self, + p: float = 0.5, + *, + forced: Optional[bool] = None, + fake_forced: bool = False, + ) -> bool: """Return True with probability p (assuming a uniform generator), shrinking towards False. If ``forced`` is set to a non-None value, this will always return that value but will write choices appropriate to having drawn that value randomly.""" # Note that this could also be implemented in terms of draw_integer(). + assert self._cd is not None # NB this function is vastly more complicated than it may seem reasonable # for it to be. This is because it is used in a lot of places and it's # important for it to shrink well, so it's worth the engineering effort. @@ -1035,7 +1140,9 @@ def draw_boolean(self, p: float = 0.5, *, forced: Optional[bool] = None) -> bool partial = True i = self._cd.draw_bits( - bits, forced=None if forced is None else int(forced) + bits, + forced=None if forced is None else int(forced), + fake_forced=fake_forced, ) # We always choose the region that causes us to repeat the loop as @@ -1079,7 +1186,10 @@ def draw_integer( weights: Optional[Sequence[float]] = None, shrink_towards: int = 0, forced: Optional[int] = None, + fake_forced: bool = False, ) -> int: + assert self._cd is not None + if min_value is not None: shrink_towards = max(min_value, shrink_towards) if max_value is not None: @@ -1100,7 +1210,7 @@ def draw_integer( forced_idx = forced - shrink_towards else: forced_idx = shrink_towards + gap - forced - idx = sampler.sample(self._cd, forced=forced_idx) + idx = sampler.sample(self._cd, forced=forced_idx, fake_forced=fake_forced) # For range -2..2, interpret idx = 0..4 as [0, 1, 2, -1, -2] if idx <= gap: @@ -1109,7 +1219,7 @@ def draw_integer( return shrink_towards - (idx - gap) if min_value is None and max_value is None: - return self._draw_unbounded_integer(forced=forced) + return self._draw_unbounded_integer(forced=forced, fake_forced=fake_forced) if min_value is None: assert max_value is not None # make mypy happy @@ -1117,7 +1227,8 @@ def draw_integer( while max_value < probe: self._cd.start_example(ONE_BOUND_INTEGERS_LABEL) probe = shrink_towards + self._draw_unbounded_integer( - forced=None if forced is None else forced - shrink_towards + forced=None if forced is None else forced - shrink_towards, + fake_forced=fake_forced, ) self._cd.stop_example() return probe @@ -1128,7 +1239,8 @@ def draw_integer( while probe < min_value: self._cd.start_example(ONE_BOUND_INTEGERS_LABEL) probe = shrink_towards + self._draw_unbounded_integer( - forced=None if forced is None else forced - shrink_towards + forced=None if forced is None else forced - shrink_towards, + fake_forced=fake_forced, ) self._cd.stop_example() return probe @@ -1138,6 +1250,7 @@ def draw_integer( max_value, center=shrink_towards, forced=forced, + fake_forced=fake_forced, ) def draw_float( @@ -1152,6 +1265,7 @@ def draw_float( # width: Literal[16, 32, 64] = 64, # exclude_min and exclude_max handled higher up, forced: Optional[float] = None, + fake_forced: bool = False, ) -> float: ( sampler, @@ -1166,6 +1280,8 @@ def draw_float( smallest_nonzero_magnitude=smallest_nonzero_magnitude, ) + assert self._cd is not None + while True: self._cd.start_example(FLOAT_STRATEGY_DO_DRAW_LABEL) # If `forced in nasty_floats`, then `forced` was *probably* @@ -1174,11 +1290,17 @@ def draw_float( # i == 0 is able to produce all possible floats, and the forcing # logic is simpler if we assume this choice. forced_i = None if forced is None else 0 - i = sampler.sample(self._cd, forced=forced_i) if sampler else 0 + i = ( + sampler.sample(self._cd, forced=forced_i, fake_forced=fake_forced) + if sampler + else 0 + ) self._cd.start_example(DRAW_FLOAT_LABEL) if i == 0: result = self._draw_float( - forced_sign_bit=forced_sign_bit, forced=forced + forced_sign_bit=forced_sign_bit, + forced=forced, + fake_forced=fake_forced, ) if allow_nan and math.isnan(result): clamped = result @@ -1191,12 +1313,12 @@ def draw_float( if clamped != result and not (math.isnan(result) and allow_nan): self._cd.stop_example() self._cd.start_example(DRAW_FLOAT_LABEL) - self._draw_float(forced=clamped) + self._draw_float(forced=clamped, fake_forced=fake_forced) result = clamped else: result = nasty_floats[i - 1] - self._draw_float(forced=result) + self._draw_float(forced=result, fake_forced=fake_forced) self._cd.stop_example() # (DRAW_FLOAT_LABEL) self._cd.stop_example() # (FLOAT_STRATEGY_DO_DRAW_LABEL) @@ -1209,11 +1331,13 @@ def draw_string( min_size: int = 0, max_size: Optional[int] = None, forced: Optional[str] = None, + fake_forced: bool = False, ) -> str: if max_size is None: max_size = DRAW_STRING_DEFAULT_MAX_SIZE assert forced is None or min_size <= len(forced) <= max_size + assert self._cd is not None average_size = min( max(min_size * 2, min_size + 5), @@ -1227,6 +1351,7 @@ def draw_string( max_size=max_size, average_size=average_size, forced=None if forced is None else len(forced), + fake_forced=fake_forced, observe=False, ) while elements.more(): @@ -1237,47 +1362,74 @@ def draw_string( if len(intervals) > 256: if self.draw_boolean( - 0.2, forced=None if forced_i is None else forced_i > 255 + 0.2, + forced=None if forced_i is None else forced_i > 255, + fake_forced=fake_forced, ): i = self._draw_bounded_integer( - 256, len(intervals) - 1, forced=forced_i + 256, + len(intervals) - 1, + forced=forced_i, + fake_forced=fake_forced, ) else: - i = self._draw_bounded_integer(0, 255, forced=forced_i) + i = self._draw_bounded_integer( + 0, 255, forced=forced_i, fake_forced=fake_forced + ) else: - i = self._draw_bounded_integer(0, len(intervals) - 1, forced=forced_i) + i = self._draw_bounded_integer( + 0, len(intervals) - 1, forced=forced_i, fake_forced=fake_forced + ) chars.append(intervals.char_in_shrink_order(i)) return "".join(chars) - def draw_bytes(self, size: int, *, forced: Optional[bytes] = None) -> bytes: + def draw_bytes( + self, size: int, *, forced: Optional[bytes] = None, fake_forced: bool = False + ) -> bytes: forced_i = None if forced is not None: forced_i = int_from_bytes(forced) size = len(forced) - return self._cd.draw_bits(8 * size, forced=forced_i).to_bytes(size, "big") + assert self._cd is not None + return self._cd.draw_bits( + 8 * size, forced=forced_i, fake_forced=fake_forced + ).to_bytes(size, "big") def _draw_float( - self, forced_sign_bit: Optional[int] = None, *, forced: Optional[float] = None + self, + forced_sign_bit: Optional[int] = None, + *, + forced: Optional[float] = None, + fake_forced: bool = False, ) -> float: """ Helper for draw_float which draws a random 64-bit float. """ + assert self._cd is not None + if forced is not None: # sign_aware_lte(forced, -0.0) does not correctly handle the # math.nan case here. forced_sign_bit = math.copysign(1, forced) == -1 - is_negative = self._cd.draw_bits(1, forced=forced_sign_bit) + is_negative = self._cd.draw_bits( + 1, forced=forced_sign_bit, fake_forced=fake_forced + ) f = lex_to_float( self._cd.draw_bits( - 64, forced=None if forced is None else float_to_lex(abs(forced)) + 64, + forced=None if forced is None else float_to_lex(abs(forced)), + fake_forced=fake_forced, ) ) return -f if is_negative else f - def _draw_unbounded_integer(self, *, forced: Optional[int] = None) -> int: + def _draw_unbounded_integer( + self, *, forced: Optional[int] = None, fake_forced: bool = False + ) -> int: + assert self._cd is not None forced_i = None if forced is not None: # Using any bucket large enough to contain this integer would be a @@ -1292,7 +1444,9 @@ def _draw_unbounded_integer(self, *, forced: Optional[int] = None) -> int: size = min(size for size in INT_SIZES if bit_size <= size) forced_i = INT_SIZES.index(size) - size = INT_SIZES[INT_SIZES_SAMPLER.sample(self._cd, forced=forced_i)] + size = INT_SIZES[ + INT_SIZES_SAMPLER.sample(self._cd, forced=forced_i, fake_forced=fake_forced) + ] forced_r = None if forced is not None: @@ -1302,7 +1456,7 @@ def _draw_unbounded_integer(self, *, forced: Optional[int] = None) -> int: forced_r = -forced_r forced_r |= 1 - r = self._cd.draw_bits(size, forced=forced_r) + r = self._cd.draw_bits(size, forced=forced_r, fake_forced=fake_forced) sign = r & 1 r >>= 1 if sign: @@ -1316,9 +1470,11 @@ def _draw_bounded_integer( *, center: Optional[int] = None, forced: Optional[int] = None, + fake_forced: bool = False, ) -> int: assert lower <= upper assert forced is None or lower <= forced <= upper + assert self._cd is not None if lower == upper: # Write a value even when this is trivial so that when a bound depends # on other values we don't suddenly disappear when the gap shrinks to @@ -1337,7 +1493,9 @@ def _draw_bounded_integer( above = True else: force_above = None if forced is None else forced < center - above = not self._cd.draw_bits(1, forced=force_above) + above = not self._cd.draw_bits( + 1, forced=force_above, fake_forced=fake_forced + ) if above: gap = upper - center @@ -1350,7 +1508,7 @@ def _draw_bounded_integer( probe = gap + 1 if bits > 24 and self.draw_boolean( - 7 / 8, forced=None if forced is None else False + 7 / 8, forced=None if forced is None else False, fake_forced=fake_forced ): # For large ranges, we combine the uniform random distribution from draw_bits # with a weighting scheme with moderate chance. Cutoff at 2 ** 24 so that our @@ -1361,7 +1519,9 @@ def _draw_bounded_integer( while probe > gap: self._cd.start_example(INTEGER_RANGE_DRAW_LABEL) probe = self._cd.draw_bits( - bits, forced=None if forced is None else abs(forced - center) + bits, + forced=None if forced is None else abs(forced - center), + fake_forced=fake_forced, ) self._cd.stop_example() @@ -1476,21 +1636,59 @@ def permitted(f): return (sampler, forced_sign_bit, neg_clamper, pos_clamper, nasty_floats) +# The set of available `PrimitiveProvider`s, by name. Other libraries, such as +# crosshair, can implement this interface and add themselves; at which point users +# can configure which backend to use via settings. Keys are the name of the library, +# which doubles as the backend= setting, and values are importable class names. +# +# NOTE: this is a temporary interface. We DO NOT promise to continue supporting it! +# (but if you want to experiment and don't mind breakage, here you go) +AVAILABLE_PROVIDERS = { + "hypothesis": "hypothesis.internal.conjecture.data.HypothesisProvider", +} + + class ConjectureData: @classmethod def for_buffer( cls, buffer: Union[List[int], bytes], + *, observer: Optional[DataObserver] = None, + provider: Union[type, PrimitiveProvider] = HypothesisProvider, ) -> "ConjectureData": - return cls(len(buffer), buffer, random=None, observer=observer) + return cls( + len(buffer), buffer, random=None, observer=observer, provider=provider + ) + + @classmethod + def for_ir_tree( + cls, + ir_tree_prefix: List[IRNode], + *, + observer: Optional[DataObserver] = None, + provider: Union[type, PrimitiveProvider] = HypothesisProvider, + ) -> "ConjectureData": + from hypothesis.internal.conjecture.engine import BUFFER_SIZE + + return cls( + BUFFER_SIZE, + b"", + random=None, + ir_tree_prefix=ir_tree_prefix, + observer=observer, + provider=provider, + ) def __init__( self, max_length: int, prefix: Union[List[int], bytes, bytearray], + *, random: Optional[Random], observer: Optional[DataObserver] = None, + provider: Union[type, PrimitiveProvider] = HypothesisProvider, + ir_tree_prefix: Optional[List[IRNode]] = None, ) -> None: if observer is None: observer = DataObserver() @@ -1503,7 +1701,8 @@ def __init__( self.__prefix = bytes(prefix) self.__random = random - assert random is not None or max_length <= len(prefix) + if ir_tree_prefix is None: + assert random is not None or max_length <= len(prefix) self.blocks = Blocks(self) self.buffer: "Union[bytes, bytearray]" = bytearray() @@ -1522,7 +1721,9 @@ def __init__( self._stateful_run_times: "DefaultDict[str, float]" = defaultdict(float) self.max_depth = 0 self.has_discards = False - self.provider = PrimitiveProvider(self) + + self.provider = provider(self) if isinstance(provider, type) else provider + assert isinstance(self.provider, PrimitiveProvider) self.__result: "Optional[ConjectureResult]" = None @@ -1556,6 +1757,7 @@ def __init__( self.extra_information = ExtraInformation() + self.ir_tree_nodes = ir_tree_prefix self.start_example(TOP_LABEL) def __repr__(self): @@ -1565,7 +1767,8 @@ def __repr__(self): ", frozen" if self.frozen else "", ) - # A bit of explanation of the `observe` argument in our draw_* functions. + # A bit of explanation of the `observe` and `fake_forced` arguments in our + # draw_* functions. # # There are two types of draws: sub-ir and super-ir. For instance, some ir # nodes use `many`, which in turn calls draw_boolean. But some strategies @@ -1577,6 +1780,17 @@ def __repr__(self): # # `observe` formalizes this distinction. The draw will only be written to # the DataTree if observe is True. + # + # `fake_forced` deals with a different problem. We use `forced=` to convert + # ir prefixes, which are potentially from other backends, into our backing + # bits representation. This works fine, except using `forced=` in this way + # also sets `was_forced=True` for all blocks, even those that weren't forced + # in the traditional way. The shrinker chokes on this due to thinking that + # nothing can be modified. + # + # Setting `fake_forced` to true says that yes, we want to force a particular + # value to be returned, but we don't want to treat that block as fixed for + # e.g. the shrinker. def draw_integer( self, @@ -1587,6 +1801,7 @@ def draw_integer( weights: Optional[Sequence[float]] = None, shrink_towards: int = 0, forced: Optional[int] = None, + fake_forced: bool = False, observe: bool = True, ) -> int: # Validate arguments @@ -1617,13 +1832,25 @@ def draw_integer( "shrink_towards": shrink_towards, }, ) - value = self.provider.draw_integer(**kwargs, forced=forced) + + if self.ir_tree_nodes is not None and observe: + node = self._pop_ir_tree_node("integer", kwargs) + assert isinstance(node.value, int) + forced = node.value + fake_forced = not node.was_forced + + value = self.provider.draw_integer( + **kwargs, forced=forced, fake_forced=fake_forced + ) if observe: self.observer.draw_integer( - value, kwargs=kwargs, was_forced=forced is not None + value, kwargs=kwargs, was_forced=forced is not None and not fake_forced ) self.__example_record.record_ir_draw( - "integer", value, kwargs=kwargs, was_forced=forced is not None + "integer", + value, + kwargs=kwargs, + was_forced=forced is not None and not fake_forced, ) return value @@ -1639,6 +1866,7 @@ def draw_float( # width: Literal[16, 32, 64] = 64, # exclude_min and exclude_max handled higher up, forced: Optional[float] = None, + fake_forced: bool = False, observe: bool = True, ) -> float: assert smallest_nonzero_magnitude > 0 @@ -1660,13 +1888,25 @@ def draw_float( "smallest_nonzero_magnitude": smallest_nonzero_magnitude, }, ) - value = self.provider.draw_float(**kwargs, forced=forced) + + if self.ir_tree_nodes is not None and observe: + node = self._pop_ir_tree_node("float", kwargs) + assert isinstance(node.value, float) + forced = node.value + fake_forced = not node.was_forced + + value = self.provider.draw_float( + **kwargs, forced=forced, fake_forced=fake_forced + ) if observe: self.observer.draw_float( - value, kwargs=kwargs, was_forced=forced is not None + value, kwargs=kwargs, was_forced=forced is not None and not fake_forced ) self.__example_record.record_ir_draw( - "float", value, kwargs=kwargs, was_forced=forced is not None + "float", + value, + kwargs=kwargs, + was_forced=forced is not None and not fake_forced, ) return value @@ -1677,6 +1917,7 @@ def draw_string( min_size: int = 0, max_size: Optional[int] = None, forced: Optional[str] = None, + fake_forced: bool = False, observe: bool = True, ) -> str: assert forced is None or min_size <= len(forced) @@ -1689,13 +1930,24 @@ def draw_string( "max_size": max_size, }, ) - value = self.provider.draw_string(**kwargs, forced=forced) + if self.ir_tree_nodes is not None and observe: + node = self._pop_ir_tree_node("string", kwargs) + assert isinstance(node.value, str) + forced = node.value + fake_forced = not node.was_forced + + value = self.provider.draw_string( + **kwargs, forced=forced, fake_forced=fake_forced + ) if observe: self.observer.draw_string( - value, kwargs=kwargs, was_forced=forced is not None + value, kwargs=kwargs, was_forced=forced is not None and not fake_forced ) self.__example_record.record_ir_draw( - "string", value, kwargs=kwargs, was_forced=forced is not None + "string", + value, + kwargs=kwargs, + was_forced=forced is not None and not fake_forced, ) return value @@ -1705,24 +1957,42 @@ def draw_bytes( size: int, *, forced: Optional[bytes] = None, + fake_forced: bool = False, observe: bool = True, ) -> bytes: assert forced is None or len(forced) == size assert size >= 0 kwargs: BytesKWargs = self._pooled_kwargs("bytes", {"size": size}) - value = self.provider.draw_bytes(**kwargs, forced=forced) + + if self.ir_tree_nodes is not None and observe: + node = self._pop_ir_tree_node("bytes", kwargs) + assert isinstance(node.value, bytes) + forced = node.value + fake_forced = not node.was_forced + + value = self.provider.draw_bytes( + **kwargs, forced=forced, fake_forced=fake_forced + ) if observe: self.observer.draw_bytes( - value, kwargs=kwargs, was_forced=forced is not None + value, kwargs=kwargs, was_forced=forced is not None and not fake_forced ) self.__example_record.record_ir_draw( - "bytes", value, kwargs=kwargs, was_forced=forced is not None + "bytes", + value, + kwargs=kwargs, + was_forced=forced is not None and not fake_forced, ) return value def draw_boolean( - self, p: float = 0.5, *, forced: Optional[bool] = None, observe: bool = True + self, + p: float = 0.5, + *, + forced: Optional[bool] = None, + observe: bool = True, + fake_forced: bool = False, ) -> bool: # Internally, we treat probabilities lower than 1 / 2**64 as # unconditionally false. @@ -1735,13 +2005,25 @@ def draw_boolean( assert p < (1 - 2 ** (-64)) kwargs: BooleanKWargs = self._pooled_kwargs("boolean", {"p": p}) - value = self.provider.draw_boolean(**kwargs, forced=forced) + + if self.ir_tree_nodes is not None and observe: + node = self._pop_ir_tree_node("boolean", kwargs) + assert isinstance(node.value, bool) + forced = node.value + fake_forced = not node.was_forced + + value = self.provider.draw_boolean( + **kwargs, forced=forced, fake_forced=fake_forced + ) if observe: self.observer.draw_boolean( - value, kwargs=kwargs, was_forced=forced is not None + value, kwargs=kwargs, was_forced=forced is not None and not fake_forced ) self.__example_record.record_ir_draw( - "boolean", value, kwargs=kwargs, was_forced=forced is not None + "boolean", + value, + kwargs=kwargs, + was_forced=forced is not None and not fake_forced, ) return value @@ -1765,6 +2047,14 @@ def _pooled_kwargs(self, ir_type, kwargs): POOLED_KWARGS_CACHE[key] = kwargs return kwargs + def _pop_ir_tree_node(self, ir_type: IRTypeName, kwargs: IRKWargsType) -> IRNode: + assert self.ir_tree_nodes is not None + node = self.ir_tree_nodes.pop(0) + assert node.ir_type == ir_type + assert kwargs == node.kwargs + + return node + def as_result(self) -> Union[ConjectureResult, _Overrun]: """Convert the result of running this test into either an Overrun object or a ConjectureResult.""" @@ -1945,13 +2235,22 @@ def choice( values: Sequence[T], *, forced: Optional[T] = None, + fake_forced: bool = False, observe: bool = True, ) -> T: forced_i = None if forced is None else values.index(forced) - i = self.draw_integer(0, len(values) - 1, forced=forced_i, observe=observe) + i = self.draw_integer( + 0, + len(values) - 1, + forced=forced_i, + fake_forced=fake_forced, + observe=observe, + ) return values[i] - def draw_bits(self, n: int, *, forced: Optional[int] = None) -> int: + def draw_bits( + self, n: int, *, forced: Optional[int] = None, fake_forced: bool = False + ) -> int: """Return an ``n``-bit integer from the underlying source of bytes. If ``forced`` is set to an integer will instead ignore the underlying source and simulate a draw as if it had @@ -1993,7 +2292,7 @@ def draw_bits(self, n: int, *, forced: Optional[int] = None) -> int: self.buffer.extend(buf) self.index = len(self.buffer) - if forced is not None: + if forced is not None and not fake_forced: self.forced_indices.update(range(initial, self.index)) self.blocks.add_endpoint(self.index) diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py index 2a011a8b11..b0cf812298 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py @@ -8,6 +8,7 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. +import importlib import math import time from collections import defaultdict @@ -15,26 +16,26 @@ from datetime import timedelta from enum import Enum from random import Random, getrandbits +from typing import Union import attr from hypothesis import HealthCheck, Phase, Verbosity, settings as Settings from hypothesis._settings import local_settings -from hypothesis.errors import StopTest +from hypothesis.errors import InvalidArgument, StopTest from hypothesis.internal.cache import LRUReusedCache from hypothesis.internal.compat import ceil, int_from_bytes from hypothesis.internal.conjecture.data import ( + AVAILABLE_PROVIDERS, ConjectureData, ConjectureResult, DataObserver, + HypothesisProvider, Overrun, + PrimitiveProvider, Status, ) -from hypothesis.internal.conjecture.datatree import ( - DataTree, - PreviouslyUnseenBehaviour, - TreeRecordingObserver, -) +from hypothesis.internal.conjecture.datatree import DataTree, PreviouslyUnseenBehaviour from hypothesis.internal.conjecture.junkdrawer import clamp, ensure_free_stackframes from hypothesis.internal.conjecture.pareto import NO_SCORE, ParetoFront, ParetoOptimiser from hypothesis.internal.conjecture.shrinker import Shrinker, sort_key @@ -114,6 +115,20 @@ class RunIsComplete(Exception): pass +def _get_provider(backend: str) -> Union[type, PrimitiveProvider]: + mname, cname = AVAILABLE_PROVIDERS[backend].rsplit(".", 1) + provider_cls = getattr(importlib.import_module(mname), cname) + if provider_cls.lifetime == "test_function": + return provider_cls(None) + elif provider_cls.lifetime == "test_case": + return provider_cls + else: + raise InvalidArgument( + f"invalid lifetime {provider_cls.lifetime} for provider {provider_cls.__name__}. " + "Expected one of 'test_function', 'test_case'." + ) + + class ConjectureRunner: def __init__( self, @@ -151,6 +166,8 @@ def __init__( self.tree = DataTree() + self.provider = _get_provider(self.settings.backend) + self.best_observed_targets = defaultdict(lambda: NO_SCORE) self.best_examples_of_observed_targets = {} @@ -171,6 +188,7 @@ def __init__( self.__data_cache = LRUReusedCache(CACHE_SIZE) self.__pending_call_explanation = None + self._switch_to_hypothesis_provider = False def explain_next_call_as(self, explanation): self.__pending_call_explanation = explanation @@ -198,7 +216,7 @@ def should_optimise(self): return Phase.target in self.settings.phases def __tree_is_exhausted(self): - return self.tree.is_exhausted + return self.tree.is_exhausted and self.settings.backend == "hypothesis" def __stoppable_test_function(self, data): """Run ``self._test_function``, but convert a ``StopTest`` exception @@ -226,7 +244,6 @@ def test_function(self, data): self.debug(self.__pending_call_explanation) self.__pending_call_explanation = None - assert isinstance(data.observer, TreeRecordingObserver) self.call_count += 1 interrupted = False @@ -284,6 +301,21 @@ def test_function(self, data): self.valid_examples += 1 if data.status == Status.INTERESTING: + if self.settings.backend != "hypothesis": + for node in data.examples.ir_tree_nodes: + value = data.provider.post_test_case_hook(node.value) + # require providers to return something valid here. + assert ( + value is not None + ), "providers must return a non-null value from post_test_case_hook" + node.value = value + + # drive the ir tree through the test function to convert it + # to a buffer + data = ConjectureData.for_ir_tree(data.examples.ir_tree_nodes) + self.__stoppable_test_function(data) + self.__data_cache[data.buffer] = data.as_result() + key = data.interesting_origin changed = False try: @@ -702,6 +734,15 @@ def generate_new_examples(self): ran_optimisations = False while self.should_generate_more(): + # Unfortunately generate_novel_prefix still operates in terms of + # a buffer and uses HypothesisProvider as its backing provider, + # not whatever is specified by the backend. We can improve this + # once more things are on the ir. + if self.settings.backend != "hypothesis": + data = self.new_conjecture_data(prefix=b"", max_length=BUFFER_SIZE) + self.test_function(data) + continue + self._current_phase = "generate" prefix = self.generate_novel_prefix() assert len(prefix) <= BUFFER_SIZE @@ -905,26 +946,40 @@ def pareto_optimise(self): ParetoOptimiser(self).run() def _run(self): + # have to use the primitive provider to interpret database bits... + self._switch_to_hypothesis_provider = True with self._log_phase_statistics("reuse"): self.reuse_existing_examples() + # ...but we should use the supplied provider when generating... + self._switch_to_hypothesis_provider = False with self._log_phase_statistics("generate"): self.generate_new_examples() # We normally run the targeting phase mixed in with the generate phase, # but if we've been asked to run it but not generation then we have to - # run it explciitly on its own here. + # run it explicitly on its own here. if Phase.generate not in self.settings.phases: self._current_phase = "target" self.optimise_targets() + # ...and back to the primitive provider when shrinking. + self._switch_to_hypothesis_provider = True with self._log_phase_statistics("shrink"): self.shrink_interesting_examples() self.exit_with(ExitReason.finished) def new_conjecture_data(self, prefix, max_length=BUFFER_SIZE, observer=None): + provider = ( + HypothesisProvider if self._switch_to_hypothesis_provider else self.provider + ) + observer = observer or self.tree.new_observer() + if self.settings.backend != "hypothesis": + observer = DataObserver() + return ConjectureData( prefix=prefix, max_length=max_length, random=self.random, - observer=observer or self.tree.new_observer(), + observer=observer, + provider=provider, ) def new_conjecture_data_for_buffer(self, buffer): @@ -1066,20 +1121,21 @@ def kill_branch(self): prefix=buffer, max_length=max_length, observer=observer ) - try: - self.tree.simulate_test_function(dummy_data) - except PreviouslyUnseenBehaviour: - pass - else: - if dummy_data.status > Status.OVERRUN: - dummy_data.freeze() - try: - return self.__data_cache[dummy_data.buffer] - except KeyError: - pass + if self.settings.backend == "hypothesis": + try: + self.tree.simulate_test_function(dummy_data) + except PreviouslyUnseenBehaviour: + pass else: - self.__data_cache[buffer] = Overrun - return Overrun + if dummy_data.status > Status.OVERRUN: + dummy_data.freeze() + try: + return self.__data_cache[dummy_data.buffer] + except KeyError: + pass + else: + self.__data_cache[buffer] = Overrun + return Overrun # We didn't find a match in the tree, so we need to run the test # function normally. Note that test_function will automatically diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/utils.py b/hypothesis-python/src/hypothesis/internal/conjecture/utils.py index 5e77437a78..509c03ef71 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/utils.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/utils.py @@ -166,7 +166,13 @@ def __init__(self, weights: Sequence[float], *, observe: bool = True): self.table.append((base, alternate, alternate_chance)) self.table.sort() - def sample(self, data: "ConjectureData", forced: Optional[int] = None) -> int: + def sample( + self, + data: "ConjectureData", + *, + forced: Optional[int] = None, + fake_forced: bool = False, + ) -> int: data.start_example(SAMPLE_IN_SAMPLER_LABEL) forced_choice = ( # pragma: no branch # https://github.com/nedbat/coveragepy/issues/1617 None @@ -178,7 +184,10 @@ def sample(self, data: "ConjectureData", forced: Optional[int] = None) -> int: ) ) base, alternate, alternate_chance = data.choice( - self.table, forced=forced_choice, observe=self.observe + self.table, + forced=forced_choice, + fake_forced=fake_forced, + observe=self.observe, ) forced_use_alternate = None if forced is not None: @@ -189,7 +198,10 @@ def sample(self, data: "ConjectureData", forced: Optional[int] = None) -> int: assert forced == base or forced_use_alternate use_alternate = data.draw_boolean( - alternate_chance, forced=forced_use_alternate, observe=self.observe + alternate_chance, + forced=forced_use_alternate, + fake_forced=fake_forced, + observe=self.observe, ) data.stop_example() if use_alternate: @@ -224,6 +236,7 @@ def __init__( average_size: Union[int, float], *, forced: Optional[int] = None, + fake_forced: bool = False, observe: bool = True, ) -> None: assert 0 <= min_size <= average_size <= max_size @@ -232,6 +245,7 @@ def __init__( self.max_size = max_size self.data = data self.forced_size = forced + self.fake_forced = fake_forced self.p_continue = _calc_p_continue(average_size - min_size, max_size - min_size) self.count = 0 self.rejections = 0 @@ -267,7 +281,10 @@ def more(self) -> bool: elif self.forced_size is not None: forced_result = self.count < self.forced_size should_continue = self.data.draw_boolean( - self.p_continue, forced=forced_result, observe=self.observe + self.p_continue, + forced=forced_result, + fake_forced=self.fake_forced, + observe=self.observe, ) if should_continue: diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/strategies.py b/hypothesis-python/src/hypothesis/strategies/_internal/strategies.py index 46d4005cdb..83b0e6b059 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/strategies.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/strategies.py @@ -28,7 +28,7 @@ ) from hypothesis._settings import HealthCheck, Phase, Verbosity, settings -from hypothesis.control import _current_build_context, assume +from hypothesis.control import _current_build_context from hypothesis.errors import ( HypothesisException, HypothesisWarning, @@ -1002,7 +1002,6 @@ def do_draw(self, data: ConjectureData) -> Ex: def do_filtered_draw(self, data): for i in range(3): - start_index = data.index data.start_example(FILTERED_SEARCH_STRATEGY_DO_DRAW_LABEL) value = data.draw(self.filtered_strategy) if self.condition(value): @@ -1012,10 +1011,6 @@ def do_filtered_draw(self, data): data.stop_example(discard=True) if i == 0: data.events[f"Retried draw from {self!r} to satisfy filter"] = "" - # This is to guard against the case where we consume no data. - # As long as we consume data, we'll eventually pass or raise. - # But if we don't this could be an infinite loop. - assume(data.index > start_index) return filter_not_satisfied diff --git a/hypothesis-python/tests/conjecture/test_alt_backend.py b/hypothesis-python/tests/conjecture/test_alt_backend.py new file mode 100644 index 0000000000..0eda6e93e4 --- /dev/null +++ b/hypothesis-python/tests/conjecture/test_alt_backend.py @@ -0,0 +1,343 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +import math +import sys +from contextlib import contextmanager +from random import Random +from typing import Optional, Sequence + +import pytest + +from hypothesis import given, settings, strategies as st +from hypothesis.database import InMemoryExampleDatabase +from hypothesis.errors import InvalidArgument +from hypothesis.internal.compat import int_to_bytes +from hypothesis.internal.conjecture.data import ( + AVAILABLE_PROVIDERS, + ConjectureData, + PrimitiveProvider, +) +from hypothesis.internal.conjecture.engine import ConjectureRunner +from hypothesis.internal.floats import SIGNALING_NAN +from hypothesis.internal.intervalsets import IntervalSet + +from tests.common.debug import minimal + + +class PrngProvider(PrimitiveProvider): + # A test-only implementation of the PrimitiveProvider interface, which uses + # a very simple PRNG to choose each value. Dumb but efficient, and entirely + # independent of our real backend + + def __init__(self, conjecturedata: "ConjectureData", /) -> None: + super().__init__(conjecturedata) + self.prng = Random() + + def draw_boolean( + self, + p: float = 0.5, + *, + forced: Optional[bool] = None, + fake_forced: bool = False, + ) -> bool: + if forced is not None: + return forced + return self.prng.random() < p + + def draw_integer( + self, + min_value: Optional[int] = None, + max_value: Optional[int] = None, + *, + # weights are for choosing an element index from a bounded range + weights: Optional[Sequence[float]] = None, + shrink_towards: int = 0, + forced: Optional[int] = None, + fake_forced: bool = False, + ) -> int: + assert isinstance(shrink_towards, int) # otherwise ignored here + if forced is not None: + return forced + + if weights is not None: + assert min_value is not None + assert max_value is not None + # use .choices so we can use the weights= param. + choices = self.prng.choices( + range(min_value, max_value + 1), weights=weights, k=1 + ) + return choices[0] + + if min_value is None and max_value is None: + min_value = -(2**127) + max_value = 2**127 - 1 + elif min_value is None: + min_value = max_value - 2**64 + elif max_value is None: + max_value = min_value + 2**64 + return self.prng.randint(min_value, max_value) + + def draw_float( + self, + *, + min_value: float = -math.inf, + max_value: float = math.inf, + allow_nan: bool = True, + smallest_nonzero_magnitude: float, + forced: Optional[float] = None, + fake_forced: bool = False, + ) -> float: + if forced is not None: + return forced + + if allow_nan and self.prng.random() < 1 / 32: + nans = [math.nan, -math.nan, SIGNALING_NAN, -SIGNALING_NAN] + return self.prng.choice(nans) + + # small chance of inf values, if they are in bounds + if min_value <= math.inf <= max_value and self.prng.random() < 1 / 32: + return math.inf + if min_value <= -math.inf <= max_value and self.prng.random() < 1 / 32: + return -math.inf + + # get rid of infs, they cause nans if we pass them to prng.uniform + if min_value in [-math.inf, math.inf]: + min_value = math.copysign(1, min_value) * sys.float_info.max + # being too close to the bounds causes prng.uniform to only return + # inf. + min_value /= 2 + if max_value in [-math.inf, math.inf]: + max_value = math.copysign(1, max_value) * sys.float_info.max + max_value /= 2 + + value = self.prng.uniform(min_value, max_value) + if value and abs(value) < smallest_nonzero_magnitude: + return math.copysign(0.0, value) + return value + + def draw_string( + self, + intervals: IntervalSet, + *, + min_size: int = 0, + max_size: Optional[int] = None, + forced: Optional[str] = None, + fake_forced: bool = False, + ) -> str: + if forced is not None: + return forced + size = self.prng.randint( + min_size, max(min_size, min(100 if max_size is None else max_size, 100)) + ) + return "".join(map(chr, self.prng.choices(intervals, k=size))) + + def draw_bytes( + self, size: int, *, forced: Optional[bytes] = None, fake_forced: bool = False + ) -> bytes: + if forced is not None: + return forced + return self.prng.randbytes(size) + + +@contextmanager +def temp_register_backend(name, cls): + try: + AVAILABLE_PROVIDERS[name] = f"{__name__}.{cls.__name__}" + yield + finally: + AVAILABLE_PROVIDERS.pop(name) + + +@pytest.mark.parametrize( + "strategy", + [ + st.booleans(), + st.integers(0, 3), + st.floats(0, 1), + st.text(max_size=3), + st.binary(max_size=3), + ], + ids=repr, +) +def test_find_with_backend_then_convert_to_buffer_shrink_and_replay(strategy): + db = InMemoryExampleDatabase() + assert not db.data + + with temp_register_backend("prng", PrngProvider): + + @settings(database=db, backend="prng") + @given(strategy) + def test(value): + if isinstance(value, float): + assert value >= 0.5 + else: + assert value + + with pytest.raises(AssertionError): + test() + + assert db.data + buffers = {x for x in db.data[next(iter(db.data))] if x} + assert buffers, db.data + + +def test_backend_can_shrink_integers(): + with temp_register_backend("prng", PrngProvider): + n = minimal( + st.integers(), + lambda n: n >= 123456, + settings=settings(backend="prng", database=None), + ) + + assert n == 123456 + + +def test_backend_can_shrink_bytes(): + with temp_register_backend("prng", PrngProvider): + b = minimal( + # this test doubles as coverage for popping draw_bytes ir nodes, + # and that path is only taken with fixed size for the moment. can + # be removed when we support variable length binary at the ir level. + st.binary(min_size=2, max_size=2), + lambda b: len(b) >= 2 and b[1] >= 10, + settings=settings(backend="prng", database=None), + ) + + assert b == int_to_bytes(10, size=2) + + +def test_backend_can_shrink_strings(): + with temp_register_backend("prng", PrngProvider): + s = minimal( + st.text(), + lambda s: len(s) >= 10, + settings=settings(backend="prng", database=None), + ) + + assert len(s) == 10 + + +def test_backend_can_shrink_booleans(): + with temp_register_backend("prng", PrngProvider): + b = minimal( + st.booleans(), lambda b: b, settings=settings(backend="prng", database=None) + ) + + assert b + + +def test_backend_can_shrink_floats(): + with temp_register_backend("prng", PrngProvider): + f = minimal( + st.floats(), + lambda f: f >= 100.5, + settings=settings(backend="prng", database=None), + ) + + assert f == 101.0 + + +# trivial provider for tests which don't care about drawn distributions. +class TrivialProvider(PrimitiveProvider): + def draw_integer(self, *args, **kwargs): + return 1 + + def draw_boolean(self, *args, **kwargs): + return True + + def draw_float(self, *args, **kwargs): + return 1.0 + + def draw_bytes(self, *args, **kwargs): + return b"" + + def draw_string(self, *args, **kwargs): + return "" + + +class InvalidLifetime(TrivialProvider): + + lifetime = "forever and a day!" + + +def test_invalid_lifetime(): + with temp_register_backend("invalid_lifetime", InvalidLifetime): + with pytest.raises(InvalidArgument): + ConjectureRunner( + lambda: True, settings=settings(backend="invalid_lifetime") + ) + + +function_lifetime_init_count = 0 + + +class LifetimeTestFunction(TrivialProvider): + lifetime = "test_function" + + def __init__(self, conjecturedata): + super().__init__(conjecturedata) + # hacky, but no easy alternative. + global function_lifetime_init_count + function_lifetime_init_count += 1 + + +def test_function_lifetime(): + with temp_register_backend("lifetime_function", LifetimeTestFunction): + + @given(st.integers()) + @settings(backend="lifetime_function") + def test_function(n): + pass + + assert function_lifetime_init_count == 0 + test_function() + assert function_lifetime_init_count == 1 + test_function() + assert function_lifetime_init_count == 2 + + +test_case_lifetime_init_count = 0 + + +class LifetimeTestCase(TrivialProvider): + lifetime = "test_case" + + def __init__(self, conjecturedata): + super().__init__(conjecturedata) + global test_case_lifetime_init_count + test_case_lifetime_init_count += 1 + + +def test_case_lifetime(): + test_function_count = 0 + + with temp_register_backend("lifetime_case", LifetimeTestCase): + + @given(st.integers()) + @settings(backend="lifetime_case", database=InMemoryExampleDatabase()) + def test_function(n): + nonlocal test_function_count + test_function_count += 1 + + assert test_case_lifetime_init_count == 0 + test_function() + + # we create a new provider each time we *try* to generate an input to the + # test function, but this could be filtered out, discarded as duplicate, + # etc. We also sometimes try predetermined inputs to the test function, + # such as the zero buffer, which does not entail creating providers. + # These two facts combined mean that the number of inits could be + # anywhere reasonably close to the number of function calls. + assert ( + test_function_count - 10 + <= test_case_lifetime_init_count + <= test_function_count + 10 + ) diff --git a/hypothesis-python/tests/cover/test_health_checks.py b/hypothesis-python/tests/cover/test_health_checks.py index 5cc9217d13..72bbb1d675 100644 --- a/hypothesis-python/tests/cover/test_health_checks.py +++ b/hypothesis-python/tests/cover/test_health_checks.py @@ -237,9 +237,9 @@ def test_does_not_trigger_health_check_on_simple_strategies(monkeypatch): # We need to make drawing data artificially slow in order to trigger this # effect. This isn't actually slow because time is fake in our CI, but # we need it to pretend to be. - def draw_bits(self, n, forced=None): + def draw_bits(self, n, *, forced=None, fake_forced=False): time.sleep(0.001) - return existing_draw_bits(self, n, forced=forced) + return existing_draw_bits(self, n, forced=forced, fake_forced=fake_forced) monkeypatch.setattr(ConjectureData, "draw_bits", draw_bits) diff --git a/hypothesis-python/tests/cover/test_settings.py b/hypothesis-python/tests/cover/test_settings.py index 2ab6838d56..b28c23dba9 100644 --- a/hypothesis-python/tests/cover/test_settings.py +++ b/hypothesis-python/tests/cover/test_settings.py @@ -456,6 +456,7 @@ def test_derandomise_with_explicit_database_is_invalid(): {"deadline": 0}, {"deadline": True}, {"deadline": False}, + {"backend": "this_backend_does_not_exist"}, ], ) def test_invalid_settings_are_errors(kwargs): diff --git a/hypothesis-python/tests/ghostwriter/recorded/hypothesis_module_magic.txt b/hypothesis-python/tests/ghostwriter/recorded/hypothesis_module_magic.txt index 4b54924753..2bfb4f1b83 100644 --- a/hypothesis-python/tests/ghostwriter/recorded/hypothesis_module_magic.txt +++ b/hypothesis-python/tests/ghostwriter/recorded/hypothesis_module_magic.txt @@ -73,6 +73,7 @@ def test_fuzz_seed(seed: typing.Hashable) -> None: suppress_health_check=st.just(not_set), deadline=st.just(not_set), print_blob=st.just(not_set), + backend=st.just(not_set), ) def test_fuzz_settings( parent: typing.Optional[hypothesis.settings], @@ -86,6 +87,7 @@ def test_fuzz_settings( suppress_health_check, deadline: typing.Union[int, float, datetime.timedelta, None], print_blob: bool, + backend: str, ) -> None: hypothesis.settings( parent=parent, @@ -99,6 +101,7 @@ def test_fuzz_settings( suppress_health_check=suppress_health_check, deadline=deadline, print_blob=print_blob, + backend=backend, )