Merge pull request #3846 from tybug/test-forced-improvements

Improvements to `forced` and forced testing

hypothesis-python/RELEASE.rst

@@ -0,0 +1,3 @@
+RELEASE_TYPE: patch
+
+This patch refactors some more internals, continuing our work on supporting alternative backends (:issue:`3086`). There is no user-visible change.

hypothesis-python/src/hypothesis/internal/conjecture/data.py

@@ -1487,7 +1487,7 @@ def draw_integer(
             assert min_value is not None
             assert max_value is not None
             width = max_value - min_value + 1
-            assert width <= 1024  # arbitrary practical limit
+            assert width <= 255  # arbitrary practical limit
             assert len(weights) == width
 
         if forced is not None and (min_value is None or max_value is None):
@@ -1558,6 +1558,16 @@ def draw_bytes(self, size: int, *, forced: Optional[bytes] = None) -> bytes:
         return self.provider.draw_bytes(size, forced=forced)
 
     def draw_boolean(self, p: float = 0.5, *, forced: Optional[bool] = None) -> bool:
+        # Internally, we treat probabilities lower than 1 / 2**64 as
+        # unconditionally false.
+        #
+        # Note that even if we lift this 64 bit restriction in the future, p
+        # cannot be 0 (1) when forced is True (False).
+        if forced is True:
+            assert p > 2 ** (-64)
+        if forced is False:
+            assert p < (1 - 2 ** (-64))
+
         return self.provider.draw_boolean(p, forced=forced)
 
     def as_result(self) -> Union[ConjectureResult, _Overrun]:

hypothesis-python/src/hypothesis/internal/conjecture/utils.py

@@ -172,14 +172,24 @@ def sample(self, data: "ConjectureData", forced: Optional[int] = None) -> int:
         forced_choice = (  # pragma: no branch # https://github.com/nedbat/coveragepy/issues/1617
             None
             if forced is None
-            else next((b, a, a_c) for (b, a, a_c) in self.table if forced in (b, a))
+            else next(
+                (base, alternate, alternate_chance)
+                for (base, alternate, alternate_chance) in self.table
+                if forced == base or (forced == alternate and alternate_chance > 0)
+            )
         )
         base, alternate, alternate_chance = data.choice(
             self.table, forced=forced_choice
         )
-        use_alternate = data.draw_boolean(
-            alternate_chance, forced=None if forced is None else forced == alternate
-        )
+        forced_use_alternate = None
+        if forced is not None:
+            # we maintain this invariant when picking forced_choice above.
+            # This song and dance about alternate_chance > 0 is to avoid forcing
+            # e.g. draw_boolean(p=0, forced=True), which is an error.
+            forced_use_alternate = forced == alternate and alternate_chance > 0
+            assert forced == base or forced_use_alternate
+
+        use_alternate = data.draw_boolean(alternate_chance, forced=forced_use_alternate)
         data.stop_example()
         if use_alternate:
             assert forced is None or alternate == forced, (forced, alternate)

hypothesis-python/src/hypothesis/strategies/_internal/strategies.py

@@ -473,6 +473,8 @@ class SampledFromStrategy(SearchStrategy):
     non-empty subset of the elements.
     """
 
+    _MAX_FILTER_CALLS = 10_000
+
     def __init__(self, elements, repr_=None, transformations=()):
         super().__init__()
         self.elements = cu.check_sample(elements, "sampled_from")
@@ -567,8 +569,7 @@ def do_filtered_draw(self, data):
 
         # Impose an arbitrary cutoff to prevent us from wasting too much time
         # on very large element lists.
-        cutoff = 10000
-        max_good_indices = min(max_good_indices, cutoff)
+        max_good_indices = min(max_good_indices, self._MAX_FILTER_CALLS - 3)
 
         # Before building the list of allowed indices, speculatively choose
         # one of them. We don't yet know how many allowed indices there will be,
@@ -580,7 +581,7 @@ def do_filtered_draw(self, data):
         # just use that and return immediately.  Note that we also track the
         # allowed elements, in case of .map(some_stateful_function)
         allowed = []
-        for i in range(min(len(self.elements), cutoff)):
+        for i in range(min(len(self.elements), self._MAX_FILTER_CALLS - 3)):
             if i not in known_bad_indices:
                 element = self.get_element(i)
                 if element is not filter_not_satisfied:

hypothesis-python/tests/common/strategies.py

@@ -8,16 +8,19 @@
 # v. 2.0. If a copy of the MPL was not distributed with this file, You can
 # obtain one at https://mozilla.org/MPL/2.0/.
 
+import sys
 import time
+from itertools import islice
 
+from hypothesis import strategies as st
+from hypothesis.internal.intervalsets import IntervalSet
 from hypothesis.strategies._internal import SearchStrategy
 
 
 class _Slow(SearchStrategy):
     def do_draw(self, data):
         time.sleep(1.01)
         data.draw_bytes(2)
-        return None
 
 
 SLOW = _Slow()
@@ -48,3 +51,27 @@ def do_draw(self, data):
                 self.accepted.add(x)
                 return True
         return False
+
+
+def build_intervals(intervals):
+    it = iter(intervals)
+    while batch := tuple(islice(it, 2)):
+        # To guarantee we return pairs of 2, drop the last batch if it's
+        # unbalanced.
+        # Dropping a random element if the list is odd would probably make for
+        # a better distribution, but a task for another day.
+        if len(batch) < 2:
+            continue
+        yield batch
+
+
+def interval_lists(min_codepoint=0, max_codepoint=sys.maxunicode):
+    return (
+        st.lists(st.integers(min_codepoint, max_codepoint), unique=True)
+        .map(sorted)
+        .map(build_intervals)
+    )
+
+
+def intervals(min_codepoint=0, max_codepoint=sys.maxunicode):
+    return st.builds(IntervalSet, interval_lists(min_codepoint, max_codepoint))

hypothesis-python/tests/conjecture/common.py

@@ -8,14 +8,19 @@
 # v. 2.0. If a copy of the MPL was not distributed with this file, You can
 # obtain one at https://mozilla.org/MPL/2.0/.
 
+import math
 from contextlib import contextmanager
+from random import Random
 
-from hypothesis import HealthCheck, settings
+from hypothesis import HealthCheck, assume, settings, strategies as st
 from hypothesis.internal.conjecture import engine as engine_module
-from hypothesis.internal.conjecture.data import Status
-from hypothesis.internal.conjecture.engine import ConjectureRunner
+from hypothesis.internal.conjecture.data import ConjectureData, Status
+from hypothesis.internal.conjecture.engine import BUFFER_SIZE, ConjectureRunner
 from hypothesis.internal.conjecture.utils import calc_label_from_name
 from hypothesis.internal.entropy import deterministic_PRNG
+from hypothesis.strategies._internal.strings import OneCharStringStrategy, TextStrategy
+
+from tests.common.strategies import intervals
 
 SOME_LABEL = calc_label_from_name("some label")
 
@@ -67,3 +72,153 @@ def accept(f):
             )
 
     return accept
+
+
+def fresh_data():
+    return ConjectureData(BUFFER_SIZE, prefix=b"", random=Random())
+
+
+@st.composite
+def draw_integer_kwargs(
+    draw,
+    *,
+    use_min_value=True,
+    use_max_value=True,
+    use_shrink_towards=True,
+    use_weights=True,
+    use_forced=False,
+):
+    min_value = None
+    max_value = None
+    shrink_towards = 0
+    weights = None
+
+    # this generation is complicated to deal with maintaining any combination of
+    # the following invariants, depending on which parameters are passed:
+    #
+    # (1) min_value <= forced <= max_value
+    # (2) max_value - min_value + 1 == len(weights)
+    # (3) len(weights) <= 255
+
+    forced = draw(st.integers()) if use_forced else None
+    if use_weights:
+        assert use_max_value
+        assert use_min_value
+        # handle the weights case entirely independently from the non-weights case.
+        # We'll treat the weights as our "key" draw and base all other draws on that.
+
+        # weights doesn't play well with super small floats, so exclude <.01
+        weights = draw(st.lists(st.floats(0.01, 1), min_size=1, max_size=255))
+
+        # we additionally pick a central value (if not forced), and then the index
+        # into the weights at which it can be found - aka the min-value offset.
+        center = forced if use_forced else draw(st.integers())
+        min_value = center - draw(st.integers(0, len(weights) - 1))
+        max_value = min_value + len(weights) - 1
+    else:
+        if use_min_value:
+            min_value = draw(st.integers(max_value=forced))
+        if use_max_value:
+            min_vals = []
+            if min_value is not None:
+                min_vals.append(min_value)
+            if forced is not None:
+                min_vals.append(forced)
+            min_val = max(min_vals) if min_vals else None
+            max_value = draw(st.integers(min_value=min_val))
+
+    if use_shrink_towards:
+        shrink_towards = draw(st.integers())
+
+    if forced is not None:
+        assume((forced - shrink_towards).bit_length() < 128)
+
+    return {
+        "min_value": min_value,
+        "max_value": max_value,
+        "shrink_towards": shrink_towards,
+        "weights": weights,
+        "forced": forced,
+    }
+
+
+@st.composite
+def draw_string_kwargs(draw, *, use_min_size=True, use_max_size=True, use_forced=False):
+    interval_set = draw(intervals())
+    # TODO relax this restriction once we handle empty pseudo-choices in the ir
+    assume(len(interval_set) > 0)
+    forced = (
+        draw(TextStrategy(OneCharStringStrategy(interval_set))) if use_forced else None
+    )
+
+    min_size = 0
+    max_size = None
+
+    if use_min_size:
+        # cap to some reasonable min size to avoid overruns.
+        n = 100
+        if forced is not None:
+            n = min(n, len(forced))
+
+        min_size = draw(st.integers(0, n))
+
+    if use_max_size:
+        n = min_size if forced is None else max(min_size, len(forced))
+        max_size = draw(st.integers(min_value=n))
+        # cap to some reasonable max size to avoid overruns.
+        max_size = min(max_size, min_size + 100)
+
+    return {
+        "intervals": interval_set,
+        "min_size": min_size,
+        "max_size": max_size,
+        "forced": forced,
+    }
+
+
+@st.composite
+def draw_bytes_kwargs(draw, *, use_forced=False):
+    forced = draw(st.binary()) if use_forced else None
+    # be reasonable with the number of bytes we ask for. We only have BUFFER_SIZE
+    # to work with before we overrun.
+    size = (
+        draw(st.integers(min_value=0, max_value=100)) if forced is None else len(forced)
+    )
+
+    return {"size": size, "forced": forced}
+
+
+@st.composite
+def draw_float_kwargs(
+    draw, *, use_min_value=True, use_max_value=True, use_forced=False
+):
+    forced = draw(st.floats()) if use_forced else None
+    pivot = forced if not math.isnan(forced) else None
+    min_value = -math.inf
+    max_value = math.inf
+
+    if use_min_value:
+        min_value = draw(st.floats(max_value=pivot, allow_nan=False))
+
+    if use_max_value:
+        min_val = min_value if not pivot is not None else max(min_value, pivot)
+        max_value = draw(st.floats(min_value=min_val, allow_nan=False))
+
+    return {"min_value": min_value, "max_value": max_value, "forced": forced}
+
+
+@st.composite
+def draw_boolean_kwargs(draw, *, use_forced=False):
+    forced = draw(st.booleans()) if use_forced else None
+    p = draw(st.floats(0, 1, allow_nan=False, allow_infinity=False))
+
+    # avoid invalid forced combinations
+    assume(p > 0 or forced is False)
+    assume(p < 1 or forced is True)
+
+    if 0 < p < 1:
+        # match internal assumption about avoiding large draws
+        bits = math.ceil(-math.log(min(p, 1 - p), 2))
+        assume(bits <= 64)
+
+    return {"p": p, "forced": forced}