Merge pull request #3903 from tybug/ir-shrinker-preparation

Some changes in preparation for the shrinker ir migration

hypothesis-python/.coveragerc

@@ -20,6 +20,7 @@ exclude_lines =
     pragma: no cover
     raise NotImplementedError
     def __repr__
+    def _repr_pretty_
     def __ne__
     def __copy__
     def __deepcopy__

hypothesis-python/RELEASE.rst

@@ -0,0 +1,3 @@
+RELEASE_TYPE: patch
+
+This patch paves the way for future shrinker improvements. There is no user-visible change.

hypothesis-python/src/hypothesis/core.py

@@ -108,6 +108,7 @@
     repr_call,
 )
 from hypothesis.internal.scrutineer import (
+    MONITORING_TOOL_ID,
     Trace,
     Tracer,
     explanatory_lines,
@@ -983,8 +984,27 @@ def _execute_once_for_engine(self, data: ConjectureData) -> None:
         """
         trace: Trace = set()
         try:
+            # this is actually covered by our tests, but only on >= 3.12.
+            if (
+                sys.version_info[:2] >= (3, 12)
+                and sys.monitoring.get_tool(MONITORING_TOOL_ID) is not None
+            ):  # pragma: no cover
+                warnings.warn(
+                    "avoiding tracing test function because tool id "
+                    f"{MONITORING_TOOL_ID} is already taken by tool "
+                    f"{sys.monitoring.get_tool(MONITORING_TOOL_ID)}.",
+                    HypothesisWarning,
+                    # I'm not sure computing a correct stacklevel is reasonable
+                    # given the number of entry points here.
+                    stacklevel=1,
+                )
+
             _can_trace = (
-                sys.gettrace() is None or sys.version_info[:2] >= (3, 12)
+                (sys.version_info[:2] < (3, 12) and sys.gettrace() is None)
+                or (
+                    sys.version_info[:2] >= (3, 12)
+                    and sys.monitoring.get_tool(MONITORING_TOOL_ID) is None
+                )
             ) and not PYPY
             _trace_obs = TESTCASE_CALLBACKS and OBSERVABILITY_COLLECT_COVERAGE
             _trace_failure = (

hypothesis-python/src/hypothesis/internal/conjecture/data.py

@@ -203,6 +203,8 @@ def structural_coverage(label: int) -> StructuralCoverageTag:
 
 FLOAT_INIT_LOGIC_CACHE = LRUReusedCache(4096)
 
+POOLED_KWARGS_CACHE = LRUReusedCache(4096)
+
 DRAW_STRING_DEFAULT_MAX_SIZE = 10**10  # "arbitrarily large"
 
 
@@ -334,6 +336,7 @@ def __init__(self, examples: "Examples"):
         self.bytes_read = 0
         self.example_count = 0
         self.block_count = 0
+        self.ir_node_count = 0
 
     def run(self) -> Any:
         """Rerun the test case with this visitor and return the
@@ -347,6 +350,10 @@ def run(self) -> Any:
                 self.block(self.block_count)
                 self.block_count += 1
                 self.__pop(discarded=False)
+            elif record == IR_NODE_RECORD:
+                data = self.examples.ir_nodes[self.ir_node_count]
+                self.ir_node(data)
+                self.ir_node_count += 1
             elif record >= START_EXAMPLE_RECORD:
                 self.__push(record - START_EXAMPLE_RECORD)
             else:
@@ -387,6 +394,9 @@ def stop_example(self, i: int, *, discarded: bool) -> None:
         index of the example and ``discarded`` being ``True`` if ``stop_example``
         was called with ``discard=True``."""
 
+    def ir_node(self, node: "IRNode") -> None:
+        """Called when an ir node is drawn."""
+
     def finish(self) -> Any:
         return self.result
 
@@ -419,6 +429,8 @@ def lazy_calculate(self: "Examples") -> IntList:
 STOP_EXAMPLE_NO_DISCARD_RECORD = 2
 START_EXAMPLE_RECORD = 3
 
+IR_NODE_RECORD = calc_label_from_name("ir draw record")
+
 
 class ExampleRecord:
     """Records the series of ``start_example``, ``stop_example``, and
@@ -435,10 +447,18 @@ def __init__(self) -> None:
         self.labels = [DRAW_BYTES_LABEL]
         self.__index_of_labels: "Optional[Dict[int, int]]" = {DRAW_BYTES_LABEL: 0}
         self.trail = IntList()
+        self.ir_nodes: List[IRNode] = []
 
     def freeze(self) -> None:
         self.__index_of_labels = None
 
+    def record_ir_draw(self, ir_type, value, *, kwargs, was_forced):
+        self.trail.append(IR_NODE_RECORD)
+        node = IRNode(
+            ir_type=ir_type, value=value, kwargs=kwargs, was_forced=was_forced
+        )
+        self.ir_nodes.append(node)
+
     def start_example(self, label: int) -> None:
         assert self.__index_of_labels is not None
         try:
@@ -454,7 +474,7 @@ def stop_example(self, *, discard: bool) -> None:
         else:
             self.trail.append(STOP_EXAMPLE_NO_DISCARD_RECORD)
 
-    def draw_bits(self, n: int, forced: Optional[int]) -> None:
+    def draw_bits(self) -> None:
         self.trail.append(DRAW_BITS_RECORD)
 
 
@@ -471,6 +491,7 @@ class Examples:
 
     def __init__(self, record: ExampleRecord, blocks: "Blocks") -> None:
         self.trail = record.trail
+        self.ir_nodes = record.ir_nodes
         self.labels = record.labels
         self.__length = (
             self.trail.count(STOP_EXAMPLE_DISCARD_RECORD)
@@ -556,6 +577,15 @@ def start_example(self, i: int, label_index: int) -> None:
 
     depths: IntList = calculated_example_property(_depths)
 
+    class _ir_tree_nodes(ExampleProperty):
+        def begin(self):
+            self.result = []
+
+        def ir_node(self, ir_node):
+            self.result.append(ir_node)
+
+    ir_tree_nodes: "List[IRNode]" = calculated_example_property(_ir_tree_nodes)
+
     class _label_indices(ExampleProperty):
         def start_example(self, i: int, label_index: int) -> None:
             self.result[i] = label_index
@@ -856,31 +886,39 @@ def kill_branch(self) -> None:
         """Mark this part of the tree as not worth re-exploring."""
 
     def draw_integer(
-        self, value: int, *, was_forced: bool, kwargs: IntegerKWargs
+        self, value: int, *, kwargs: IntegerKWargs, was_forced: bool
     ) -> None:
         pass
 
     def draw_float(
-        self, value: float, *, was_forced: bool, kwargs: FloatKWargs
+        self, value: float, *, kwargs: FloatKWargs, was_forced: bool
     ) -> None:
         pass
 
     def draw_string(
-        self, value: str, *, was_forced: bool, kwargs: StringKWargs
+        self, value: str, *, kwargs: StringKWargs, was_forced: bool
     ) -> None:
         pass
 
     def draw_bytes(
-        self, value: bytes, *, was_forced: bool, kwargs: BytesKWargs
+        self, value: bytes, *, kwargs: BytesKWargs, was_forced: bool
     ) -> None:
         pass
 
     def draw_boolean(
-        self, value: bool, *, was_forced: bool, kwargs: BooleanKWargs
+        self, value: bool, *, kwargs: BooleanKWargs, was_forced: bool
     ) -> None:
         pass
 
 
+@attr.s(slots=True)
+class IRNode:
+    ir_type: IRTypeName = attr.ib()
+    value: IRType = attr.ib()
+    kwargs: IRKWargsType = attr.ib()
+    was_forced: bool = attr.ib()
+
+
 @dataclass_transform()
 @attr.s(slots=True)
 class ConjectureResult:
@@ -1142,7 +1180,9 @@ def draw_float(
                 result = self._draw_float(
                     forced_sign_bit=forced_sign_bit, forced=forced
                 )
-                if math.copysign(1.0, result) == -1:
+                if allow_nan and math.isnan(result):
+                    clamped = result
+                elif math.copysign(1.0, result) == -1:
                     assert neg_clamper is not None
                     clamped = -neg_clamper(-result)
                 else:
@@ -1568,16 +1608,22 @@ def draw_integer(
         if forced is not None and max_value is not None:
             assert forced <= max_value
 
-        kwargs: IntegerKWargs = {
-            "min_value": min_value,
-            "max_value": max_value,
-            "weights": weights,
-            "shrink_towards": shrink_towards,
-        }
+        kwargs: IntegerKWargs = self._pooled_kwargs(
+            "integer",
+            {
+                "min_value": min_value,
+                "max_value": max_value,
+                "weights": weights,
+                "shrink_towards": shrink_towards,
+            },
+        )
         value = self.provider.draw_integer(**kwargs, forced=forced)
         if observe:
             self.observer.draw_integer(
-                value, was_forced=forced is not None, kwargs=kwargs
+                value, kwargs=kwargs, was_forced=forced is not None
+            )
+            self.__example_record.record_ir_draw(
+                "integer", value, kwargs=kwargs, was_forced=forced is not None
             )
         return value
 
@@ -1605,17 +1651,23 @@ def draw_float(
                 sign_aware_lte(min_value, forced) and sign_aware_lte(forced, max_value)
             )
 
-        kwargs: FloatKWargs = {
-            "min_value": min_value,
-            "max_value": max_value,
-            "allow_nan": allow_nan,
-            "smallest_nonzero_magnitude": smallest_nonzero_magnitude,
-        }
+        kwargs: FloatKWargs = self._pooled_kwargs(
+            "float",
+            {
+                "min_value": min_value,
+                "max_value": max_value,
+                "allow_nan": allow_nan,
+                "smallest_nonzero_magnitude": smallest_nonzero_magnitude,
+            },
+        )
         value = self.provider.draw_float(**kwargs, forced=forced)
         if observe:
             self.observer.draw_float(
                 value, kwargs=kwargs, was_forced=forced is not None
             )
+            self.__example_record.record_ir_draw(
+                "float", value, kwargs=kwargs, was_forced=forced is not None
+            )
         return value
 
     def draw_string(
@@ -1629,16 +1681,22 @@ def draw_string(
     ) -> str:
         assert forced is None or min_size <= len(forced)
 
-        kwargs: StringKWargs = {
-            "intervals": intervals,
-            "min_size": min_size,
-            "max_size": max_size,
-        }
+        kwargs: StringKWargs = self._pooled_kwargs(
+            "string",
+            {
+                "intervals": intervals,
+                "min_size": min_size,
+                "max_size": max_size,
+            },
+        )
         value = self.provider.draw_string(**kwargs, forced=forced)
         if observe:
             self.observer.draw_string(
                 value, kwargs=kwargs, was_forced=forced is not None
             )
+            self.__example_record.record_ir_draw(
+                "string", value, kwargs=kwargs, was_forced=forced is not None
+            )
         return value
 
     def draw_bytes(
@@ -1652,12 +1710,15 @@ def draw_bytes(
         assert forced is None or len(forced) == size
         assert size >= 0
 
-        kwargs: BytesKWargs = {"size": size}
+        kwargs: BytesKWargs = self._pooled_kwargs("bytes", {"size": size})
         value = self.provider.draw_bytes(**kwargs, forced=forced)
         if observe:
             self.observer.draw_bytes(
                 value, kwargs=kwargs, was_forced=forced is not None
             )
+            self.__example_record.record_ir_draw(
+                "bytes", value, kwargs=kwargs, was_forced=forced is not None
+            )
         return value
 
     def draw_boolean(
@@ -1673,14 +1734,37 @@ def draw_boolean(
         if forced is False:
             assert p < (1 - 2 ** (-64))
 
-        kwargs: BooleanKWargs = {"p": p}
+        kwargs: BooleanKWargs = self._pooled_kwargs("boolean", {"p": p})
         value = self.provider.draw_boolean(**kwargs, forced=forced)
         if observe:
             self.observer.draw_boolean(
                 value, kwargs=kwargs, was_forced=forced is not None
             )
+            self.__example_record.record_ir_draw(
+                "boolean", value, kwargs=kwargs, was_forced=forced is not None
+            )
         return value
 
+    def _pooled_kwargs(self, ir_type, kwargs):
+        """Memoize common dictionary objects to reduce memory pressure."""
+        key = []
+        for k, v in kwargs.items():
+            if ir_type == "float" and k in ["min_value", "max_value"]:
+                # handle -0.0 vs 0.0, etc.
+                v = float_to_int(v)
+            elif ir_type == "integer" and k == "weights":
+                # make hashable
+                v = v if v is None else tuple(v)
+            key.append((k, v))
+
+        key = (ir_type, *sorted(key))
+
+        try:
+            return POOLED_KWARGS_CACHE[key]
+        except KeyError:
+            POOLED_KWARGS_CACHE[key] = kwargs
+            return kwargs
+
     def as_result(self) -> Union[ConjectureResult, _Overrun]:
         """Convert the result of running this test into
         either an Overrun object or a ConjectureResult."""
@@ -1901,7 +1985,7 @@ def draw_bits(self, n: int, *, forced: Optional[int] = None) -> int:
         buf = bytes(buf)
         result = int_from_bytes(buf)
 
-        self.__example_record.draw_bits(n, forced)
+        self.__example_record.draw_bits()
 
         initial = self.index
 

hypothesis-python/src/hypothesis/internal/intervalsets.py

@@ -102,6 +102,9 @@ def __and__(self, other):
     def __eq__(self, other):
         return isinstance(other, IntervalSet) and (other.intervals == self.intervals)
 
+    def __hash__(self):
+        return hash(self.intervals)
+
     def union(self, other):
         """Merge two sequences of intervals into a single tuple of intervals.