From 31c445a1269795c5aa270cec382bca6bfc4c0885 Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Sat, 25 May 2024 16:01:26 -0400 Subject: [PATCH] add shrinker coverage tests and pragmas --- .../internal/conjecture/shrinker.py | 11 ++-- .../tests/conjecture/test_engine.py | 56 ++++++++++++++++--- 2 files changed, 55 insertions(+), 12 deletions(-) diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py b/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py index 61992505bc..9ceceb07bc 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py @@ -1014,7 +1014,7 @@ def try_shrinking_nodes(self, nodes, n): # the indices are out of bounds, give up on the replacement. # we probably want to narrow down the root cause here at some point. if any(node.index >= len(self.nodes) for node in nodes): - return + return # pragma: no cover initial_attempt = replace_all( self.nodes, @@ -1060,12 +1060,16 @@ def try_shrinking_nodes(self, nodes, n): # a collection of that size...and not much else. In practice this # helps because this antipattern is fairly common. + # TODO we'll probably want to apply the same trick as in the valid + # case of this function of preserving from the right instead of + # preserving from the left. see test_can_shrink_variable_string_draws. + node = self.nodes[len(attempt.examples.ir_tree_nodes)] (attempt_ir_type, attempt_kwargs, _attempt_forced) = attempt.invalid_at if node.ir_type != attempt_ir_type: return False if node.was_forced: - return False + return False # pragma: no cover if node.ir_type == "string": # if the size *increased*, we would have to guess what to pad with @@ -1122,9 +1126,8 @@ def try_shrinking_nodes(self, nodes, n): if ex.ir_end <= end: continue - # TODO convince myself this check is reasonable and not hiding a bug if ex.index >= len(attempt.examples): - continue + continue # pragma: no cover replacement = attempt.examples[ex.index] in_original = [c for c in ex.children if c.ir_start >= end] diff --git a/hypothesis-python/tests/conjecture/test_engine.py b/hypothesis-python/tests/conjecture/test_engine.py index dabaf03f56..8fe6c1ec24 100644 --- a/hypothesis-python/tests/conjecture/test_engine.py +++ b/hypothesis-python/tests/conjecture/test_engine.py @@ -8,6 +8,7 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. +import enum import re import time from random import Random @@ -26,7 +27,7 @@ ) from hypothesis.database import ExampleDatabase, InMemoryExampleDatabase from hypothesis.errors import FailedHealthCheck, Flaky -from hypothesis.internal.compat import int_from_bytes +from hypothesis.internal.compat import bit_count, int_from_bytes from hypothesis.internal.conjecture import engine as engine_module from hypothesis.internal.conjecture.data import ConjectureData, IRNode, Overrun, Status from hypothesis.internal.conjecture.datatree import compute_max_children @@ -460,6 +461,34 @@ def strategy(draw): assert ints == [target % 255] + [255] * (len(ints) - 1) +def test_can_shrink_variable_string_draws(): + @st.composite + def strategy(draw): + n = draw(st.integers(min_value=0, max_value=20)) + return draw(st.text(st.characters(codec="ascii"), min_size=n, max_size=n)) + + s = minimal(strategy(), lambda s: len(s) >= 10 and "a" in s) + + # this should be + # assert s == "0" * 9 + "a" + # but we first shrink to having a single a at the end of the string and then + # fail to apply our special case invalid logic when shrinking the min_size n, + # because that logic removes from the end of the string (which fails our + # precondition). + assert re.match("0+a", s) + + +def test_variable_size_string_increasing(): + # coverage test for min_size increasing during shrinking (because the test + # function inverts n). + @st.composite + def strategy(draw): + n = 10 - draw(st.integers(0, 10)) + return draw(st.text(st.characters(codec="ascii"), min_size=n, max_size=n)) + + assert minimal(strategy(), lambda s: len(s) >= 5 and "a" in s) == "0000a" + + def test_run_nothing(): def f(data): raise AssertionError @@ -1647,10 +1676,21 @@ def test(data): assert runner.call_count == 3 -def test_mildly_complicated_strategy(): - # there are some code paths in engine.py that are easily covered by any mildly - # compliated strategy and aren't worth testing explicitly for. This covers - # those. - n = 5 - s = st.lists(st.integers(), min_size=n) - assert minimal(s, lambda x: sum(x) >= 2 * n) == [0, 0, 0, 0, n * 2] +@pytest.mark.parametrize( + "strategy, condition", + [ + (st.lists(st.integers(), min_size=5), lambda v: True), + (st.lists(st.text(), min_size=2, unique=True), lambda v: True), + ( + st.sampled_from( + enum.Flag("LargeFlag", {f"bit{i}": enum.auto() for i in range(64)}) + ), + lambda f: bit_count(f.value) > 1, + ), + ], +) +def test_mildly_complicated_strategies(strategy, condition): + # There are some code paths in engine.py and shrinker.py that are easily + # covered by shrinking any mildly compliated strategy and aren't worth + # testing explicitly for. This covers those. + minimal(strategy, condition)