From f41f5428b2adc00fec39632d32b5ef3fddf48ae3 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Tue, 24 Aug 2021 17:00:25 +0100 Subject: [PATCH 01/24] Seperate private shape and index strategies from NumPy extra --- .../src/hypothesis/extra/__array_helpers.py | 225 ++++++++++++++++++ .../src/hypothesis/extra/numpy.py | 212 +---------------- 2 files changed, 234 insertions(+), 203 deletions(-) create mode 100644 hypothesis-python/src/hypothesis/extra/__array_helpers.py diff --git a/hypothesis-python/src/hypothesis/extra/__array_helpers.py b/hypothesis-python/src/hypothesis/extra/__array_helpers.py new file mode 100644 index 0000000000..37cfd3c5ae --- /dev/null +++ b/hypothesis-python/src/hypothesis/extra/__array_helpers.py @@ -0,0 +1,225 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Most of this work is copyright (C) 2013-2021 David R. MacIver +# (david@drmaciver.com), but it contains contributions by others. See +# CONTRIBUTING.rst for a full list of people who may hold copyright, and +# consult the git log if you need to determine who owns an individual +# contribution. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. +# +# END HEADER + +from typing import NamedTuple, Tuple, Union + +from hypothesis import assume, strategies as st +from hypothesis.internal.conjecture import utils as cu + +__all__ = [ + "Shape", + "BroadcastableShapes", + "BasicIndex", + "MutuallyBroadcastableShapesStrategy", + "BasicIndexStrategy", +] + +Shape = Tuple[int, ...] +BasicIndex = Tuple[Union[int, slice, None, "ellipsis"], ...] # noqa: F821 + + +class BroadcastableShapes(NamedTuple): + input_shapes: Tuple[Shape, ...] + result_shape: Shape + + +class MutuallyBroadcastableShapesStrategy(st.SearchStrategy): + def __init__( + self, + num_shapes, + signature=None, + base_shape=(), + min_dims=0, + max_dims=None, + min_side=1, + max_side=None, + ): + st.SearchStrategy.__init__(self) + self.base_shape = base_shape + self.side_strat = st.integers(min_side, max_side) + self.num_shapes = num_shapes + self.signature = signature + self.min_dims = min_dims + self.max_dims = max_dims + self.min_side = min_side + self.max_side = max_side + + self.size_one_allowed = self.min_side <= 1 <= self.max_side + + def do_draw(self, data): + # We don't usually have a gufunc signature; do the common case first & fast. + if self.signature is None: + return self._draw_loop_dimensions(data) + + # When we *do*, draw the core dims, then draw loop dims, and finally combine. + core_in, core_res = self._draw_core_dimensions(data) + + # If some core shape has omitted optional dimensions, it's an error to add + # loop dimensions to it. We never omit core dims if min_dims >= 1. + # This ensures that we respect Numpy's gufunc broadcasting semantics and user + # constraints without needing to check whether the loop dims will be + # interpreted as an invalid substitute for the omitted core dims. + # We may implement this check later! + use = [None not in shp for shp in core_in] + loop_in, loop_res = self._draw_loop_dimensions(data, use=use) + + def add_shape(loop, core): + return tuple(x for x in (loop + core)[-32:] if x is not None) + + return BroadcastableShapes( + input_shapes=tuple(add_shape(l_in, c) for l_in, c in zip(loop_in, core_in)), + result_shape=add_shape(loop_res, core_res), + ) + + def _draw_core_dimensions(self, data): + # Draw gufunc core dimensions, with None standing for optional dimensions + # that will not be present in the final shape. We track omitted dims so + # that we can do an accurate per-shape length cap. + dims = {} + shapes = [] + for shape in self.signature.input_shapes + (self.signature.result_shape,): + shapes.append([]) + for name in shape: + if name.isdigit(): + shapes[-1].append(int(name)) + continue + if name not in dims: + dim = name.strip("?") + dims[dim] = data.draw(self.side_strat) + if self.min_dims == 0 and not data.draw_bits(3): + dims[dim + "?"] = None + else: + dims[dim + "?"] = dims[dim] + shapes[-1].append(dims[name]) + return tuple(tuple(s) for s in shapes[:-1]), tuple(shapes[-1]) + + def _draw_loop_dimensions(self, data, use=None): + # All shapes are handled in column-major order; i.e. they are reversed + base_shape = self.base_shape[::-1] + result_shape = list(base_shape) + shapes = [[] for _ in range(self.num_shapes)] + if use is None: + use = [True for _ in range(self.num_shapes)] + else: + assert len(use) == self.num_shapes + assert all(isinstance(x, bool) for x in use) + + for dim_count in range(1, self.max_dims + 1): + dim = dim_count - 1 + + # We begin by drawing a valid dimension-size for the given + # dimension. This restricts the variability across the shapes + # at this dimension such that they can only choose between + # this size and a singleton dimension. + if len(base_shape) < dim_count or base_shape[dim] == 1: + # dim is unrestricted by the base-shape: shrink to min_side + dim_side = data.draw(self.side_strat) + elif base_shape[dim] <= self.max_side: + # dim is aligned with non-singleton base-dim + dim_side = base_shape[dim] + else: + # only a singleton is valid in alignment with the base-dim + dim_side = 1 + + for shape_id, shape in enumerate(shapes): + # Populating this dimension-size for each shape, either + # the drawn size is used or, if permitted, a singleton + # dimension. + if dim_count <= len(base_shape) and self.size_one_allowed: + # aligned: shrink towards size 1 + side = data.draw(st.sampled_from([1, dim_side])) + else: + side = dim_side + + # Use a trick where where a biased coin is queried to see + # if the given shape-tuple will continue to be grown. All + # of the relevant draws will still be made for the given + # shape-tuple even if it is no longer being added to. + # This helps to ensure more stable shrinking behavior. + if self.min_dims < dim_count: + use[shape_id] &= cu.biased_coin( + data, 1 - 1 / (1 + self.max_dims - dim) + ) + + if use[shape_id]: + shape.append(side) + if len(result_shape) < len(shape): + result_shape.append(shape[-1]) + elif shape[-1] != 1 and result_shape[dim] == 1: + result_shape[dim] = shape[-1] + if not any(use): + break + + result_shape = result_shape[: max(map(len, [self.base_shape] + shapes))] + + assert len(shapes) == self.num_shapes + assert all(self.min_dims <= len(s) <= self.max_dims for s in shapes) + assert all(self.min_side <= s <= self.max_side for side in shapes for s in side) + + return BroadcastableShapes( + input_shapes=tuple(tuple(reversed(shape)) for shape in shapes), + result_shape=tuple(reversed(result_shape)), + ) + + +class BasicIndexStrategy(st.SearchStrategy): + def __init__(self, shape, min_dims, max_dims, allow_ellipsis, allow_none): + self.shape = shape + self.min_dims = min_dims + self.max_dims = max_dims + self.allow_ellipsis = allow_ellipsis + self.allow_none = allow_none + + def do_draw(self, data): + # General plan: determine the actual selection up front with a straightforward + # approach that shrinks well, then complicate it by inserting other things. + result = [] + for dim_size in self.shape: + if dim_size == 0: + result.append(slice(None)) + continue + strategy = st.integers(-dim_size, dim_size - 1) | st.slices(dim_size) + result.append(data.draw(strategy)) + # Insert some number of new size-one dimensions if allowed + result_dims = sum(isinstance(idx, slice) for idx in result) + while ( + self.allow_none + and result_dims < self.max_dims + and (result_dims < self.min_dims or data.draw(st.booleans())) + ): + i = data.draw(st.integers(0, len(result))) + result.insert(i, None) + result_dims += 1 + # Check that we'll have the right number of dimensions; reject if not. + # It's easy to do this by construction if you don't care about shrinking, + # which is really important for array shapes. So we filter instead. + assume(self.min_dims <= result_dims <= self.max_dims) + # This is a quick-and-dirty way to insert ..., xor shorten the indexer, + # but it means we don't have to do any structural analysis. + if self.allow_ellipsis and data.draw(st.booleans()): + # Choose an index; then replace all adjacent whole-dimension slices. + i = j = data.draw(st.integers(0, len(result))) + while i > 0 and result[i - 1] == slice(None): + i -= 1 + while j < len(result) and result[j] == slice(None): + j += 1 + result[i:j] = [Ellipsis] + else: + while result[-1:] == [slice(None, None)] and data.draw(st.integers(0, 7)): + result.pop() + if len(result) == 1 and data.draw(st.booleans()): + # Sometimes generate bare element equivalent to a length-one tuple + return result[0] + return tuple(result) diff --git a/hypothesis-python/src/hypothesis/extra/numpy.py b/hypothesis-python/src/hypothesis/extra/numpy.py index c8287b6725..d841fb7373 100644 --- a/hypothesis-python/src/hypothesis/extra/numpy.py +++ b/hypothesis-python/src/hypothesis/extra/numpy.py @@ -19,8 +19,15 @@ import numpy as np -from hypothesis import assume, strategies as st +from hypothesis import strategies as st from hypothesis.errors import InvalidArgument +from hypothesis.extra.__array_helpers import ( + BasicIndex, + BasicIndexStrategy, + BroadcastableShapes, + MutuallyBroadcastableShapesStrategy, + Shape, +) from hypothesis.internal.conjecture import utils as cu from hypothesis.internal.coverage import check_function from hypothesis.internal.reflection import proxies @@ -29,17 +36,9 @@ from hypothesis.strategies._internal.utils import defines_strategy from hypothesis.utils.conventions import UniqueIdentifier, not_set -Shape = Tuple[int, ...] -# flake8 and mypy disagree about `ellipsis` (the type of `...`), and hence: -BasicIndex = Tuple[Union[int, slice, "ellipsis", np.newaxis], ...] # noqa: F821 TIME_RESOLUTIONS = tuple("Y M D h m s ms us ns ps fs as".split()) -class BroadcastableShapes(NamedTuple): - input_shapes: Tuple[Shape, ...] - result_shape: Shape - - @defines_strategy(force_reusable_values=True) def from_dtype( dtype: np.dtype, @@ -908,147 +907,6 @@ def broadcastable_shapes( ).map(lambda x: x.input_shapes[0]) -class MutuallyBroadcastableShapesStrategy(st.SearchStrategy): - def __init__( - self, - num_shapes, - signature=None, - base_shape=(), - min_dims=0, - max_dims=None, - min_side=1, - max_side=None, - ): - assert 0 <= min_side <= max_side - assert 0 <= min_dims <= max_dims <= 32 - st.SearchStrategy.__init__(self) - self.base_shape = base_shape - self.side_strat = st.integers(min_side, max_side) - self.num_shapes = num_shapes - self.signature = signature - self.min_dims = min_dims - self.max_dims = max_dims - self.min_side = min_side - self.max_side = max_side - - self.size_one_allowed = self.min_side <= 1 <= self.max_side - - def do_draw(self, data): - # We don't usually have a gufunc signature; do the common case first & fast. - if self.signature is None: - return self._draw_loop_dimensions(data) - - # When we *do*, draw the core dims, then draw loop dims, and finally combine. - core_in, core_res = self._draw_core_dimensions(data) - - # If some core shape has omitted optional dimensions, it's an error to add - # loop dimensions to it. We never omit core dims if min_dims >= 1. - # This ensures that we respect Numpy's gufunc broadcasting semantics and user - # constraints without needing to check whether the loop dims will be - # interpreted as an invalid substitute for the omitted core dims. - # We may implement this check later! - use = [None not in shp for shp in core_in] - loop_in, loop_res = self._draw_loop_dimensions(data, use=use) - - def add_shape(loop, core): - return tuple(x for x in (loop + core)[-32:] if x is not None) - - return BroadcastableShapes( - input_shapes=tuple(add_shape(l_in, c) for l_in, c in zip(loop_in, core_in)), - result_shape=add_shape(loop_res, core_res), - ) - - def _draw_core_dimensions(self, data): - # Draw gufunc core dimensions, with None standing for optional dimensions - # that will not be present in the final shape. We track omitted dims so - # that we can do an accurate per-shape length cap. - dims = {} - shapes = [] - for shape in self.signature.input_shapes + (self.signature.result_shape,): - shapes.append([]) - for name in shape: - if name.isdigit(): - shapes[-1].append(int(name)) - continue - if name not in dims: - dim = name.strip("?") - dims[dim] = data.draw(self.side_strat) - if self.min_dims == 0 and not data.draw_bits(3): - dims[dim + "?"] = None - else: - dims[dim + "?"] = dims[dim] - shapes[-1].append(dims[name]) - return tuple(tuple(s) for s in shapes[:-1]), tuple(shapes[-1]) - - def _draw_loop_dimensions(self, data, use=None): - # All shapes are handled in column-major order; i.e. they are reversed - base_shape = self.base_shape[::-1] - result_shape = list(base_shape) - shapes = [[] for _ in range(self.num_shapes)] - if use is None: - use = [True for _ in range(self.num_shapes)] - else: - assert len(use) == self.num_shapes - assert all(isinstance(x, bool) for x in use) - - for dim_count in range(1, self.max_dims + 1): - dim = dim_count - 1 - - # We begin by drawing a valid dimension-size for the given - # dimension. This restricts the variability across the shapes - # at this dimension such that they can only choose between - # this size and a singleton dimension. - if len(base_shape) < dim_count or base_shape[dim] == 1: - # dim is unrestricted by the base-shape: shrink to min_side - dim_side = data.draw(self.side_strat) - elif base_shape[dim] <= self.max_side: - # dim is aligned with non-singleton base-dim - dim_side = base_shape[dim] - else: - # only a singleton is valid in alignment with the base-dim - dim_side = 1 - - for shape_id, shape in enumerate(shapes): - # Populating this dimension-size for each shape, either - # the drawn size is used or, if permitted, a singleton - # dimension. - if dim_count <= len(base_shape) and self.size_one_allowed: - # aligned: shrink towards size 1 - side = data.draw(st.sampled_from([1, dim_side])) - else: - side = dim_side - - # Use a trick where where a biased coin is queried to see - # if the given shape-tuple will continue to be grown. All - # of the relevant draws will still be made for the given - # shape-tuple even if it is no longer being added to. - # This helps to ensure more stable shrinking behavior. - if self.min_dims < dim_count: - use[shape_id] &= cu.biased_coin( - data, 1 - 1 / (1 + self.max_dims - dim) - ) - - if use[shape_id]: - shape.append(side) - if len(result_shape) < len(shape): - result_shape.append(shape[-1]) - elif shape[-1] != 1 and result_shape[dim] == 1: - result_shape[dim] = shape[-1] - if not any(use): - break - - result_shape = result_shape[: max(map(len, [self.base_shape] + shapes))] - - assert len(shapes) == self.num_shapes - assert all(self.min_dims <= len(s) <= self.max_dims for s in shapes) - assert all(self.min_side <= s <= self.max_side for side in shapes for s in side) - - return BroadcastableShapes( - input_shapes=tuple(tuple(reversed(shape)) for shape in shapes), - result_shape=tuple(reversed(result_shape)), - ) - - # See https://numpy.org/doc/stable/reference/c-api/generalized-ufuncs.html # Implementation based on numpy.lib.function_base._parse_gufunc_signature # with minor upgrades to handle numeric and optional dimensions. Examples: @@ -1308,58 +1166,6 @@ def mutually_broadcastable_shapes( ) -class BasicIndexStrategy(st.SearchStrategy): - def __init__(self, shape, min_dims, max_dims, allow_ellipsis, allow_newaxis): - assert 0 <= min_dims <= max_dims <= 32 - st.SearchStrategy.__init__(self) - self.shape = shape - self.min_dims = min_dims - self.max_dims = max_dims - self.allow_ellipsis = allow_ellipsis - self.allow_newaxis = allow_newaxis - - def do_draw(self, data): - # General plan: determine the actual selection up front with a straightforward - # approach that shrinks well, then complicate it by inserting other things. - result = [] - for dim_size in self.shape: - if dim_size == 0: - result.append(slice(None)) - continue - strategy = st.integers(-dim_size, dim_size - 1) | st.slices(dim_size) - result.append(data.draw(strategy)) - # Insert some number of new size-one dimensions if allowed - result_dims = sum(isinstance(idx, slice) for idx in result) - while ( - self.allow_newaxis - and result_dims < self.max_dims - and (result_dims < self.min_dims or data.draw(st.booleans())) - ): - result.insert(data.draw(st.integers(0, len(result))), np.newaxis) - result_dims += 1 - # Check that we'll have the right number of dimensions; reject if not. - # It's easy to do this by construction iff you don't care about shrinking, - # which is really important for array shapes. So we filter instead. - assume(self.min_dims <= result_dims <= self.max_dims) - # This is a quick-and-dirty way to insert ..., xor shorten the indexer, - # but it means we don't have to do any structural analysis. - if self.allow_ellipsis and data.draw(st.booleans()): - # Choose an index; then replace all adjacent whole-dimension slices. - i = j = data.draw(st.integers(0, len(result))) - while i > 0 and result[i - 1] == slice(None): - i -= 1 - while j < len(result) and result[j] == slice(None): - j += 1 - result[i:j] = [Ellipsis] - else: - while result[-1:] == [slice(None, None)] and data.draw(st.integers(0, 7)): - result.pop() - if len(result) == 1 and data.draw(st.booleans()): - # Sometimes generate bare element equivalent to a length-one tuple - return result[0] - return tuple(result) - - @defines_strategy() def basic_indices( shape: Shape, @@ -1420,7 +1226,7 @@ def basic_indices( min_dims=min_dims, max_dims=max_dims, allow_ellipsis=allow_ellipsis, - allow_newaxis=allow_newaxis, + allow_none=allow_newaxis, ) From d437b054cada292479b6ede01eca2e97f5258018 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Wed, 25 Aug 2021 11:19:26 +0100 Subject: [PATCH 02/24] Comment np.newaxis is None in BasicIndexStrategy Co-authored-by: Zac Hatfield-Dodds --- hypothesis-python/src/hypothesis/extra/__array_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hypothesis-python/src/hypothesis/extra/__array_helpers.py b/hypothesis-python/src/hypothesis/extra/__array_helpers.py index 37cfd3c5ae..da2617668d 100644 --- a/hypothesis-python/src/hypothesis/extra/__array_helpers.py +++ b/hypothesis-python/src/hypothesis/extra/__array_helpers.py @@ -200,7 +200,7 @@ def do_draw(self, data): and (result_dims < self.min_dims or data.draw(st.booleans())) ): i = data.draw(st.integers(0, len(result))) - result.insert(i, None) + result.insert(i, None) # Note that `np.newaxis is None` result_dims += 1 # Check that we'll have the right number of dimensions; reject if not. # It's easy to do this by construction if you don't care about shrinking, From d3ce1cb6de69c12110d59b6ee1a7f8f13cdbc8d9 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Wed, 25 Aug 2021 11:20:34 +0100 Subject: [PATCH 03/24] Carry over ellipsis comment into __array_helpers.py Co-authored-by: Zac Hatfield-Dodds --- hypothesis-python/src/hypothesis/extra/__array_helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hypothesis-python/src/hypothesis/extra/__array_helpers.py b/hypothesis-python/src/hypothesis/extra/__array_helpers.py index da2617668d..33fd350a93 100644 --- a/hypothesis-python/src/hypothesis/extra/__array_helpers.py +++ b/hypothesis-python/src/hypothesis/extra/__array_helpers.py @@ -27,6 +27,7 @@ ] Shape = Tuple[int, ...] +# We silence flake8 here because it disagrees with mypy about `ellipsis` (`type(...)`) BasicIndex = Tuple[Union[int, slice, None, "ellipsis"], ...] # noqa: F821 From 3c3754fabd11cdcdde2ea38b88e6289bdd17cda6 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Wed, 25 Aug 2021 19:22:13 +0100 Subject: [PATCH 04/24] All non-NumPy dependent array strategies in __array_helpers.py Boundary checks were temporarily removed --- .../src/hypothesis/extra/__array_helpers.py | 364 +++++++++++++- .../src/hypothesis/extra/numpy.py | 452 +++--------------- .../tests/numpy/test_gen_data.py | 6 +- 3 files changed, 435 insertions(+), 387 deletions(-) diff --git a/hypothesis-python/src/hypothesis/extra/__array_helpers.py b/hypothesis-python/src/hypothesis/extra/__array_helpers.py index 33fd350a93..f9eecbd7c6 100644 --- a/hypothesis-python/src/hypothesis/extra/__array_helpers.py +++ b/hypothesis-python/src/hypothesis/extra/__array_helpers.py @@ -13,19 +13,28 @@ # # END HEADER -from typing import NamedTuple, Tuple, Union +from typing import NamedTuple, Optional, Tuple, Union from hypothesis import assume, strategies as st +from hypothesis.errors import InvalidArgument from hypothesis.internal.conjecture import utils as cu +from hypothesis.internal.coverage import check_function +from hypothesis.internal.validation import check_type, check_valid_interval +from hypothesis.strategies._internal.utils import defines_strategy __all__ = [ "Shape", "BroadcastableShapes", "BasicIndex", - "MutuallyBroadcastableShapesStrategy", - "BasicIndexStrategy", + "order_check", + "array_shapes", + "valid_tuple_axes", + "broadcastable_shapes", + "mutually_broadcastable_shapes", + "basic_indices", ] + Shape = Tuple[int, ...] # We silence flake8 here because it disagrees with mypy about `ellipsis` (`type(...)`) BasicIndex = Tuple[Union[int, slice, None, "ellipsis"], ...] # noqa: F821 @@ -36,6 +45,93 @@ class BroadcastableShapes(NamedTuple): result_shape: Shape +@check_function +def order_check(name, floor, min_, max_): + if floor > min_: + raise InvalidArgument(f"min_{name} must be at least {floor} but was {min_}") + if min_ > max_: + raise InvalidArgument(f"min_{name}={min_} is larger than max_{name}={max_}") + + +@defines_strategy() +def array_shapes( + *, + min_dims: int = 1, + max_dims: Optional[int] = None, + min_side: int = 1, + max_side: Optional[int] = None, +) -> st.SearchStrategy[Shape]: + """Return a strategy for array shapes (tuples of int >= 1). + + * ``min_dims`` is the smallest length that the generated shape can possess. + * ``max_dims`` is the largest length that the generated shape can possess, + defaulting to ``min_dims + 2``. + * ``min_side`` is the smallest size that a dimension can possess. + * ``max_side`` is the largest size that a dimension can possess, + defaulting to ``min_side + 5``. + """ + check_type(int, min_dims, "min_dims") + check_type(int, min_side, "min_side") + + if max_dims is None: + max_dims = min_dims + 2 + check_type(int, max_dims, "max_dims") + + if max_side is None: + max_side = min_side + 5 + check_type(int, max_side, "max_side") + + order_check("dims", 0, min_dims, max_dims) + order_check("side", 0, min_side, max_side) + + return st.lists( + st.integers(min_side, max_side), min_size=min_dims, max_size=max_dims + ).map(tuple) + + +@defines_strategy() +def valid_tuple_axes( + ndim: int, + *, + min_size: int = 0, + max_size: Optional[int] = None, +) -> st.SearchStrategy[Tuple[int, ...]]: + """All tuples will have a length >= ``min_size`` and <= ``max_size``. The default + value for ``max_size`` is ``ndim``. + + Examples from this strategy shrink towards an empty tuple, which render most + sequential functions as no-ops. + + The following are some examples drawn from this strategy. + + .. code-block:: pycon + + >>> [valid_tuple_axes(3).example() for i in range(4)] + [(-3, 1), (0, 1, -1), (0, 2), (0, -2, 2)] + + ``valid_tuple_axes`` can be joined with other strategies to generate + any type of valid axis object, i.e. integers, tuples, and ``None``: + + .. code-block:: python + + any_axis_strategy = none() | integers(-ndim, ndim - 1) | valid_tuple_axes(ndim) + + """ + if max_size is None: + max_size = ndim + check_type(int, ndim, "ndim") + check_type(int, min_size, "min_size") + check_type(int, max_size, "max_size") + order_check("size", 0, min_size, max_size) + check_valid_interval(max_size, ndim, "max_size", "ndim") + axes = st.integers(0, max(0, 2 * ndim - 1)).map( + lambda x: x if x < ndim else x - 2 * ndim + ) + return st.lists( + axes, min_size=min_size, max_size=max_size, unique_by=lambda x: x % ndim + ).map(tuple) + + class MutuallyBroadcastableShapesStrategy(st.SearchStrategy): def __init__( self, @@ -175,13 +271,219 @@ def _draw_loop_dimensions(self, data, use=None): ) +@defines_strategy() +def broadcastable_shapes( + shape: Shape, + *, + min_dims: int = 0, + max_dims: Optional[int] = None, + min_side: int = 1, + max_side: Optional[int] = None, +) -> st.SearchStrategy[Shape]: + """Return a strategy for shapes that are broadcast-compatible with the + provided shape. + + Examples from this strategy shrink towards a shape with length ``min_dims``. + The size of an aligned dimension shrinks towards size ``1``. The size of an + unaligned dimension shrink towards ``min_side``. + + * ``shape`` is a tuple of integers. + * ``min_dims`` is the smallest length that the generated shape can possess. + * ``max_dims`` is the largest length that the generated shape can possess, + defaulting to ``min(32, max(len(shape), min_dims) + 2)``. + * ``min_side`` is the smallest size that an unaligned dimension can possess. + * ``max_side`` is the largest size that an unaligned dimension can possess, + defaulting to 2 plus the size of the largest aligned dimension. + + The following are some examples drawn from this strategy. + + .. code-block:: pycon + + >>> [broadcastable_shapes(shape=(2, 3)).example() for i in range(5)] + [(1, 3), (), (2, 3), (2, 1), (4, 1, 3), (3, )] + """ + check_type(tuple, shape, "shape") + check_type(int, min_side, "min_side") + check_type(int, min_dims, "min_dims") + + strict_check = max_side is None or max_dims is None + + if max_dims is None: + max_dims = min(32, max(len(shape), min_dims) + 2) + check_type(int, max_dims, "max_dims") + + if max_side is None: + max_side = max(shape[-max_dims:] + (min_side,)) + 2 + check_type(int, max_side, "max_side") + + order_check("dims", 0, min_dims, max_dims) + order_check("side", 0, min_side, max_side) + + if strict_check: + dims = max_dims + bound_name = "max_dims" + else: + dims = min_dims + bound_name = "min_dims" + + # check for unsatisfiable min_side + if not all(min_side <= s for s in shape[::-1][:dims] if s != 1): + raise InvalidArgument( + f"Given shape={shape}, there are no broadcast-compatible " + f"shapes that satisfy: {bound_name}={dims} and min_side={min_side}" + ) + + # check for unsatisfiable [min_side, max_side] + if not ( + min_side <= 1 <= max_side or all(s <= max_side for s in shape[::-1][:dims]) + ): + raise InvalidArgument( + f"Given base_shape={shape}, there are no broadcast-compatible " + f"shapes that satisfy all of {bound_name}={dims}, " + f"min_side={min_side}, and max_side={max_side}" + ) + + if not strict_check: + # reduce max_dims to exclude unsatisfiable dimensions + for n, s in zip(range(max_dims), shape[::-1]): + if s < min_side and s != 1: + max_dims = n + break + elif not (min_side <= 1 <= max_side or s <= max_side): + max_dims = n + break + + return MutuallyBroadcastableShapesStrategy( + num_shapes=1, + base_shape=shape, + min_dims=min_dims, + max_dims=max_dims, + min_side=min_side, + max_side=max_side, + ).map(lambda x: x.input_shapes[0]) + + +@defines_strategy() +def mutually_broadcastable_shapes( + num_shapes: int, + *, + signature: Optional[str] = None, + base_shape: Shape = (), + min_dims: int = 0, + max_dims: Optional[int] = None, + min_side: int = 1, + max_side: Optional[int] = None, +) -> st.SearchStrategy[BroadcastableShapes]: + """Return a strategy for a specified number of shapes N that are + mutually-broadcastable with one another and with the provided base shape. + + * ``num_shapes`` is the number of mutually broadcast-compatible shapes to generate. + * ``base_shape`` is the shape against which all generated shapes can broadcast. + The default shape is empty, which corresponds to a scalar and thus does + not constrain broadcasting at all. + * ``shape`` is a tuple of integers. + * ``min_dims`` is the smallest length that the generated shape can possess. + * ``max_dims`` is the largest length that the generated shape can possess, + defaulting to ``min(32, max(len(shape), min_dims) + 2)``. + * ``min_side`` is the smallest size that an unaligned dimension can possess. + * ``max_side`` is the largest size that an unaligned dimension can possess, + defaulting to 2 plus the size of the largest aligned dimension. + + The strategy will generate a :obj:`python:typing.NamedTuple` containing: + + * ``input_shapes`` as a tuple of the N generated shapes. + * ``result_shape`` as the resulting shape produced by broadcasting the N shapes + with the base shape. + + The following are some examples drawn from this strategy. + + .. code-block:: pycon + + >>> # Draw three shapes where each shape is broadcast-compatible with (2, 3) + ... strat = mutually_broadcastable_shapes(num_shapes=3, base_shape=(2, 3)) + >>> for _ in range(5): + ... print(strat.example()) + BroadcastableShapes(input_shapes=((4, 1, 3), (4, 2, 3), ()), result_shape=(4, 2, 3)) + BroadcastableShapes(input_shapes=((3,), (1, 3), (2, 3)), result_shape=(2, 3)) + BroadcastableShapes(input_shapes=((), (), ()), result_shape=(2, 3)) + BroadcastableShapes(input_shapes=((3,), (), (3,)), result_shape=(2, 3)) + BroadcastableShapes(input_shapes=((1, 2, 3), (3,), ()), result_shape=(1, 2, 3)) + + """ + + check_type(int, num_shapes, "num_shapes") + if num_shapes < 1: + raise InvalidArgument(f"num_shapes={num_shapes} must be at least 1") + + check_type(tuple, base_shape, "base_shape") + check_type(int, min_side, "min_side") + check_type(int, min_dims, "min_dims") + + strict_check = max_dims is not None + + if max_dims is None: + max_dims = min(32, max(len(base_shape), min_dims) + 2) + check_type(int, max_dims, "max_dims") + + if max_side is None: + max_side = max(base_shape[-max_dims:] + (min_side,)) + 2 + check_type(int, max_side, "max_side") + + order_check("dims", 0, min_dims, max_dims) + order_check("side", 0, min_side, max_side) + + if strict_check: + dims = max_dims + bound_name = "max_dims" + else: + dims = min_dims + bound_name = "min_dims" + + # check for unsatisfiable min_side + if not all(min_side <= s for s in base_shape[::-1][:dims] if s != 1): + raise InvalidArgument( + f"Given base_shape={base_shape}, there are no broadcast-compatible " + f"shapes that satisfy: {bound_name}={dims} and min_side={min_side}" + ) + + # check for unsatisfiable [min_side, max_side] + if not ( + min_side <= 1 <= max_side or all(s <= max_side for s in base_shape[::-1][:dims]) + ): + raise InvalidArgument( + f"Given base_shape={base_shape}, there are no broadcast-compatible " + f"shapes that satisfy all of {bound_name}={dims}, " + f"min_side={min_side}, and max_side={max_side}" + ) + + if not strict_check: + # reduce max_dims to exclude unsatisfiable dimensions + for n, s in zip(range(max_dims), base_shape[::-1]): + if s < min_side and s != 1: + max_dims = n + break + elif not (min_side <= 1 <= max_side or s <= max_side): + max_dims = n + break + + return MutuallyBroadcastableShapesStrategy( + num_shapes=num_shapes, + signature=signature, + base_shape=base_shape, + min_dims=min_dims, + max_dims=max_dims, + min_side=min_side, + max_side=max_side, + ) + + class BasicIndexStrategy(st.SearchStrategy): - def __init__(self, shape, min_dims, max_dims, allow_ellipsis, allow_none): + def __init__(self, shape, min_dims, max_dims, allow_ellipsis, allow_newaxis): self.shape = shape self.min_dims = min_dims self.max_dims = max_dims self.allow_ellipsis = allow_ellipsis - self.allow_none = allow_none + self.allow_newaxis = allow_newaxis def do_draw(self, data): # General plan: determine the actual selection up front with a straightforward @@ -196,7 +498,7 @@ def do_draw(self, data): # Insert some number of new size-one dimensions if allowed result_dims = sum(isinstance(idx, slice) for idx in result) while ( - self.allow_none + self.allow_newaxis and result_dims < self.max_dims and (result_dims < self.min_dims or data.draw(st.booleans())) ): @@ -224,3 +526,53 @@ def do_draw(self, data): # Sometimes generate bare element equivalent to a length-one tuple return result[0] return tuple(result) + + +@defines_strategy() +def basic_indices( + shape: Shape, + *, + min_dims: int = 0, + max_dims: Optional[int] = None, + allow_newaxis: bool = False, + allow_ellipsis: bool = True, +) -> st.SearchStrategy[BasicIndex]: + """It generates tuples containing some mix of integers, :obj:`python:slice` + objects, ``...`` (an ``Ellipsis``), and ``None``. When a length-one tuple + would be generated, this strategy may instead return the element which will + index the first axis, e.g. ``5`` instead of ``(5,)``. + + * ``shape`` is the shape of the array that will be indexed, as a tuple of + integers >= 0. This must be at least two-dimensional for a tuple to be a + valid index; for one-dimensional arrays use + :func:`~hypothesis.strategies.slices` instead. + * ``min_dims`` is the minimum dimensionality of the resulting array from use of + the generated index. + * ``max_dims`` is the the maximum dimensionality of the resulting array, + defaulting to ``max(len(shape), min_dims) + 2``. + * ``allow_newaxis`` specifies whether ``None`` is allowed in the index. + * ``allow_ellipsis`` specifies whether ``...`` is allowed in the index. + """ + # Arguments to exclude scalars, zero-dim arrays, and dims of size zero were + # all considered and rejected. We want users to explicitly consider those + # cases if they're dealing in general indexers, and while it's fiddly we can + # back-compatibly add them later (hence using kwonlyargs). + check_type(tuple, shape, "shape") + check_type(bool, allow_ellipsis, "allow_ellipsis") + check_type(bool, allow_newaxis, "allow_newaxis") + check_type(int, min_dims, "min_dims") + if max_dims is None: + max_dims = min(max(len(shape), min_dims) + 2, 32) + check_type(int, max_dims, "max_dims") + order_check("dims", 0, min_dims, max_dims) + if not all(isinstance(x, int) and x >= 0 for x in shape): + raise InvalidArgument( + f"shape={shape!r}, but all dimensions must be of integer size >= 0" + ) + return BasicIndexStrategy( + shape, + min_dims=min_dims, + max_dims=max_dims, + allow_ellipsis=allow_ellipsis, + allow_newaxis=allow_newaxis, + ) diff --git a/hypothesis-python/src/hypothesis/extra/numpy.py b/hypothesis-python/src/hypothesis/extra/numpy.py index d841fb7373..c886859a61 100644 --- a/hypothesis-python/src/hypothesis/extra/numpy.py +++ b/hypothesis-python/src/hypothesis/extra/numpy.py @@ -22,20 +22,47 @@ from hypothesis import strategies as st from hypothesis.errors import InvalidArgument from hypothesis.extra.__array_helpers import ( - BasicIndex, - BasicIndexStrategy, BroadcastableShapes, - MutuallyBroadcastableShapesStrategy, Shape, + array_shapes, + basic_indices, + broadcastable_shapes, + mutually_broadcastable_shapes as _mutually_broadcastable_shapes, + order_check, + valid_tuple_axes, ) from hypothesis.internal.conjecture import utils as cu from hypothesis.internal.coverage import check_function from hypothesis.internal.reflection import proxies -from hypothesis.internal.validation import check_type, check_valid_interval +from hypothesis.internal.validation import check_type from hypothesis.strategies._internal.strategies import T, check_strategy from hypothesis.strategies._internal.utils import defines_strategy from hypothesis.utils.conventions import UniqueIdentifier, not_set +__all__ = [ + "from_dtype", + "arrays", + "array_shapes", + "scalar_dtypes", + "boolean_dtypes", + "unsigned_integer_dtypes", + "integer_dtypes", + "floating_dtypes", + "complex_number_dtypes", + "datetime64_dtypes", + "timedelta64_dtypes", + "byte_string_dtypes", + "unicode_string_dtypes", + "array_dtypes", + "nested_dtypes", + "valid_tuple_axes", + "valid_tuple_axes", + "broadcastable_shapes", + "mutually_broadcastable_shapes", + "basic_indices", + "integer_array_indices", +] + TIME_RESOLUTIONS = tuple("Y M D h m s ms us ns ps fs as".split()) @@ -153,24 +180,6 @@ def check_argument(condition, fail_message, *f_args, **f_kwargs): raise InvalidArgument(fail_message.format(*f_args, **f_kwargs)) -@check_function -def order_check(name, floor, small, large): - check_argument( - floor <= small, - "min_{name} must be at least {} but was {}", - floor, - small, - name=name, - ) - check_argument( - small <= large, - "min_{name}={} is larger than max_{name}={}", - small, - large, - name=name, - ) - - class ArrayStrategy(st.SearchStrategy): def __init__(self, element_strategy, shape, dtype, fill, unique): self.shape = tuple(shape) @@ -460,41 +469,6 @@ def arrays( return ArrayStrategy(elements, shape, dtype, fill, unique) -@defines_strategy() -def array_shapes( - *, - min_dims: int = 1, - max_dims: Optional[int] = None, - min_side: int = 1, - max_side: Optional[int] = None, -) -> st.SearchStrategy[Shape]: - """Return a strategy for array shapes (tuples of int >= 1).""" - check_type(int, min_dims, "min_dims") - check_type(int, min_side, "min_side") - if min_dims > 32: - raise InvalidArgument( - "Got min_dims=%r, but numpy does not support arrays greater than 32 dimensions" - % min_dims - ) - if max_dims is None: - max_dims = min(min_dims + 2, 32) - check_type(int, max_dims, "max_dims") - if max_dims > 32: - raise InvalidArgument( - "Got max_dims=%r, but numpy does not support arrays greater than 32 dimensions" - % max_dims - ) - if max_side is None: - max_side = min_side + 5 - check_type(int, max_side, "max_side") - order_check("dims", 0, min_dims, max_dims) - order_check("side", 0, min_side, max_side) - - return st.lists( - st.integers(min_side, max_side), min_size=min_dims, max_size=max_dims - ).map(tuple) - - @defines_strategy() def scalar_dtypes() -> st.SearchStrategy[np.dtype]: """Return a strategy that can return any non-flexible scalar dtype.""" @@ -763,149 +737,14 @@ def nested_dtypes( ).filter(lambda d: max_itemsize is None or d.itemsize <= max_itemsize) -@defines_strategy() -def valid_tuple_axes( - ndim: int, - *, - min_size: int = 0, - max_size: Optional[int] = None, -) -> st.SearchStrategy[Shape]: - """Return a strategy for generating permissible tuple-values for the +valid_tuple_axes.__doc__ = f""" + Return a strategy for generating permissible tuple-values for the ``axis`` argument for a numpy sequential function (e.g. :func:`numpy:numpy.sum`), given an array of the specified dimensionality. - All tuples will have an length >= min_size and <= max_size. The default - value for max_size is ``ndim``. - - Examples from this strategy shrink towards an empty tuple, which render - most sequential functions as no-ops. - - The following are some examples drawn from this strategy. - - .. code-block:: pycon - - >>> [valid_tuple_axes(3).example() for i in range(4)] - [(-3, 1), (0, 1, -1), (0, 2), (0, -2, 2)] - - ``valid_tuple_axes`` can be joined with other strategies to generate - any type of valid axis object, i.e. integers, tuples, and ``None``: - - .. code-block:: pycon - - any_axis_strategy = none() | integers(-ndim, ndim - 1) | valid_tuple_axes(ndim) - - """ - if max_size is None: - max_size = ndim - - check_type(int, ndim, "ndim") - check_type(int, min_size, "min_size") - check_type(int, max_size, "max_size") - order_check("size", 0, min_size, max_size) - check_valid_interval(max_size, ndim, "max_size", "ndim") - - # shrink axis values from negative to positive - axes = st.integers(0, max(0, 2 * ndim - 1)).map( - lambda x: x if x < ndim else x - 2 * ndim - ) - return st.lists( - axes, min_size=min_size, max_size=max_size, unique_by=lambda x: x % ndim - ).map(tuple) - - -@defines_strategy() -def broadcastable_shapes( - shape: Shape, - *, - min_dims: int = 0, - max_dims: Optional[int] = None, - min_side: int = 1, - max_side: Optional[int] = None, -) -> st.SearchStrategy[Shape]: - """Return a strategy for generating shapes that are broadcast-compatible - with the provided shape. - - Examples from this strategy shrink towards a shape with length ``min_dims``. - The size of an aligned dimension shrinks towards size ``1``. The - size of an unaligned dimension shrink towards ``min_side``. - - * ``shape`` a tuple of integers - * ``min_dims`` The smallest length that the generated shape can possess. - * ``max_dims`` The largest length that the generated shape can possess. - The default-value for ``max_dims`` is ``min(32, max(len(shape), min_dims) + 2)``. - * ``min_side`` The smallest size that an unaligned dimension can possess. - * ``max_side`` The largest size that an unaligned dimension can possess. - The default value is 2 + 'size-of-largest-aligned-dimension'. - - The following are some examples drawn from this strategy. - - .. code-block:: pycon - - >>> [broadcastable_shapes(shape=(2, 3)).example() for i in range(5)] - [(1, 3), (), (2, 3), (2, 1), (4, 1, 3), (3, )] - + {valid_tuple_axes.__doc__} """ - check_type(tuple, shape, "shape") - strict_check = max_side is None or max_dims is None - check_type(int, min_side, "min_side") - check_type(int, min_dims, "min_dims") - - if max_dims is None: - max_dims = min(32, max(len(shape), min_dims) + 2) - else: - check_type(int, max_dims, "max_dims") - - if max_side is None: - max_side = max(tuple(shape[-max_dims:]) + (min_side,)) + 2 - else: - check_type(int, max_side, "max_side") - - order_check("dims", 0, min_dims, max_dims) - order_check("side", 0, min_side, max_side) - - if 32 < max_dims: - raise InvalidArgument("max_dims cannot exceed 32") - - dims, bnd_name = (max_dims, "max_dims") if strict_check else (min_dims, "min_dims") - - # check for unsatisfiable min_side - if not all(min_side <= s for s in shape[::-1][:dims] if s != 1): - raise InvalidArgument( - "Given shape=%r, there are no broadcast-compatible " - "shapes that satisfy: %s=%s and min_side=%s" - % (shape, bnd_name, dims, min_side) - ) - - # check for unsatisfiable [min_side, max_side] - if not ( - min_side <= 1 <= max_side or all(s <= max_side for s in shape[::-1][:dims]) - ): - raise InvalidArgument( - "Given shape=%r, there are no broadcast-compatible shapes " - "that satisfy: %s=%s and [min_side=%s, max_side=%s]" - % (shape, bnd_name, dims, min_side, max_side) - ) - - if not strict_check: - # reduce max_dims to exclude unsatisfiable dimensions - for n, s in zip(range(max_dims), reversed(shape)): - if s < min_side and s != 1: - max_dims = n - break - elif not (min_side <= 1 <= max_side or s <= max_side): - max_dims = n - break - - return MutuallyBroadcastableShapesStrategy( - num_shapes=1, - base_shape=shape, - min_dims=min_dims, - max_dims=max_dims, - min_side=min_side, - max_side=max_side, - ).map(lambda x: x.input_shapes[0]) - # See https://numpy.org/doc/stable/reference/c-api/generalized-ufuncs.html # Implementation based on numpy.lib.function_base._parse_gufunc_signature @@ -1003,44 +842,42 @@ def mutually_broadcastable_shapes( min_side: int = 1, max_side: Optional[int] = None, ) -> st.SearchStrategy[BroadcastableShapes]: - """Return a strategy for generating a specified number of shapes, N, that are - mutually-broadcastable with one another and with the provided "base-shape". - - The strategy will generate a named-tuple of: - - * input_shapes: the N generated shapes - * result_shape: the resulting shape, produced by broadcasting the - N shapes with the base-shape - - Each shape produced from this strategy shrinks towards a shape with length - ``min_dims``. The size of an aligned dimension shrinks towards being having - a size of 1. The size of an unaligned dimension shrink towards ``min_side``. - - * ``num_shapes`` The number of mutually broadcast-compatible shapes to generate. - * ``base-shape`` The shape against which all generated shapes can broadcast. - The default shape is empty, which corresponds to a scalar and thus does not - constrain broadcasting at all. - * ``min_dims`` The smallest length that any generated shape can possess. - * ``max_dims`` The largest length that any generated shape can possess. - It cannot exceed 32, which is the greatest supported dimensionality for a - numpy array. The default-value for ``max_dims`` is - ``2 + max(len(shape), min_dims)``, capped at 32. - * ``min_side`` The smallest size that an unaligned dimension can possess. - * ``max_side`` The largest size that an unaligned dimension can possess. - The default value is 2 + 'size-of-largest-aligned-dimension'. - - The following are some examples drawn from this strategy. + arg_msg = "Pass either the `num_shapes` or the `signature` argument, but not both." + if num_shapes is not not_set: + check_argument(signature is not_set, arg_msg) + check_type(int, num_shapes, "num_shapes") + assert isinstance(num_shapes, int) # for mypy + check_argument(num_shapes >= 1, "num_shapes={} must be at least 1", num_shapes) + parsed_signature = None + sig_dims = 0 + else: + check_argument(signature is not not_set, arg_msg) + if signature is None: + raise InvalidArgument( + "Expected a string, but got invalid signature=None. " + "(maybe .signature attribute of an element-wise ufunc?)" + ) + check_type(str, signature, "signature") + parsed_signature = _hypothesis_parse_gufunc_signature(signature) + sig_dims = min( + map(len, parsed_signature.input_shapes + (parsed_signature.result_shape,)) + ) + num_shapes = len(parsed_signature.input_shapes) + assert num_shapes >= 1 + + return _mutually_broadcastable_shapes( + num_shapes=num_shapes, + signature=parsed_signature, + base_shape=base_shape, + min_dims=min_dims, + max_dims=max_dims, + min_side=min_side, + max_side=max_side, + ) - .. code-block:: pycon - >>> # Draw three shapes, and each shape is broadcast-compatible with `(2, 3)` - >>> for _ in range(5): - ... mutually_broadcastable_shapes(num_shapes=3, base_shape=(2, 3)).example() - BroadcastableShapes(input_shapes=((4, 1, 3), (4, 2, 3), ()), result_shape=(4, 2, 3)) - BroadcastableShapes(input_shapes=((3,), (1,), (2, 1)), result_shape=(2, 3)) - BroadcastableShapes(input_shapes=((3,), (1, 3), (2, 3)), result_shape=(2, 3)) - BroadcastableShapes(input_shapes=((), (), ()), result_shape=(2, 3)) - BroadcastableShapes(input_shapes=((3,), (), (3,)), result_shape=(2, 3)) +mutually_broadcastable_shapes.__doc__ = f""" + {_mutually_broadcastable_shapes.__doc__} **Use with Generalised Universal Function signatures** @@ -1075,159 +912,18 @@ def mutually_broadcastable_shapes( BroadcastableShapes(input_shapes=((2,), (2,)), result_shape=()) BroadcastableShapes(input_shapes=((3, 4, 2), (1, 2)), result_shape=(3, 4)) BroadcastableShapes(input_shapes=((4, 2), (1, 2, 3)), result_shape=(4, 3)) - """ - arg_msg = "Pass either the `num_shapes` or the `signature` argument, but not both." - if num_shapes is not not_set: - check_argument(signature is not_set, arg_msg) - check_type(int, num_shapes, "num_shapes") - assert isinstance(num_shapes, int) # for mypy - check_argument(num_shapes >= 1, "num_shapes={} must be at least 1", num_shapes) - parsed_signature = None - sig_dims = 0 - else: - check_argument(signature is not not_set, arg_msg) - if signature is None: - raise InvalidArgument( - "Expected a string, but got invalid signature=None. " - "(maybe .signature attribute of an element-wise ufunc?)" - ) - check_type(str, signature, "signature") - parsed_signature = _hypothesis_parse_gufunc_signature(signature) - sig_dims = min( - map(len, parsed_signature.input_shapes + (parsed_signature.result_shape,)) - ) - num_shapes = len(parsed_signature.input_shapes) - assert num_shapes >= 1 - - check_type(tuple, base_shape, "base_shape") - strict_check = max_dims is not None - check_type(int, min_side, "min_side") - check_type(int, min_dims, "min_dims") - - if max_dims is None: - max_dims = min(32 - sig_dims, max(len(base_shape), min_dims) + 2) - else: - check_type(int, max_dims, "max_dims") - - if max_side is None: - max_side = max(tuple(base_shape[-max_dims:]) + (min_side,)) + 2 - else: - check_type(int, max_side, "max_side") - - order_check("dims", 0, min_dims, max_dims) - order_check("side", 0, min_side, max_side) - if 32 - sig_dims < max_dims: - if sig_dims == 0: - raise InvalidArgument("max_dims cannot exceed 32") - raise InvalidArgument( - f"max_dims={signature!r} would exceed the 32-dimension limit given " - f"signature={parsed_signature!r}" - ) + """ - dims, bnd_name = (max_dims, "max_dims") if strict_check else (min_dims, "min_dims") - # check for unsatisfiable min_side - if not all(min_side <= s for s in base_shape[::-1][:dims] if s != 1): - raise InvalidArgument( - "Given base_shape=%r, there are no broadcast-compatible " - "shapes that satisfy: %s=%s and min_side=%s" - % (base_shape, bnd_name, dims, min_side) - ) +basic_indices.__doc__ = f""" + Return a strategy for :np-ref:`basic indexes ` of + arrays with the specified shape, which may include dimensions of size zero. - # check for unsatisfiable [min_side, max_side] - if not ( - min_side <= 1 <= max_side or all(s <= max_side for s in base_shape[::-1][:dims]) - ): - raise InvalidArgument( - "Given base_shape=%r, there are no broadcast-compatible shapes " - "that satisfy all of %s=%s, min_side=%s, and max_side=%s" - % (base_shape, bnd_name, dims, min_side, max_side) - ) + {basic_indices.__doc__} - if not strict_check: - # reduce max_dims to exclude unsatisfiable dimensions - for n, s in zip(range(max_dims), reversed(base_shape)): - if s < min_side and s != 1: - max_dims = n - break - elif not (min_side <= 1 <= max_side or s <= max_side): - max_dims = n - break - - return MutuallyBroadcastableShapesStrategy( - num_shapes=num_shapes, - signature=parsed_signature, - base_shape=base_shape, - min_dims=min_dims, - max_dims=max_dims, - min_side=min_side, - max_side=max_side, - ) - - -@defines_strategy() -def basic_indices( - shape: Shape, - *, - min_dims: int = 0, - max_dims: Optional[int] = None, - allow_newaxis: bool = False, - allow_ellipsis: bool = True, -) -> st.SearchStrategy[BasicIndex]: - """ - The ``basic_indices`` strategy generates :np-ref:`basic indexes ` - for arrays of the specified shape, which may include dimensions of size zero. - - It generates tuples containing some mix of integers, :obj:`python:slice` objects, - ``...`` (Ellipsis), and :obj:`numpy:numpy.newaxis`; which when used to index a - ``shape``-shaped array will produce either a scalar or a shared-memory view. - When a length-one tuple would be generated, this strategy may instead return - the element which will index the first axis, e.g. ``5`` instead of ``(5,)``. - - * ``shape``: the array shape that will be indexed, as a tuple of integers >= 0. - This must be at least two-dimensional for a tuple to be a valid basic index; - for one-dimensional arrays use :func:`~hypothesis.strategies.slices` instead. - * ``min_dims``: the minimum dimensionality of the resulting view from use of - the generated index. When ``min_dims == 0``, scalars and zero-dimensional - arrays are both allowed. - * ``max_dims``: the maximum dimensionality of the resulting view. - If not specified, it defaults to ``max(len(shape), min_dims) + 2``. - * ``allow_ellipsis``: whether ``...``` is allowed in the index. - * ``allow_newaxis``: whether :obj:`numpy:numpy.newaxis` is allowed in the index. - - Note that the length of the generated tuple may be anywhere between zero - and ``min_dims``. It may not match the length of ``shape``, or even the - dimensionality of the array view resulting from its use! + Note if ``min_dims == 0``, zero-dimensional arrays are allowed. """ - # Arguments to exclude scalars, zero-dim arrays, and dims of size zero were - # all considered and rejected. We want users to explicitly consider those - # cases if they're dealing in general indexers, and while it's fiddly we can - # back-compatibly add them later (hence using kwonlyargs). - check_type(tuple, shape, "shape") - check_type(bool, allow_ellipsis, "allow_ellipsis") - check_type(bool, allow_newaxis, "allow_newaxis") - check_type(int, min_dims, "min_dims") - if max_dims is None: - max_dims = min(max(len(shape), min_dims) + 2, 32) - else: - check_type(int, max_dims, "max_dims") - order_check("dims", 0, min_dims, max_dims) - check_argument( - max_dims <= 32, - f"max_dims={max_dims!r}, but numpy arrays have at most 32 dimensions", - ) - check_argument( - all(isinstance(x, int) and x >= 0 for x in shape), - f"shape={shape!r}, but all dimensions must be of integer size >= 0", - ) - return BasicIndexStrategy( - shape, - min_dims=min_dims, - max_dims=max_dims, - allow_ellipsis=allow_ellipsis, - allow_none=allow_newaxis, - ) @defines_strategy() diff --git a/hypothesis-python/tests/numpy/test_gen_data.py b/hypothesis-python/tests/numpy/test_gen_data.py index b3d95a6484..57e22d9afb 100644 --- a/hypothesis-python/tests/numpy/test_gen_data.py +++ b/hypothesis-python/tests/numpy/test_gen_data.py @@ -1053,8 +1053,8 @@ def test_advanced_integer_index_can_generate_any_pattern(shape, data): [ lambda ix: Ellipsis in ix, lambda ix: Ellipsis not in ix, - lambda ix: np.newaxis in ix, - lambda ix: np.newaxis not in ix, + lambda ix: None in ix, + lambda ix: None not in ix, ], ) def test_basic_indices_options(condition): @@ -1121,7 +1121,7 @@ def test_basic_indices_generate_valid_indexers( assert 0 <= len(indexer) <= len(shape) + int(allow_ellipsis) else: assert 1 <= len(shape) + int(allow_ellipsis) - assert np.newaxis not in shape + assert None not in shape if not allow_ellipsis: assert Ellipsis not in shape From 70bbe837075b58f17eb0956b56a12bc9c1dc4ea4 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Thu, 26 Aug 2021 19:38:30 +0100 Subject: [PATCH 05/24] Strategy factories in __array_helpers.py --- .../src/hypothesis/extra/__array_helpers.py | 641 ++++++++++-------- .../src/hypothesis/extra/numpy.py | 25 +- 2 files changed, 362 insertions(+), 304 deletions(-) diff --git a/hypothesis-python/src/hypothesis/extra/__array_helpers.py b/hypothesis-python/src/hypothesis/extra/__array_helpers.py index f9eecbd7c6..4c9743e67c 100644 --- a/hypothesis-python/src/hypothesis/extra/__array_helpers.py +++ b/hypothesis-python/src/hypothesis/extra/__array_helpers.py @@ -13,7 +13,8 @@ # # END HEADER -from typing import NamedTuple, Optional, Tuple, Union +from collections import defaultdict +from typing import NamedTuple, Optional, Tuple, Union, Callable from hypothesis import assume, strategies as st from hypothesis.errors import InvalidArgument @@ -27,11 +28,11 @@ "BroadcastableShapes", "BasicIndex", "order_check", - "array_shapes", + "make_array_shapes", "valid_tuple_axes", - "broadcastable_shapes", - "mutually_broadcastable_shapes", - "basic_indices", + "make_broadcastable_shapes", + "make_mutually_broadcastable_shapes", + "make_basic_indices", ] @@ -53,40 +54,53 @@ def order_check(name, floor, min_, max_): raise InvalidArgument(f"min_{name}={min_} is larger than max_{name}={max_}") -@defines_strategy() -def array_shapes( - *, - min_dims: int = 1, - max_dims: Optional[int] = None, - min_side: int = 1, - max_side: Optional[int] = None, -) -> st.SearchStrategy[Shape]: - """Return a strategy for array shapes (tuples of int >= 1). - - * ``min_dims`` is the smallest length that the generated shape can possess. - * ``max_dims`` is the largest length that the generated shape can possess, - defaulting to ``min_dims + 2``. - * ``min_side`` is the smallest size that a dimension can possess. - * ``max_side`` is the largest size that a dimension can possess, - defaulting to ``min_side + 5``. - """ - check_type(int, min_dims, "min_dims") - check_type(int, min_side, "min_side") +lib_max_dims = defaultdict(lambda: float("inf"), {"numpy": 32}) - if max_dims is None: - max_dims = min_dims + 2 - check_type(int, max_dims, "max_dims") - if max_side is None: - max_side = min_side + 5 - check_type(int, max_side, "max_side") +@check_function +def check_valid_dims(lib, name, dims): + if dims > lib_max_dims[lib]: + raise InvalidArgument( + f"{name}={dims}, but {lib} does not support arrays greater than " + f"{lib_max_dims[lib]} dimensions" + ) - order_check("dims", 0, min_dims, max_dims) - order_check("side", 0, min_side, max_side) - return st.lists( - st.integers(min_side, max_side), min_size=min_dims, max_size=max_dims - ).map(tuple) +def make_array_shapes(lib: Optional[str] = None) -> Callable: + @defines_strategy() + def array_shapes( + *, + min_dims: int = 1, + max_dims: Optional[int] = None, + min_side: int = 1, + max_side: Optional[int] = None, + ) -> st.SearchStrategy[Shape]: + """Return a strategy for array shapes (tuples of int >= 1). + + * ``min_dims`` is the smallest length that the generated shape can possess. + * ``max_dims`` is the largest length that the generated shape can possess, + defaulting to ``min_dims + 2``. + * ``min_side`` is the smallest size that a dimension can possess. + * ``max_side`` is the largest size that a dimension can possess, + defaulting to ``min_side + 5``. + """ + check_type(int, min_dims, "min_dims") + check_type(int, min_side, "min_side") + check_valid_dims(lib, "min_dims", min_dims) + if max_dims is None: + max_dims = min(min_dims + 2, lib_max_dims[lib]) + check_type(int, max_dims, "max_dims") + check_valid_dims(lib, "max_dims", max_dims) + if max_side is None: + max_side = min_side + 5 + check_type(int, max_side, "max_side") + order_check("dims", 0, min_dims, max_dims) + order_check("side", 0, min_side, max_side) + return st.lists( + st.integers(min_side, max_side), min_size=min_dims, max_size=max_dims + ).map(tuple) + + return array_shapes @defines_strategy() @@ -106,15 +120,15 @@ def valid_tuple_axes( .. code-block:: pycon - >>> [valid_tuple_axes(3).example() for i in range(4)] - [(-3, 1), (0, 1, -1), (0, 2), (0, -2, 2)] + >>> [valid_tuple_axes(3).example() for i in range(4)] + [(-3, 1), (0, 1, -1), (0, 2), (0, -2, 2)] ``valid_tuple_axes`` can be joined with other strategies to generate any type of valid axis object, i.e. integers, tuples, and ``None``: .. code-block:: python - any_axis_strategy = none() | integers(-ndim, ndim - 1) | valid_tuple_axes(ndim) + any_axis_strategy = none() | integers(-ndim, ndim - 1) | valid_tuple_axes(ndim) """ if max_size is None: @@ -132,6 +146,301 @@ def valid_tuple_axes( ).map(tuple) +def make_broadcastable_shapes(lib: Optional[str] = None) -> Callable: + @defines_strategy() + def broadcastable_shapes( + shape: Shape, + *, + min_dims: int = 0, + max_dims: Optional[int] = None, + min_side: int = 1, + max_side: Optional[int] = None, + ) -> st.SearchStrategy[Shape]: + """Return a strategy for shapes that are broadcast-compatible with the + provided shape. + + Examples from this strategy shrink towards a shape with length ``min_dims``. + The size of an aligned dimension shrinks towards size ``1``. The size of an + unaligned dimension shrink towards ``min_side``. + + * ``shape`` is a tuple of integers. + * ``min_dims`` is the smallest length that the generated shape can possess. + * ``max_dims`` is the largest length that the generated shape can possess, + defaulting to ``max(len(shape), min_dims) + 2``. + * ``min_side`` is the smallest size that an unaligned dimension can possess. + * ``max_side`` is the largest size that an unaligned dimension can possess, + defaulting to 2 plus the size of the largest aligned dimension. + + The following are some examples drawn from this strategy. + + .. code-block:: pycon + + >>> [broadcastable_shapes(shape=(2, 3)).example() for i in range(5)] + [(1, 3), (), (2, 3), (2, 1), (4, 1, 3), (3, )] + + """ + check_type(tuple, shape, "shape") + check_type(int, min_side, "min_side") + check_type(int, min_dims, "min_dims") + check_valid_dims(lib, "min_dims", min_dims) + + strict_check = max_side is None or max_dims is None + + if max_dims is None: + max_dims = min(max(len(shape), min_dims) + 2, lib_max_dims[lib]) + check_type(int, max_dims, "max_dims") + check_valid_dims(lib, "max_dims", max_dims) + + if max_side is None: + max_side = max(shape[-max_dims:] + (min_side,)) + 2 + check_type(int, max_side, "max_side") + + order_check("dims", 0, min_dims, max_dims) + order_check("side", 0, min_side, max_side) + + if strict_check: + dims = max_dims + bound_name = "max_dims" + else: + dims = min_dims + bound_name = "min_dims" + + # check for unsatisfiable min_side + if not all(min_side <= s for s in shape[::-1][:dims] if s != 1): + raise InvalidArgument( + f"Given shape={shape}, there are no broadcast-compatible " + f"shapes that satisfy: {bound_name}={dims} and min_side={min_side}" + ) + + # check for unsatisfiable [min_side, max_side] + if not ( + min_side <= 1 <= max_side or all(s <= max_side for s in shape[::-1][:dims]) + ): + raise InvalidArgument( + f"Given base_shape={shape}, there are no broadcast-compatible " + f"shapes that satisfy all of {bound_name}={dims}, " + f"min_side={min_side}, and max_side={max_side}" + ) + + if not strict_check: + # reduce max_dims to exclude unsatisfiable dimensions + for n, s in zip(range(max_dims), shape[::-1]): + if s < min_side and s != 1: + max_dims = n + break + elif not (min_side <= 1 <= max_side or s <= max_side): + max_dims = n + break + + return MutuallyBroadcastableShapesStrategy( + num_shapes=1, + base_shape=shape, + min_dims=min_dims, + max_dims=max_dims, + min_side=min_side, + max_side=max_side, + ).map(lambda x: x.input_shapes[0]) + + return broadcastable_shapes + + +def make_mutually_broadcastable_shapes(lib: Optional[str] = None) -> Callable: + @defines_strategy() + def mutually_broadcastable_shapes( + num_shapes: int, + *, + signature: Optional[BroadcastableShapes] = None, + base_shape: Shape = (), + min_dims: int = 0, + max_dims: Optional[int] = None, + min_side: int = 1, + max_side: Optional[int] = None, + ) -> st.SearchStrategy[BroadcastableShapes]: + """Return a strategy for a specified number of shapes N that are + mutually-broadcastable with one another and with the provided base shape. + + * ``num_shapes`` is the number of mutually broadcast-compatible shapes to generate. + * ``base_shape`` is the shape against which all generated shapes can broadcast. + The default shape is empty, which corresponds to a scalar and thus does + not constrain broadcasting at all. + * ``shape`` is a tuple of integers. + * ``min_dims`` is the smallest length that the generated shape can possess. + * ``max_dims`` is the largest length that the generated shape can possess, + defaulting to ``max(len(shape), min_dims) + 2``. + * ``min_side`` is the smallest size that an unaligned dimension can possess. + * ``max_side`` is the largest size that an unaligned dimension can possess, + defaulting to 2 plus the size of the largest aligned dimension. + + The strategy will generate a :obj:`python:typing.NamedTuple` containing: + + * ``input_shapes`` as a tuple of the N generated shapes. + * ``result_shape`` as the resulting shape produced by broadcasting the N shapes + with the base shape. + + The following are some examples drawn from this strategy. + + .. code-block:: pycon + + >>> # Draw three shapes where each shape is broadcast-compatible with (2, 3) + ... strat = mutually_broadcastable_shapes(num_shapes=3, base_shape=(2, 3)) + >>> for _ in range(5): + ... print(strat.example()) + BroadcastableShapes(input_shapes=((4, 1, 3), (4, 2, 3), ()), result_shape=(4, 2, 3)) + BroadcastableShapes(input_shapes=((3,), (1, 3), (2, 3)), result_shape=(2, 3)) + BroadcastableShapes(input_shapes=((), (), ()), result_shape=(2, 3)) + BroadcastableShapes(input_shapes=((3,), (), (3,)), result_shape=(2, 3)) + BroadcastableShapes(input_shapes=((1, 2, 3), (3,), ()), result_shape=(1, 2, 3)) + + """ + + if signature is None: + sig_dims = 0 + else: + sig_dims = min( + len(s) for s in signature.input_shapes + (signature.result_shape,) + ) + + check_type(int, num_shapes, "num_shapes") + if num_shapes < 1: + raise InvalidArgument(f"num_shapes={num_shapes} must be at least 1") + + check_type(tuple, base_shape, "base_shape") + check_type(int, min_side, "min_side") + check_type(int, min_dims, "min_dims") + check_valid_dims(lib, "min_dims", min_dims) + + strict_check = max_dims is not None + + if max_dims is None: + max_dims = min( + max(len(base_shape), min_dims) + 2, lib_max_dims[lib] - sig_dims + ) + check_type(int, max_dims, "max_dims") + check_valid_dims(lib, "max_dims", max_dims) + + if max_side is None: + max_side = max(base_shape[-max_dims:] + (min_side,)) + 2 + check_type(int, max_side, "max_side") + + order_check("dims", 0, min_dims, max_dims) + order_check("side", 0, min_side, max_side) + + if lib_max_dims[lib] - sig_dims < max_dims: + if signature is None: + raise InvalidArgument(f"max_dims cannot exceed {lib_max_dims[lib]}") + raise InvalidArgument( + f"max_dims={max_dims} would exceed the {lib_max_dims[lib]}-dimension " + f"limit given signature=(input_shapes={signature.input_shapes}, " + f"result_shape={signature.result_shape})" + ) + + if strict_check: + dims = max_dims + bound_name = "max_dims" + else: + dims = min_dims + bound_name = "min_dims" + + # check for unsatisfiable min_side + if not all(min_side <= s for s in base_shape[::-1][:dims] if s != 1): + raise InvalidArgument( + f"Given base_shape={base_shape}, there are no broadcast-compatible " + f"shapes that satisfy: {bound_name}={dims} and min_side={min_side}" + ) + + # check for unsatisfiable [min_side, max_side] + if not ( + min_side <= 1 <= max_side + or all(s <= max_side for s in base_shape[::-1][:dims]) + ): + raise InvalidArgument( + f"Given base_shape={base_shape}, there are no broadcast-compatible " + f"shapes that satisfy all of {bound_name}={dims}, " + f"min_side={min_side}, and max_side={max_side}" + ) + + if not strict_check: + # reduce max_dims to exclude unsatisfiable dimensions + for n, s in zip(range(max_dims), base_shape[::-1]): + if s < min_side and s != 1: + max_dims = n + break + elif not (min_side <= 1 <= max_side or s <= max_side): + max_dims = n + break + + return MutuallyBroadcastableShapesStrategy( + num_shapes=num_shapes, + signature=signature, + base_shape=base_shape, + min_dims=min_dims, + max_dims=max_dims, + min_side=min_side, + max_side=max_side, + ) + + return mutually_broadcastable_shapes + + +def make_basic_indices(lib: Optional[str] = None, *, allow_0d_index=False) -> Callable: + min_index_dim = 0 if allow_0d_index else 1 + + @defines_strategy() + def basic_indices( + shape: Shape, + *, + min_dims: int = min_index_dim, + max_dims: Optional[int] = None, + allow_newaxis: bool = False, + allow_ellipsis: bool = True, + ) -> st.SearchStrategy[BasicIndex]: + """It generates tuples containing some mix of integers, :obj:`python:slice` + objects, ``...`` (an ``Ellipsis``), and ``None``. When a length-one tuple + would be generated, this strategy may instead return the element which will + index the first axis, e.g. ``5`` instead of ``(5,)``. + + * ``shape`` is the shape of the array that will be indexed, as a tuple of + positive integers. This must be at least two-dimensional for a tuple to be a + valid index; for one-dimensional arrays use + :func:`~hypothesis.strategies.slices` instead. + * ``min_dims`` is the minimum dimensionality of the resulting array from use of + the generated index. + * ``max_dims`` is the the maximum dimensionality of the resulting array, + defaulting to ``max(len(shape), min_dims) + 2``. + * ``allow_newaxis`` specifies whether ``None`` is allowed in the index. + * ``allow_ellipsis`` specifies whether ``...`` is allowed in the index. + """ + # Arguments to exclude scalars, zero-dim arrays, and dims of size zero were + # all considered and rejected. We want users to explicitly consider those + # cases if they're dealing in general indexers, and while it's fiddly we can + # back-compatibly add them later (hence using kwonlyargs). + check_type(tuple, shape, "shape") + if not allow_0d_index and len(shape) == 0: + raise InvalidArgument("Indices for 0-dimensional arrays are not allowed") + check_type(bool, allow_ellipsis, "allow_ellipsis") + check_type(bool, allow_newaxis, "allow_newaxis") + check_type(int, min_dims, "min_dims") + check_valid_dims(lib, "min_dims", min_dims) + if max_dims is None: + max_dims = min(max(len(shape), min_dims) + 2, lib_max_dims[lib]) + check_type(int, max_dims, "max_dims") + check_valid_dims(lib, "max_dims", max_dims) + order_check("dims", min_index_dim, min_dims, max_dims) + if not all(isinstance(x, int) and x >= 0 for x in shape): + raise InvalidArgument( + f"shape={shape!r}, but all dimensions must be of integer size >= 0" + ) + return BasicIndexStrategy( + shape, + min_dims=min_dims, + max_dims=max_dims, + allow_ellipsis=allow_ellipsis, + allow_newaxis=allow_newaxis, + ) + + return basic_indices + + class MutuallyBroadcastableShapesStrategy(st.SearchStrategy): def __init__( self, @@ -271,212 +580,6 @@ def _draw_loop_dimensions(self, data, use=None): ) -@defines_strategy() -def broadcastable_shapes( - shape: Shape, - *, - min_dims: int = 0, - max_dims: Optional[int] = None, - min_side: int = 1, - max_side: Optional[int] = None, -) -> st.SearchStrategy[Shape]: - """Return a strategy for shapes that are broadcast-compatible with the - provided shape. - - Examples from this strategy shrink towards a shape with length ``min_dims``. - The size of an aligned dimension shrinks towards size ``1``. The size of an - unaligned dimension shrink towards ``min_side``. - - * ``shape`` is a tuple of integers. - * ``min_dims`` is the smallest length that the generated shape can possess. - * ``max_dims`` is the largest length that the generated shape can possess, - defaulting to ``min(32, max(len(shape), min_dims) + 2)``. - * ``min_side`` is the smallest size that an unaligned dimension can possess. - * ``max_side`` is the largest size that an unaligned dimension can possess, - defaulting to 2 plus the size of the largest aligned dimension. - - The following are some examples drawn from this strategy. - - .. code-block:: pycon - - >>> [broadcastable_shapes(shape=(2, 3)).example() for i in range(5)] - [(1, 3), (), (2, 3), (2, 1), (4, 1, 3), (3, )] - """ - check_type(tuple, shape, "shape") - check_type(int, min_side, "min_side") - check_type(int, min_dims, "min_dims") - - strict_check = max_side is None or max_dims is None - - if max_dims is None: - max_dims = min(32, max(len(shape), min_dims) + 2) - check_type(int, max_dims, "max_dims") - - if max_side is None: - max_side = max(shape[-max_dims:] + (min_side,)) + 2 - check_type(int, max_side, "max_side") - - order_check("dims", 0, min_dims, max_dims) - order_check("side", 0, min_side, max_side) - - if strict_check: - dims = max_dims - bound_name = "max_dims" - else: - dims = min_dims - bound_name = "min_dims" - - # check for unsatisfiable min_side - if not all(min_side <= s for s in shape[::-1][:dims] if s != 1): - raise InvalidArgument( - f"Given shape={shape}, there are no broadcast-compatible " - f"shapes that satisfy: {bound_name}={dims} and min_side={min_side}" - ) - - # check for unsatisfiable [min_side, max_side] - if not ( - min_side <= 1 <= max_side or all(s <= max_side for s in shape[::-1][:dims]) - ): - raise InvalidArgument( - f"Given base_shape={shape}, there are no broadcast-compatible " - f"shapes that satisfy all of {bound_name}={dims}, " - f"min_side={min_side}, and max_side={max_side}" - ) - - if not strict_check: - # reduce max_dims to exclude unsatisfiable dimensions - for n, s in zip(range(max_dims), shape[::-1]): - if s < min_side and s != 1: - max_dims = n - break - elif not (min_side <= 1 <= max_side or s <= max_side): - max_dims = n - break - - return MutuallyBroadcastableShapesStrategy( - num_shapes=1, - base_shape=shape, - min_dims=min_dims, - max_dims=max_dims, - min_side=min_side, - max_side=max_side, - ).map(lambda x: x.input_shapes[0]) - - -@defines_strategy() -def mutually_broadcastable_shapes( - num_shapes: int, - *, - signature: Optional[str] = None, - base_shape: Shape = (), - min_dims: int = 0, - max_dims: Optional[int] = None, - min_side: int = 1, - max_side: Optional[int] = None, -) -> st.SearchStrategy[BroadcastableShapes]: - """Return a strategy for a specified number of shapes N that are - mutually-broadcastable with one another and with the provided base shape. - - * ``num_shapes`` is the number of mutually broadcast-compatible shapes to generate. - * ``base_shape`` is the shape against which all generated shapes can broadcast. - The default shape is empty, which corresponds to a scalar and thus does - not constrain broadcasting at all. - * ``shape`` is a tuple of integers. - * ``min_dims`` is the smallest length that the generated shape can possess. - * ``max_dims`` is the largest length that the generated shape can possess, - defaulting to ``min(32, max(len(shape), min_dims) + 2)``. - * ``min_side`` is the smallest size that an unaligned dimension can possess. - * ``max_side`` is the largest size that an unaligned dimension can possess, - defaulting to 2 plus the size of the largest aligned dimension. - - The strategy will generate a :obj:`python:typing.NamedTuple` containing: - - * ``input_shapes`` as a tuple of the N generated shapes. - * ``result_shape`` as the resulting shape produced by broadcasting the N shapes - with the base shape. - - The following are some examples drawn from this strategy. - - .. code-block:: pycon - - >>> # Draw three shapes where each shape is broadcast-compatible with (2, 3) - ... strat = mutually_broadcastable_shapes(num_shapes=3, base_shape=(2, 3)) - >>> for _ in range(5): - ... print(strat.example()) - BroadcastableShapes(input_shapes=((4, 1, 3), (4, 2, 3), ()), result_shape=(4, 2, 3)) - BroadcastableShapes(input_shapes=((3,), (1, 3), (2, 3)), result_shape=(2, 3)) - BroadcastableShapes(input_shapes=((), (), ()), result_shape=(2, 3)) - BroadcastableShapes(input_shapes=((3,), (), (3,)), result_shape=(2, 3)) - BroadcastableShapes(input_shapes=((1, 2, 3), (3,), ()), result_shape=(1, 2, 3)) - - """ - - check_type(int, num_shapes, "num_shapes") - if num_shapes < 1: - raise InvalidArgument(f"num_shapes={num_shapes} must be at least 1") - - check_type(tuple, base_shape, "base_shape") - check_type(int, min_side, "min_side") - check_type(int, min_dims, "min_dims") - - strict_check = max_dims is not None - - if max_dims is None: - max_dims = min(32, max(len(base_shape), min_dims) + 2) - check_type(int, max_dims, "max_dims") - - if max_side is None: - max_side = max(base_shape[-max_dims:] + (min_side,)) + 2 - check_type(int, max_side, "max_side") - - order_check("dims", 0, min_dims, max_dims) - order_check("side", 0, min_side, max_side) - - if strict_check: - dims = max_dims - bound_name = "max_dims" - else: - dims = min_dims - bound_name = "min_dims" - - # check for unsatisfiable min_side - if not all(min_side <= s for s in base_shape[::-1][:dims] if s != 1): - raise InvalidArgument( - f"Given base_shape={base_shape}, there are no broadcast-compatible " - f"shapes that satisfy: {bound_name}={dims} and min_side={min_side}" - ) - - # check for unsatisfiable [min_side, max_side] - if not ( - min_side <= 1 <= max_side or all(s <= max_side for s in base_shape[::-1][:dims]) - ): - raise InvalidArgument( - f"Given base_shape={base_shape}, there are no broadcast-compatible " - f"shapes that satisfy all of {bound_name}={dims}, " - f"min_side={min_side}, and max_side={max_side}" - ) - - if not strict_check: - # reduce max_dims to exclude unsatisfiable dimensions - for n, s in zip(range(max_dims), base_shape[::-1]): - if s < min_side and s != 1: - max_dims = n - break - elif not (min_side <= 1 <= max_side or s <= max_side): - max_dims = n - break - - return MutuallyBroadcastableShapesStrategy( - num_shapes=num_shapes, - signature=signature, - base_shape=base_shape, - min_dims=min_dims, - max_dims=max_dims, - min_side=min_side, - max_side=max_side, - ) - - class BasicIndexStrategy(st.SearchStrategy): def __init__(self, shape, min_dims, max_dims, allow_ellipsis, allow_newaxis): self.shape = shape @@ -526,53 +629,3 @@ def do_draw(self, data): # Sometimes generate bare element equivalent to a length-one tuple return result[0] return tuple(result) - - -@defines_strategy() -def basic_indices( - shape: Shape, - *, - min_dims: int = 0, - max_dims: Optional[int] = None, - allow_newaxis: bool = False, - allow_ellipsis: bool = True, -) -> st.SearchStrategy[BasicIndex]: - """It generates tuples containing some mix of integers, :obj:`python:slice` - objects, ``...`` (an ``Ellipsis``), and ``None``. When a length-one tuple - would be generated, this strategy may instead return the element which will - index the first axis, e.g. ``5`` instead of ``(5,)``. - - * ``shape`` is the shape of the array that will be indexed, as a tuple of - integers >= 0. This must be at least two-dimensional for a tuple to be a - valid index; for one-dimensional arrays use - :func:`~hypothesis.strategies.slices` instead. - * ``min_dims`` is the minimum dimensionality of the resulting array from use of - the generated index. - * ``max_dims`` is the the maximum dimensionality of the resulting array, - defaulting to ``max(len(shape), min_dims) + 2``. - * ``allow_newaxis`` specifies whether ``None`` is allowed in the index. - * ``allow_ellipsis`` specifies whether ``...`` is allowed in the index. - """ - # Arguments to exclude scalars, zero-dim arrays, and dims of size zero were - # all considered and rejected. We want users to explicitly consider those - # cases if they're dealing in general indexers, and while it's fiddly we can - # back-compatibly add them later (hence using kwonlyargs). - check_type(tuple, shape, "shape") - check_type(bool, allow_ellipsis, "allow_ellipsis") - check_type(bool, allow_newaxis, "allow_newaxis") - check_type(int, min_dims, "min_dims") - if max_dims is None: - max_dims = min(max(len(shape), min_dims) + 2, 32) - check_type(int, max_dims, "max_dims") - order_check("dims", 0, min_dims, max_dims) - if not all(isinstance(x, int) and x >= 0 for x in shape): - raise InvalidArgument( - f"shape={shape!r}, but all dimensions must be of integer size >= 0" - ) - return BasicIndexStrategy( - shape, - min_dims=min_dims, - max_dims=max_dims, - allow_ellipsis=allow_ellipsis, - allow_newaxis=allow_newaxis, - ) diff --git a/hypothesis-python/src/hypothesis/extra/numpy.py b/hypothesis-python/src/hypothesis/extra/numpy.py index c886859a61..122f2347aa 100644 --- a/hypothesis-python/src/hypothesis/extra/numpy.py +++ b/hypothesis-python/src/hypothesis/extra/numpy.py @@ -24,10 +24,10 @@ from hypothesis.extra.__array_helpers import ( BroadcastableShapes, Shape, - array_shapes, - basic_indices, - broadcastable_shapes, - mutually_broadcastable_shapes as _mutually_broadcastable_shapes, + make_array_shapes, + make_basic_indices, + make_broadcastable_shapes, + make_mutually_broadcastable_shapes, order_check, valid_tuple_axes, ) @@ -469,6 +469,9 @@ def arrays( return ArrayStrategy(elements, shape, dtype, fill, unique) +array_shapes = make_array_shapes("numpy") + + @defines_strategy() def scalar_dtypes() -> st.SearchStrategy[np.dtype]: """Return a strategy that can return any non-flexible scalar dtype.""" @@ -831,6 +834,12 @@ def _hypothesis_parse_gufunc_signature(signature, all_checks=True): return _GUfuncSig(input_shapes=input_shapes, result_shape=result_shape) +broadcastable_shapes = make_broadcastable_shapes("numpy") + + +_mutually_broadcastable_shapes = make_mutually_broadcastable_shapes("numpy") + + @defines_strategy() def mutually_broadcastable_shapes( *, @@ -849,7 +858,6 @@ def mutually_broadcastable_shapes( assert isinstance(num_shapes, int) # for mypy check_argument(num_shapes >= 1, "num_shapes={} must be at least 1", num_shapes) parsed_signature = None - sig_dims = 0 else: check_argument(signature is not not_set, arg_msg) if signature is None: @@ -859,9 +867,6 @@ def mutually_broadcastable_shapes( ) check_type(str, signature, "signature") parsed_signature = _hypothesis_parse_gufunc_signature(signature) - sig_dims = min( - map(len, parsed_signature.input_shapes + (parsed_signature.result_shape,)) - ) num_shapes = len(parsed_signature.input_shapes) assert num_shapes >= 1 @@ -915,14 +920,14 @@ def mutually_broadcastable_shapes( """ - +basic_indices = make_basic_indices("numpy", allow_0d_index=True) basic_indices.__doc__ = f""" Return a strategy for :np-ref:`basic indexes ` of arrays with the specified shape, which may include dimensions of size zero. {basic_indices.__doc__} - Note if ``min_dims == 0``, zero-dimensional arrays are allowed. + Note if ``min_dims == 0``, indices for zero-dimensional arrays are generated. """ From eba9b1bb6cb6758f5a4dd0c0c09ab989a8dcc4f1 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Fri, 27 Aug 2021 11:26:49 +0100 Subject: [PATCH 06/24] Hard limit array dimensions to 32 --- .../src/hypothesis/extra/__array_helpers.py | 567 +++++++++--------- .../src/hypothesis/extra/numpy.py | 20 +- 2 files changed, 293 insertions(+), 294 deletions(-) diff --git a/hypothesis-python/src/hypothesis/extra/__array_helpers.py b/hypothesis-python/src/hypothesis/extra/__array_helpers.py index 4c9743e67c..c02bffbced 100644 --- a/hypothesis-python/src/hypothesis/extra/__array_helpers.py +++ b/hypothesis-python/src/hypothesis/extra/__array_helpers.py @@ -13,8 +13,7 @@ # # END HEADER -from collections import defaultdict -from typing import NamedTuple, Optional, Tuple, Union, Callable +from typing import Callable, NamedTuple, Optional, Tuple, Union from hypothesis import assume, strategies as st from hypothesis.errors import InvalidArgument @@ -28,10 +27,10 @@ "BroadcastableShapes", "BasicIndex", "order_check", - "make_array_shapes", + "array_shapes", "valid_tuple_axes", - "make_broadcastable_shapes", - "make_mutually_broadcastable_shapes", + "broadcastable_shapes", + "mutually_broadcastable_shapes", "make_basic_indices", ] @@ -54,53 +53,58 @@ def order_check(name, floor, min_, max_): raise InvalidArgument(f"min_{name}={min_} is larger than max_{name}={max_}") -lib_max_dims = defaultdict(lambda: float("inf"), {"numpy": 32}) +# 32 is a dimension limit specific to NumPy, and does not necessarily apply to +# other array/tensor libraries. Historically these strategies were built for the +# NumPy extra, so it's nice to keep these limits, and it's seemingly unlikely +# someone would want to generate >32 dim arrays anyway. +# See https://github.com/HypothesisWorks/hypothesis/pull/3067. +NDIM_MAX = 32 @check_function -def check_valid_dims(lib, name, dims): - if dims > lib_max_dims[lib]: +def check_valid_dims(dims, name): + if dims > NDIM_MAX: raise InvalidArgument( - f"{name}={dims}, but {lib} does not support arrays greater than " - f"{lib_max_dims[lib]} dimensions" + f"{name}={dims}, but Hypothesis does not support arrays with " + f"dimensions greater than {NDIM_MAX}" ) -def make_array_shapes(lib: Optional[str] = None) -> Callable: - @defines_strategy() - def array_shapes( - *, - min_dims: int = 1, - max_dims: Optional[int] = None, - min_side: int = 1, - max_side: Optional[int] = None, - ) -> st.SearchStrategy[Shape]: - """Return a strategy for array shapes (tuples of int >= 1). - - * ``min_dims`` is the smallest length that the generated shape can possess. - * ``max_dims`` is the largest length that the generated shape can possess, - defaulting to ``min_dims + 2``. - * ``min_side`` is the smallest size that a dimension can possess. - * ``max_side`` is the largest size that a dimension can possess, - defaulting to ``min_side + 5``. - """ - check_type(int, min_dims, "min_dims") - check_type(int, min_side, "min_side") - check_valid_dims(lib, "min_dims", min_dims) - if max_dims is None: - max_dims = min(min_dims + 2, lib_max_dims[lib]) - check_type(int, max_dims, "max_dims") - check_valid_dims(lib, "max_dims", max_dims) - if max_side is None: - max_side = min_side + 5 - check_type(int, max_side, "max_side") - order_check("dims", 0, min_dims, max_dims) - order_check("side", 0, min_side, max_side) - return st.lists( - st.integers(min_side, max_side), min_size=min_dims, max_size=max_dims - ).map(tuple) +def array_shapes( + *, + min_dims: int = 1, + max_dims: Optional[int] = None, + min_side: int = 1, + max_side: Optional[int] = None, +) -> st.SearchStrategy[Shape]: + """Return a strategy for array shapes (tuples of int >= 1). + + * ``min_dims`` is the smallest length that the generated shape can possess. + * ``max_dims`` is the largest length that the generated shape can possess, + defaulting to ``min_dims + 2``. + * ``min_side`` is the smallest size that a dimension can possess. + * ``max_side`` is the largest size that a dimension can possess, + defaulting to ``min_side + 5``. + """ + check_type(int, min_dims, "min_dims") + check_type(int, min_side, "min_side") + check_valid_dims(min_dims, "min_dims") + + if max_dims is None: + max_dims = min(min_dims + 2, NDIM_MAX) + check_type(int, max_dims, "max_dims") + check_valid_dims(max_dims, "max_dims") - return array_shapes + if max_side is None: + max_side = min_side + 5 + check_type(int, max_side, "max_side") + + order_check("dims", 0, min_dims, max_dims) + order_check("side", 0, min_side, max_side) + + return st.lists( + st.integers(min_side, max_side), min_size=min_dims, max_size=max_dims + ).map(tuple) @defines_strategy() @@ -120,270 +124,271 @@ def valid_tuple_axes( .. code-block:: pycon - >>> [valid_tuple_axes(3).example() for i in range(4)] - [(-3, 1), (0, 1, -1), (0, 2), (0, -2, 2)] + >>> [valid_tuple_axes(3).example() for i in range(4)] + [(-3, 1), (0, 1, -1), (0, 2), (0, -2, 2)] ``valid_tuple_axes`` can be joined with other strategies to generate any type of valid axis object, i.e. integers, tuples, and ``None``: .. code-block:: python - any_axis_strategy = none() | integers(-ndim, ndim - 1) | valid_tuple_axes(ndim) + any_axis_strategy = none() | integers(-ndim, ndim - 1) | valid_tuple_axes(ndim) """ - if max_size is None: - max_size = ndim check_type(int, ndim, "ndim") check_type(int, min_size, "min_size") + if max_size is None: + max_size = ndim check_type(int, max_size, "max_size") order_check("size", 0, min_size, max_size) check_valid_interval(max_size, ndim, "max_size", "ndim") + axes = st.integers(0, max(0, 2 * ndim - 1)).map( lambda x: x if x < ndim else x - 2 * ndim ) + return st.lists( axes, min_size=min_size, max_size=max_size, unique_by=lambda x: x % ndim ).map(tuple) -def make_broadcastable_shapes(lib: Optional[str] = None) -> Callable: - @defines_strategy() - def broadcastable_shapes( - shape: Shape, - *, - min_dims: int = 0, - max_dims: Optional[int] = None, - min_side: int = 1, - max_side: Optional[int] = None, - ) -> st.SearchStrategy[Shape]: - """Return a strategy for shapes that are broadcast-compatible with the - provided shape. +@defines_strategy() +def broadcastable_shapes( + shape: Shape, + *, + min_dims: int = 0, + max_dims: Optional[int] = None, + min_side: int = 1, + max_side: Optional[int] = None, +) -> st.SearchStrategy[Shape]: + """Return a strategy for shapes that are broadcast-compatible with the + provided shape. + + Examples from this strategy shrink towards a shape with length ``min_dims``. + The size of an aligned dimension shrinks towards size ``1``. The size of an + unaligned dimension shrink towards ``min_side``. + + * ``shape`` is a tuple of integers. + * ``min_dims`` is the smallest length that the generated shape can possess. + * ``max_dims`` is the largest length that the generated shape can possess, + defaulting to ``max(len(shape), min_dims) + 2``. + * ``min_side`` is the smallest size that an unaligned dimension can possess. + * ``max_side`` is the largest size that an unaligned dimension can possess, + defaulting to 2 plus the size of the largest aligned dimension. - Examples from this strategy shrink towards a shape with length ``min_dims``. - The size of an aligned dimension shrinks towards size ``1``. The size of an - unaligned dimension shrink towards ``min_side``. + The following are some examples drawn from this strategy. - * ``shape`` is a tuple of integers. - * ``min_dims`` is the smallest length that the generated shape can possess. - * ``max_dims`` is the largest length that the generated shape can possess, - defaulting to ``max(len(shape), min_dims) + 2``. - * ``min_side`` is the smallest size that an unaligned dimension can possess. - * ``max_side`` is the largest size that an unaligned dimension can possess, - defaulting to 2 plus the size of the largest aligned dimension. + .. code-block:: pycon - The following are some examples drawn from this strategy. + >>> [broadcastable_shapes(shape=(2, 3)).example() for i in range(5)] + [(1, 3), (), (2, 3), (2, 1), (4, 1, 3), (3, )] - .. code-block:: pycon + """ + check_type(tuple, shape, "shape") + check_type(int, min_side, "min_side") + check_type(int, min_dims, "min_dims") + check_valid_dims(min_dims, "min_dims") + + strict_check = max_side is None or max_dims is None + + if max_dims is None: + max_dims = min(max(len(shape), min_dims) + 2, NDIM_MAX) + check_type(int, max_dims, "max_dims") + check_valid_dims(max_dims, "max_dims") + + if max_side is None: + max_side = max(shape[-max_dims:] + (min_side,)) + 2 + check_type(int, max_side, "max_side") + + order_check("dims", 0, min_dims, max_dims) + order_check("side", 0, min_side, max_side) + + if strict_check: + dims = max_dims + bound_name = "max_dims" + else: + dims = min_dims + bound_name = "min_dims" + + # check for unsatisfiable min_side + if not all(min_side <= s for s in shape[::-1][:dims] if s != 1): + raise InvalidArgument( + f"Given shape={shape}, there are no broadcast-compatible " + f"shapes that satisfy: {bound_name}={dims} and min_side={min_side}" + ) - >>> [broadcastable_shapes(shape=(2, 3)).example() for i in range(5)] - [(1, 3), (), (2, 3), (2, 1), (4, 1, 3), (3, )] + # check for unsatisfiable [min_side, max_side] + if not ( + min_side <= 1 <= max_side or all(s <= max_side for s in shape[::-1][:dims]) + ): + raise InvalidArgument( + f"Given base_shape={shape}, there are no broadcast-compatible " + f"shapes that satisfy all of {bound_name}={dims}, " + f"min_side={min_side}, and max_side={max_side}" + ) - """ - check_type(tuple, shape, "shape") - check_type(int, min_side, "min_side") - check_type(int, min_dims, "min_dims") - check_valid_dims(lib, "min_dims", min_dims) + if not strict_check: + # reduce max_dims to exclude unsatisfiable dimensions + for n, s in zip(range(max_dims), shape[::-1]): + if s < min_side and s != 1: + max_dims = n + break + elif not (min_side <= 1 <= max_side or s <= max_side): + max_dims = n + break - strict_check = max_side is None or max_dims is None + return MutuallyBroadcastableShapesStrategy( + num_shapes=1, + base_shape=shape, + min_dims=min_dims, + max_dims=max_dims, + min_side=min_side, + max_side=max_side, + ).map(lambda x: x.input_shapes[0]) - if max_dims is None: - max_dims = min(max(len(shape), min_dims) + 2, lib_max_dims[lib]) - check_type(int, max_dims, "max_dims") - check_valid_dims(lib, "max_dims", max_dims) - if max_side is None: - max_side = max(shape[-max_dims:] + (min_side,)) + 2 - check_type(int, max_side, "max_side") +@defines_strategy() +def mutually_broadcastable_shapes( + num_shapes: int, + *, + signature: Optional[BroadcastableShapes] = None, + base_shape: Shape = (), + min_dims: int = 0, + max_dims: Optional[int] = None, + min_side: int = 1, + max_side: Optional[int] = None, +) -> st.SearchStrategy[BroadcastableShapes]: + """Return a strategy for a specified number of shapes N that are + mutually-broadcastable with one another and with the provided base shape. + + * ``num_shapes`` is the number of mutually broadcast-compatible shapes to generate. + * ``base_shape`` is the shape against which all generated shapes can broadcast. + The default shape is empty, which corresponds to a scalar and thus does + not constrain broadcasting at all. + * ``shape`` is a tuple of integers. + * ``min_dims`` is the smallest length that the generated shape can possess. + * ``max_dims`` is the largest length that the generated shape can possess, + defaulting to ``max(len(shape), min_dims) + 2``. + * ``min_side`` is the smallest size that an unaligned dimension can possess. + * ``max_side`` is the largest size that an unaligned dimension can possess, + defaulting to 2 plus the size of the largest aligned dimension. + + The strategy will generate a :obj:`python:typing.NamedTuple` containing: + + * ``input_shapes`` as a tuple of the N generated shapes. + * ``result_shape`` as the resulting shape produced by broadcasting the N shapes + with the base shape. - order_check("dims", 0, min_dims, max_dims) - order_check("side", 0, min_side, max_side) + The following are some examples drawn from this strategy. - if strict_check: - dims = max_dims - bound_name = "max_dims" - else: - dims = min_dims - bound_name = "min_dims" + .. code-block:: pycon - # check for unsatisfiable min_side - if not all(min_side <= s for s in shape[::-1][:dims] if s != 1): - raise InvalidArgument( - f"Given shape={shape}, there are no broadcast-compatible " - f"shapes that satisfy: {bound_name}={dims} and min_side={min_side}" - ) + >>> # Draw three shapes where each shape is broadcast-compatible with (2, 3) + ... strat = mutually_broadcastable_shapes(num_shapes=3, base_shape=(2, 3)) + >>> for _ in range(5): + ... print(strat.example()) + BroadcastableShapes(input_shapes=((4, 1, 3), (4, 2, 3), ()), result_shape=(4, 2, 3)) + BroadcastableShapes(input_shapes=((3,), (1, 3), (2, 3)), result_shape=(2, 3)) + BroadcastableShapes(input_shapes=((), (), ()), result_shape=(2, 3)) + BroadcastableShapes(input_shapes=((3,), (), (3,)), result_shape=(2, 3)) + BroadcastableShapes(input_shapes=((1, 2, 3), (3,), ()), result_shape=(1, 2, 3)) - # check for unsatisfiable [min_side, max_side] - if not ( - min_side <= 1 <= max_side or all(s <= max_side for s in shape[::-1][:dims]) - ): - raise InvalidArgument( - f"Given base_shape={shape}, there are no broadcast-compatible " - f"shapes that satisfy all of {bound_name}={dims}, " - f"min_side={min_side}, and max_side={max_side}" - ) + """ - if not strict_check: - # reduce max_dims to exclude unsatisfiable dimensions - for n, s in zip(range(max_dims), shape[::-1]): - if s < min_side and s != 1: - max_dims = n - break - elif not (min_side <= 1 <= max_side or s <= max_side): - max_dims = n - break - - return MutuallyBroadcastableShapesStrategy( - num_shapes=1, - base_shape=shape, - min_dims=min_dims, - max_dims=max_dims, - min_side=min_side, - max_side=max_side, - ).map(lambda x: x.input_shapes[0]) + if signature is None: + sig_dims = 0 + else: + all_shapes = signature.input_shapes + (signature.result_shape,) + sig_dims = min(len(s) for s in all_shapes) - return broadcastable_shapes + check_type(int, num_shapes, "num_shapes") + if num_shapes < 1: + raise InvalidArgument(f"num_shapes={num_shapes} must be at least 1") + check_type(tuple, base_shape, "base_shape") + check_type(int, min_side, "min_side") + check_type(int, min_dims, "min_dims") + check_valid_dims(min_dims, "min_dims") -def make_mutually_broadcastable_shapes(lib: Optional[str] = None) -> Callable: - @defines_strategy() - def mutually_broadcastable_shapes( - num_shapes: int, - *, - signature: Optional[BroadcastableShapes] = None, - base_shape: Shape = (), - min_dims: int = 0, - max_dims: Optional[int] = None, - min_side: int = 1, - max_side: Optional[int] = None, - ) -> st.SearchStrategy[BroadcastableShapes]: - """Return a strategy for a specified number of shapes N that are - mutually-broadcastable with one another and with the provided base shape. - - * ``num_shapes`` is the number of mutually broadcast-compatible shapes to generate. - * ``base_shape`` is the shape against which all generated shapes can broadcast. - The default shape is empty, which corresponds to a scalar and thus does - not constrain broadcasting at all. - * ``shape`` is a tuple of integers. - * ``min_dims`` is the smallest length that the generated shape can possess. - * ``max_dims`` is the largest length that the generated shape can possess, - defaulting to ``max(len(shape), min_dims) + 2``. - * ``min_side`` is the smallest size that an unaligned dimension can possess. - * ``max_side`` is the largest size that an unaligned dimension can possess, - defaulting to 2 plus the size of the largest aligned dimension. - - The strategy will generate a :obj:`python:typing.NamedTuple` containing: - - * ``input_shapes`` as a tuple of the N generated shapes. - * ``result_shape`` as the resulting shape produced by broadcasting the N shapes - with the base shape. - - The following are some examples drawn from this strategy. - - .. code-block:: pycon - - >>> # Draw three shapes where each shape is broadcast-compatible with (2, 3) - ... strat = mutually_broadcastable_shapes(num_shapes=3, base_shape=(2, 3)) - >>> for _ in range(5): - ... print(strat.example()) - BroadcastableShapes(input_shapes=((4, 1, 3), (4, 2, 3), ()), result_shape=(4, 2, 3)) - BroadcastableShapes(input_shapes=((3,), (1, 3), (2, 3)), result_shape=(2, 3)) - BroadcastableShapes(input_shapes=((), (), ()), result_shape=(2, 3)) - BroadcastableShapes(input_shapes=((3,), (), (3,)), result_shape=(2, 3)) - BroadcastableShapes(input_shapes=((1, 2, 3), (3,), ()), result_shape=(1, 2, 3)) - - """ - - if signature is None: - sig_dims = 0 - else: - sig_dims = min( - len(s) for s in signature.input_shapes + (signature.result_shape,) - ) + strict_check = max_dims is not None - check_type(int, num_shapes, "num_shapes") - if num_shapes < 1: - raise InvalidArgument(f"num_shapes={num_shapes} must be at least 1") + if max_dims is None: + max_dims = min(max(len(base_shape), min_dims) + 2, NDIM_MAX - sig_dims) + check_type(int, max_dims, "max_dims") + check_valid_dims(max_dims, "max_dims") - check_type(tuple, base_shape, "base_shape") - check_type(int, min_side, "min_side") - check_type(int, min_dims, "min_dims") - check_valid_dims(lib, "min_dims", min_dims) + if max_side is None: + max_side = max(base_shape[-max_dims:] + (min_side,)) + 2 + check_type(int, max_side, "max_side") - strict_check = max_dims is not None + order_check("dims", 0, min_dims, max_dims) + order_check("side", 0, min_side, max_side) - if max_dims is None: - max_dims = min( - max(len(base_shape), min_dims) + 2, lib_max_dims[lib] - sig_dims - ) - check_type(int, max_dims, "max_dims") - check_valid_dims(lib, "max_dims", max_dims) + if signature is not None and max_dims > NDIM_MAX - sig_dims: + raise InvalidArgument( + f"max_dims={max_dims} would exceed the {NDIM_MAX}-dimension " + "limit Hypothesis imposes on array shapes, given signature=" + f"(input_shapes={signature.input_shapes}," + f" result_shape={signature.result_shape})" + ) - if max_side is None: - max_side = max(base_shape[-max_dims:] + (min_side,)) + 2 - check_type(int, max_side, "max_side") + if strict_check: + dims = max_dims + bound_name = "max_dims" + else: + dims = min_dims + bound_name = "min_dims" - order_check("dims", 0, min_dims, max_dims) - order_check("side", 0, min_side, max_side) + # check for unsatisfiable min_side + if not all(min_side <= s for s in base_shape[::-1][:dims] if s != 1): + raise InvalidArgument( + f"Given base_shape={base_shape}, there are no broadcast-compatible " + f"shapes that satisfy: {bound_name}={dims} and min_side={min_side}" + ) - if lib_max_dims[lib] - sig_dims < max_dims: - if signature is None: - raise InvalidArgument(f"max_dims cannot exceed {lib_max_dims[lib]}") - raise InvalidArgument( - f"max_dims={max_dims} would exceed the {lib_max_dims[lib]}-dimension " - f"limit given signature=(input_shapes={signature.input_shapes}, " - f"result_shape={signature.result_shape})" - ) + # check for unsatisfiable [min_side, max_side] + if not ( + min_side <= 1 <= max_side or all(s <= max_side for s in base_shape[::-1][:dims]) + ): + raise InvalidArgument( + f"Given base_shape={base_shape}, there are no broadcast-compatible " + f"shapes that satisfy all of {bound_name}={dims}, " + f"min_side={min_side}, and max_side={max_side}" + ) - if strict_check: - dims = max_dims - bound_name = "max_dims" - else: - dims = min_dims - bound_name = "min_dims" + if not strict_check: + # reduce max_dims to exclude unsatisfiable dimensions + for n, s in zip(range(max_dims), base_shape[::-1]): + if s < min_side and s != 1: + max_dims = n + break + elif not (min_side <= 1 <= max_side or s <= max_side): + max_dims = n + break - # check for unsatisfiable min_side - if not all(min_side <= s for s in base_shape[::-1][:dims] if s != 1): - raise InvalidArgument( - f"Given base_shape={base_shape}, there are no broadcast-compatible " - f"shapes that satisfy: {bound_name}={dims} and min_side={min_side}" - ) + return MutuallyBroadcastableShapesStrategy( + num_shapes=num_shapes, + signature=signature, + base_shape=base_shape, + min_dims=min_dims, + max_dims=max_dims, + min_side=min_side, + max_side=max_side, + ) - # check for unsatisfiable [min_side, max_side] - if not ( - min_side <= 1 <= max_side - or all(s <= max_side for s in base_shape[::-1][:dims]) - ): - raise InvalidArgument( - f"Given base_shape={base_shape}, there are no broadcast-compatible " - f"shapes that satisfy all of {bound_name}={dims}, " - f"min_side={min_side}, and max_side={max_side}" - ) - if not strict_check: - # reduce max_dims to exclude unsatisfiable dimensions - for n, s in zip(range(max_dims), base_shape[::-1]): - if s < min_side and s != 1: - max_dims = n - break - elif not (min_side <= 1 <= max_side or s <= max_side): - max_dims = n - break - - return MutuallyBroadcastableShapesStrategy( - num_shapes=num_shapes, - signature=signature, - base_shape=base_shape, - min_dims=min_dims, - max_dims=max_dims, - min_side=min_side, - max_side=max_side, +def make_basic_indices(allow_0d_index: bool) -> Callable: + if allow_0d_index: + min_index_dim = 0 + min_dims_note = ( + "When ``min_dims == 0``, indices for zero-dimensional arrays are generated." + ) + else: + min_index_dim = 1 + min_dims_note = ( + "Indices for zero-dimensional arrays cannot be generated, " + "so ``min_dims`` must be greater than zero." ) - - return mutually_broadcastable_shapes - - -def make_basic_indices(lib: Optional[str] = None, *, allow_0d_index=False) -> Callable: - min_index_dim = 0 if allow_0d_index else 1 @defines_strategy() def basic_indices( @@ -394,22 +399,6 @@ def basic_indices( allow_newaxis: bool = False, allow_ellipsis: bool = True, ) -> st.SearchStrategy[BasicIndex]: - """It generates tuples containing some mix of integers, :obj:`python:slice` - objects, ``...`` (an ``Ellipsis``), and ``None``. When a length-one tuple - would be generated, this strategy may instead return the element which will - index the first axis, e.g. ``5`` instead of ``(5,)``. - - * ``shape`` is the shape of the array that will be indexed, as a tuple of - positive integers. This must be at least two-dimensional for a tuple to be a - valid index; for one-dimensional arrays use - :func:`~hypothesis.strategies.slices` instead. - * ``min_dims`` is the minimum dimensionality of the resulting array from use of - the generated index. - * ``max_dims`` is the the maximum dimensionality of the resulting array, - defaulting to ``max(len(shape), min_dims) + 2``. - * ``allow_newaxis`` specifies whether ``None`` is allowed in the index. - * ``allow_ellipsis`` specifies whether ``...`` is allowed in the index. - """ # Arguments to exclude scalars, zero-dim arrays, and dims of size zero were # all considered and rejected. We want users to explicitly consider those # cases if they're dealing in general indexers, and while it's fiddly we can @@ -420,16 +409,20 @@ def basic_indices( check_type(bool, allow_ellipsis, "allow_ellipsis") check_type(bool, allow_newaxis, "allow_newaxis") check_type(int, min_dims, "min_dims") - check_valid_dims(lib, "min_dims", min_dims) + check_valid_dims(min_dims, "min_dims") + if max_dims is None: - max_dims = min(max(len(shape), min_dims) + 2, lib_max_dims[lib]) + max_dims = min(max(len(shape), min_dims) + 2, NDIM_MAX) check_type(int, max_dims, "max_dims") - check_valid_dims(lib, "max_dims", max_dims) + check_valid_dims(max_dims, "max_dims") + order_check("dims", min_index_dim, min_dims, max_dims) + if not all(isinstance(x, int) and x >= 0 for x in shape): raise InvalidArgument( f"shape={shape!r}, but all dimensions must be of integer size >= 0" ) + return BasicIndexStrategy( shape, min_dims=min_dims, @@ -438,6 +431,24 @@ def basic_indices( allow_newaxis=allow_newaxis, ) + basic_indices.__doc__ = f""" + It generates tuples containing some mix of integers, :obj:`python:slice` + objects, ``...`` (an ``Ellipsis``), and ``None``. When a length-one tuple + would be generated, this strategy may instead return the element which will + index the first axis, e.g. ``5`` instead of ``(5,)``. + + * ``shape`` is the shape of the array that will be indexed, as a tuple of + positive integers. This must be at least two-dimensional for a tuple to be a + valid index; for one-dimensional arrays use + :func:`~hypothesis.strategies.slices` instead. + * ``min_dims`` is the minimum dimensionality of the resulting array from use of + the generated index. {min_dims_note} + * ``max_dims`` is the the maximum dimensionality of the resulting array, + defaulting to ``max(len(shape), min_dims) + 2``. + * ``allow_newaxis`` specifies whether ``None`` is allowed in the index. + * ``allow_ellipsis`` specifies whether ``...`` is allowed in the index. + """ + return basic_indices @@ -482,7 +493,7 @@ def do_draw(self, data): loop_in, loop_res = self._draw_loop_dimensions(data, use=use) def add_shape(loop, core): - return tuple(x for x in (loop + core)[-32:] if x is not None) + return tuple(x for x in (loop + core)[-NDIM_MAX:] if x is not None) return BroadcastableShapes( input_shapes=tuple(add_shape(l_in, c) for l_in, c in zip(loop_in, core_in)), diff --git a/hypothesis-python/src/hypothesis/extra/numpy.py b/hypothesis-python/src/hypothesis/extra/numpy.py index 122f2347aa..d3414e8310 100644 --- a/hypothesis-python/src/hypothesis/extra/numpy.py +++ b/hypothesis-python/src/hypothesis/extra/numpy.py @@ -24,10 +24,10 @@ from hypothesis.extra.__array_helpers import ( BroadcastableShapes, Shape, - make_array_shapes, + array_shapes, + broadcastable_shapes, make_basic_indices, - make_broadcastable_shapes, - make_mutually_broadcastable_shapes, + mutually_broadcastable_shapes as _mutually_broadcastable_shapes, order_check, valid_tuple_axes, ) @@ -56,7 +56,6 @@ "array_dtypes", "nested_dtypes", "valid_tuple_axes", - "valid_tuple_axes", "broadcastable_shapes", "mutually_broadcastable_shapes", "basic_indices", @@ -469,9 +468,6 @@ def arrays( return ArrayStrategy(elements, shape, dtype, fill, unique) -array_shapes = make_array_shapes("numpy") - - @defines_strategy() def scalar_dtypes() -> st.SearchStrategy[np.dtype]: """Return a strategy that can return any non-flexible scalar dtype.""" @@ -834,12 +830,6 @@ def _hypothesis_parse_gufunc_signature(signature, all_checks=True): return _GUfuncSig(input_shapes=input_shapes, result_shape=result_shape) -broadcastable_shapes = make_broadcastable_shapes("numpy") - - -_mutually_broadcastable_shapes = make_mutually_broadcastable_shapes("numpy") - - @defines_strategy() def mutually_broadcastable_shapes( *, @@ -920,14 +910,12 @@ def mutually_broadcastable_shapes( """ -basic_indices = make_basic_indices("numpy", allow_0d_index=True) +basic_indices = make_basic_indices(allow_0d_index=True) basic_indices.__doc__ = f""" Return a strategy for :np-ref:`basic indexes ` of arrays with the specified shape, which may include dimensions of size zero. {basic_indices.__doc__} - - Note if ``min_dims == 0``, indices for zero-dimensional arrays are generated. """ From fa3dfed3d0336274c86b7c24926d22ba65d69df3 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Fri, 27 Aug 2021 12:06:06 +0100 Subject: [PATCH 07/24] More explicit import style for array helpers --- .../{__array_helpers.py => _array_helpers.py} | 0 .../src/hypothesis/extra/numpy.py | 25 +++++++++---------- 2 files changed, 12 insertions(+), 13 deletions(-) rename hypothesis-python/src/hypothesis/extra/{__array_helpers.py => _array_helpers.py} (100%) diff --git a/hypothesis-python/src/hypothesis/extra/__array_helpers.py b/hypothesis-python/src/hypothesis/extra/_array_helpers.py similarity index 100% rename from hypothesis-python/src/hypothesis/extra/__array_helpers.py rename to hypothesis-python/src/hypothesis/extra/_array_helpers.py diff --git a/hypothesis-python/src/hypothesis/extra/numpy.py b/hypothesis-python/src/hypothesis/extra/numpy.py index d3414e8310..7ded699b08 100644 --- a/hypothesis-python/src/hypothesis/extra/numpy.py +++ b/hypothesis-python/src/hypothesis/extra/numpy.py @@ -21,16 +21,8 @@ from hypothesis import strategies as st from hypothesis.errors import InvalidArgument -from hypothesis.extra.__array_helpers import ( - BroadcastableShapes, - Shape, - array_shapes, - broadcastable_shapes, - make_basic_indices, - mutually_broadcastable_shapes as _mutually_broadcastable_shapes, - order_check, - valid_tuple_axes, -) +from hypothesis.extra import _array_helpers +from hypothesis.extra._array_helpers import BroadcastableShapes, Shape, order_check from hypothesis.internal.conjecture import utils as cu from hypothesis.internal.coverage import check_function from hypothesis.internal.reflection import proxies @@ -468,6 +460,9 @@ def arrays( return ArrayStrategy(elements, shape, dtype, fill, unique) +array_shapes = _array_helpers.array_shapes + + @defines_strategy() def scalar_dtypes() -> st.SearchStrategy[np.dtype]: """Return a strategy that can return any non-flexible scalar dtype.""" @@ -736,6 +731,7 @@ def nested_dtypes( ).filter(lambda d: max_itemsize is None or d.itemsize <= max_itemsize) +valid_tuple_axes = _array_helpers.valid_tuple_axes valid_tuple_axes.__doc__ = f""" Return a strategy for generating permissible tuple-values for the ``axis`` argument for a numpy sequential function (e.g. @@ -745,6 +741,9 @@ def nested_dtypes( {valid_tuple_axes.__doc__} """ + +broadcastable_shapes = _array_helpers.broadcastable_shapes + # See https://numpy.org/doc/stable/reference/c-api/generalized-ufuncs.html # Implementation based on numpy.lib.function_base._parse_gufunc_signature # with minor upgrades to handle numeric and optional dimensions. Examples: @@ -860,7 +859,7 @@ def mutually_broadcastable_shapes( num_shapes = len(parsed_signature.input_shapes) assert num_shapes >= 1 - return _mutually_broadcastable_shapes( + return _array_helpers.mutually_broadcastable_shapes( num_shapes=num_shapes, signature=parsed_signature, base_shape=base_shape, @@ -872,7 +871,7 @@ def mutually_broadcastable_shapes( mutually_broadcastable_shapes.__doc__ = f""" - {_mutually_broadcastable_shapes.__doc__} + {_array_helpers.mutually_broadcastable_shapes.__doc__} **Use with Generalised Universal Function signatures** @@ -910,7 +909,7 @@ def mutually_broadcastable_shapes( """ -basic_indices = make_basic_indices(allow_0d_index=True) +basic_indices = _array_helpers.make_basic_indices(allow_0d_index=True) basic_indices.__doc__ = f""" Return a strategy for :np-ref:`basic indexes ` of arrays with the specified shape, which may include dimensions of size zero. From 18ae20c945f981f22b830d78d5c82e9f950a0ef3 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Fri, 27 Aug 2021 12:18:38 +0100 Subject: [PATCH 08/24] Add RELEASE.rst --- hypothesis-python/RELEASE.rst | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 hypothesis-python/RELEASE.rst diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 0000000000..8a5d4b203e --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,6 @@ +RELEASE_TYPE: patch + +This patch changes the internal structure of some strategies in the NumPy extra +which were not dependent on NumPy. They are moved to a separate private module +so that in the future Hypothesis can re-use these strategies for other purposes +(i.e. Array API support in :issue:`3065`). From 4b4ee5c2fbdd11cea694fa5802dd0b1c83570c46 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Fri, 27 Aug 2021 14:57:23 +0100 Subject: [PATCH 09/24] Have array helpers responsible for ufunc behaviour --- .../src/hypothesis/extra/_array_helpers.py | 126 ++++++++++++++-- .../src/hypothesis/extra/numpy.py | 141 +----------------- .../tests/numpy/test_argument_validation.py | 1 + hypothesis-python/tests/numpy/test_gufunc.py | 18 ++- 4 files changed, 131 insertions(+), 155 deletions(-) diff --git a/hypothesis-python/src/hypothesis/extra/_array_helpers.py b/hypothesis-python/src/hypothesis/extra/_array_helpers.py index c02bffbced..2973cfa688 100644 --- a/hypothesis-python/src/hypothesis/extra/_array_helpers.py +++ b/hypothesis-python/src/hypothesis/extra/_array_helpers.py @@ -13,6 +13,7 @@ # # END HEADER +import re from typing import Callable, NamedTuple, Optional, Tuple, Union from hypothesis import assume, strategies as st @@ -21,11 +22,13 @@ from hypothesis.internal.coverage import check_function from hypothesis.internal.validation import check_type, check_valid_interval from hypothesis.strategies._internal.utils import defines_strategy +from hypothesis.utils.conventions import UniqueIdentifier, not_set __all__ = [ "Shape", "BroadcastableShapes", "BasicIndex", + "check_argument", "order_check", "array_shapes", "valid_tuple_axes", @@ -45,6 +48,12 @@ class BroadcastableShapes(NamedTuple): result_shape: Shape +@check_function +def check_argument(condition, fail_message, *f_args, **f_kwargs): + if not condition: + raise InvalidArgument(fail_message.format(*f_args, **f_kwargs)) + + @check_function def order_check(name, floor, min_, max_): if floor > min_: @@ -247,11 +256,95 @@ def broadcastable_shapes( ).map(lambda x: x.input_shapes[0]) +# See https://numpy.org/doc/stable/reference/c-api/generalized-ufuncs.html +# Implementation based on numpy.lib.function_base._parse_gufunc_signature +# with minor upgrades to handle numeric and optional dimensions. Examples: +# +# add (),()->() binary ufunc +# sum1d (i)->() reduction +# inner1d (i),(i)->() vector-vector multiplication +# matmat (m,n),(n,p)->(m,p) matrix multiplication +# vecmat (n),(n,p)->(p) vector-matrix multiplication +# matvec (m,n),(n)->(m) matrix-vector multiplication +# matmul (m?,n),(n,p?)->(m?,p?) combination of the four above +# cross1d (3),(3)->(3) cross product with frozen dimensions +# +# Note that while no examples of such usage are given, Numpy does allow +# generalised ufuncs that have *multiple output arrays*. This is not +# currently supported by Hypothesis - please contact us if you would use it! +# +# We are unsure if gufuncs allow frozen dimensions to be optional, but it's +# easy enough to support here - and so we will unless we learn otherwise. +_DIMENSION = r"\w+\??" # Note that \w permits digits too! +_SHAPE = r"\((?:{0}(?:,{0})".format(_DIMENSION) + r"{0,31})?\)" +_ARGUMENT_LIST = "{0}(?:,{0})*".format(_SHAPE) +_SIGNATURE = fr"^{_ARGUMENT_LIST}->{_SHAPE}$" +_SIGNATURE_MULTIPLE_OUTPUT = r"^{0}->{0}$".format(_ARGUMENT_LIST) + + +class _GUfuncSig(NamedTuple): + input_shapes: Tuple[Shape, ...] + result_shape: Shape + + +def _hypothesis_parse_gufunc_signature(signature, all_checks=True): + # Disable all_checks to better match the Numpy version, for testing + if not re.match(_SIGNATURE, signature): + if re.match(_SIGNATURE_MULTIPLE_OUTPUT, signature): + raise InvalidArgument( + "Hypothesis does not yet support generalised ufunc signatures " + "with multiple output arrays - mostly because we don't know of " + "anyone who uses them! Please get in touch with us to fix that." + f"\n (signature={signature!r})" + ) + if re.match("^{0:}->{0:}$".format(_ARGUMENT_LIST), signature): + raise InvalidArgument( + f"signature={signature!r} matches Numpy's regex for gufunc signatures, " + "but contains shapes with more than 32 dimensions and is thus invalid." + ) + raise InvalidArgument(f"{signature!r} is not a valid gufunc signature") + input_shapes, output_shapes = ( + tuple(tuple(re.findall(_DIMENSION, a)) for a in re.findall(_SHAPE, arg_list)) + for arg_list in signature.split("->") + ) + assert len(output_shapes) == 1 + result_shape = output_shapes[0] + if all_checks: + # Check that there are no names in output shape that do not appear in inputs. + # (kept out of parser function for easier generation of test values) + # We also disallow frozen optional dimensions - this is ambiguous as there is + # no way to share an un-named dimension between shapes. Maybe just padding? + # Anyway, we disallow it pending clarification from upstream. + frozen_optional_err = ( + "Got dimension %r, but handling of frozen optional dimensions " + "is ambiguous. If you known how this should work, please " + "contact us to get this fixed and documented (signature=%r)." + ) + only_out_err = ( + "The %r dimension only appears in the output shape, and is " + "not frozen, so the size is not determined (signature=%r)." + ) + names_in = {n.strip("?") for shp in input_shapes for n in shp} + names_out = {n.strip("?") for n in result_shape} + for shape in input_shapes + (result_shape,): + for name in shape: + try: + int(name.strip("?")) + if "?" in name: + raise InvalidArgument(frozen_optional_err % (name, signature)) + except ValueError: + if name.strip("?") in (names_out - names_in): + raise InvalidArgument( + only_out_err % (name, signature) + ) from None + return _GUfuncSig(input_shapes=input_shapes, result_shape=result_shape) + + @defines_strategy() def mutually_broadcastable_shapes( - num_shapes: int, *, - signature: Optional[BroadcastableShapes] = None, + num_shapes: Union[UniqueIdentifier, int] = not_set, + signature: Union[UniqueIdentifier, str] = not_set, base_shape: Shape = (), min_dims: int = 0, max_dims: Optional[int] = None, @@ -294,14 +387,26 @@ def mutually_broadcastable_shapes( BroadcastableShapes(input_shapes=((1, 2, 3), (3,), ()), result_shape=(1, 2, 3)) """ - - if signature is None: + arg_msg = "Pass either the `num_shapes` or the `signature` argument, but not both." + if num_shapes is not not_set: + check_argument(signature is not_set, arg_msg) + check_type(int, num_shapes, "num_shapes") + assert isinstance(num_shapes, int) # for mypy + parsed_signature = None sig_dims = 0 else: - all_shapes = signature.input_shapes + (signature.result_shape,) + check_argument(signature is not not_set, arg_msg) + if signature is None: + raise InvalidArgument( + "Expected a string, but got invalid signature=None. " + "(maybe .signature attribute of an element-wise ufunc?)" + ) + check_type(str, signature, "signature") + parsed_signature = _hypothesis_parse_gufunc_signature(signature) + all_shapes = parsed_signature.input_shapes + (parsed_signature.result_shape,) sig_dims = min(len(s) for s in all_shapes) + num_shapes = len(parsed_signature.input_shapes) - check_type(int, num_shapes, "num_shapes") if num_shapes < 1: raise InvalidArgument(f"num_shapes={num_shapes} must be at least 1") @@ -326,10 +431,9 @@ def mutually_broadcastable_shapes( if signature is not None and max_dims > NDIM_MAX - sig_dims: raise InvalidArgument( - f"max_dims={max_dims} would exceed the {NDIM_MAX}-dimension " - "limit Hypothesis imposes on array shapes, given signature=" - f"(input_shapes={signature.input_shapes}," - f" result_shape={signature.result_shape})" + f"max_dims={signature!r} would exceed the {NDIM_MAX}-dimension" + "limit Hypothesis imposes on array shapes, " + f"given signature={parsed_signature!r}" ) if strict_check: @@ -368,7 +472,7 @@ def mutually_broadcastable_shapes( return MutuallyBroadcastableShapesStrategy( num_shapes=num_shapes, - signature=signature, + signature=parsed_signature, base_shape=base_shape, min_dims=min_dims, max_dims=max_dims, diff --git a/hypothesis-python/src/hypothesis/extra/numpy.py b/hypothesis-python/src/hypothesis/extra/numpy.py index 7ded699b08..478e8ba78f 100644 --- a/hypothesis-python/src/hypothesis/extra/numpy.py +++ b/hypothesis-python/src/hypothesis/extra/numpy.py @@ -14,22 +14,20 @@ # END HEADER import math -import re -from typing import Any, Mapping, NamedTuple, Optional, Sequence, Tuple, Union +from typing import Any, Mapping, Optional, Sequence, Tuple, Union import numpy as np from hypothesis import strategies as st from hypothesis.errors import InvalidArgument from hypothesis.extra import _array_helpers -from hypothesis.extra._array_helpers import BroadcastableShapes, Shape, order_check +from hypothesis.extra._array_helpers import Shape, check_argument, order_check from hypothesis.internal.conjecture import utils as cu from hypothesis.internal.coverage import check_function from hypothesis.internal.reflection import proxies from hypothesis.internal.validation import check_type from hypothesis.strategies._internal.strategies import T, check_strategy from hypothesis.strategies._internal.utils import defines_strategy -from hypothesis.utils.conventions import UniqueIdentifier, not_set __all__ = [ "from_dtype", @@ -165,12 +163,6 @@ def compat_kw(*args, **kw): return result.map(dtype.type) -@check_function -def check_argument(condition, fail_message, *f_args, **f_kwargs): - if not condition: - raise InvalidArgument(fail_message.format(*f_args, **f_kwargs)) - - class ArrayStrategy(st.SearchStrategy): def __init__(self, element_strategy, shape, dtype, fill, unique): self.shape = tuple(shape) @@ -744,134 +736,9 @@ def nested_dtypes( broadcastable_shapes = _array_helpers.broadcastable_shapes -# See https://numpy.org/doc/stable/reference/c-api/generalized-ufuncs.html -# Implementation based on numpy.lib.function_base._parse_gufunc_signature -# with minor upgrades to handle numeric and optional dimensions. Examples: -# -# add (),()->() binary ufunc -# sum1d (i)->() reduction -# inner1d (i),(i)->() vector-vector multiplication -# matmat (m,n),(n,p)->(m,p) matrix multiplication -# vecmat (n),(n,p)->(p) vector-matrix multiplication -# matvec (m,n),(n)->(m) matrix-vector multiplication -# matmul (m?,n),(n,p?)->(m?,p?) combination of the four above -# cross1d (3),(3)->(3) cross product with frozen dimensions -# -# Note that while no examples of such usage are given, Numpy does allow -# generalised ufuncs that have *multiple output arrays*. This is not -# currently supported by Hypothesis - please contact us if you would use it! -# -# We are unsure if gufuncs allow frozen dimensions to be optional, but it's -# easy enough to support here - and so we will unless we learn otherwise. -# -_DIMENSION = r"\w+\??" # Note that \w permits digits too! -_SHAPE = r"\((?:{0}(?:,{0})".format(_DIMENSION) + r"{0,31})?\)" -_ARGUMENT_LIST = "{0}(?:,{0})*".format(_SHAPE) -_SIGNATURE = fr"^{_ARGUMENT_LIST}->{_SHAPE}$" -_SIGNATURE_MULTIPLE_OUTPUT = r"^{0}->{0}$".format(_ARGUMENT_LIST) - - -class _GUfuncSig(NamedTuple): - input_shapes: Tuple[Shape, ...] - result_shape: Shape - - -def _hypothesis_parse_gufunc_signature(signature, all_checks=True): - # Disable all_checks to better match the Numpy version, for testing - if not re.match(_SIGNATURE, signature): - if re.match(_SIGNATURE_MULTIPLE_OUTPUT, signature): - raise InvalidArgument( - "Hypothesis does not yet support generalised ufunc signatures " - "with multiple output arrays - mostly because we don't know of " - "anyone who uses them! Please get in touch with us to fix that." - f"\n (signature={signature!r})" - ) - if re.match(np.lib.function_base._SIGNATURE, signature): - raise InvalidArgument( - f"signature={signature!r} matches Numpy's regex for gufunc signatures, " - "but contains shapes with more than 32 dimensions and is thus invalid." - ) - raise InvalidArgument(f"{signature!r} is not a valid gufunc signature") - input_shapes, output_shapes = ( - tuple(tuple(re.findall(_DIMENSION, a)) for a in re.findall(_SHAPE, arg_list)) - for arg_list in signature.split("->") - ) - assert len(output_shapes) == 1 - result_shape = output_shapes[0] - if all_checks: - # Check that there are no names in output shape that do not appear in inputs. - # (kept out of parser function for easier generation of test values) - # We also disallow frozen optional dimensions - this is ambiguous as there is - # no way to share an un-named dimension between shapes. Maybe just padding? - # Anyway, we disallow it pending clarification from upstream. - frozen_optional_err = ( - "Got dimension %r, but handling of frozen optional dimensions " - "is ambiguous. If you known how this should work, please " - "contact us to get this fixed and documented (signature=%r)." - ) - only_out_err = ( - "The %r dimension only appears in the output shape, and is " - "not frozen, so the size is not determined (signature=%r)." - ) - names_in = {n.strip("?") for shp in input_shapes for n in shp} - names_out = {n.strip("?") for n in result_shape} - for shape in input_shapes + (result_shape,): - for name in shape: - try: - int(name.strip("?")) - if "?" in name: - raise InvalidArgument(frozen_optional_err % (name, signature)) - except ValueError: - if name.strip("?") in (names_out - names_in): - raise InvalidArgument( - only_out_err % (name, signature) - ) from None - return _GUfuncSig(input_shapes=input_shapes, result_shape=result_shape) - - -@defines_strategy() -def mutually_broadcastable_shapes( - *, - num_shapes: Union[UniqueIdentifier, int] = not_set, - signature: Union[UniqueIdentifier, str] = not_set, - base_shape: Shape = (), - min_dims: int = 0, - max_dims: Optional[int] = None, - min_side: int = 1, - max_side: Optional[int] = None, -) -> st.SearchStrategy[BroadcastableShapes]: - arg_msg = "Pass either the `num_shapes` or the `signature` argument, but not both." - if num_shapes is not not_set: - check_argument(signature is not_set, arg_msg) - check_type(int, num_shapes, "num_shapes") - assert isinstance(num_shapes, int) # for mypy - check_argument(num_shapes >= 1, "num_shapes={} must be at least 1", num_shapes) - parsed_signature = None - else: - check_argument(signature is not not_set, arg_msg) - if signature is None: - raise InvalidArgument( - "Expected a string, but got invalid signature=None. " - "(maybe .signature attribute of an element-wise ufunc?)" - ) - check_type(str, signature, "signature") - parsed_signature = _hypothesis_parse_gufunc_signature(signature) - num_shapes = len(parsed_signature.input_shapes) - assert num_shapes >= 1 - - return _array_helpers.mutually_broadcastable_shapes( - num_shapes=num_shapes, - signature=parsed_signature, - base_shape=base_shape, - min_dims=min_dims, - max_dims=max_dims, - min_side=min_side, - max_side=max_side, - ) - - +mutually_broadcastable_shapes = _array_helpers.mutually_broadcastable_shapes mutually_broadcastable_shapes.__doc__ = f""" - {_array_helpers.mutually_broadcastable_shapes.__doc__} + {mutually_broadcastable_shapes.__doc__} **Use with Generalised Universal Function signatures** diff --git a/hypothesis-python/tests/numpy/test_argument_validation.py b/hypothesis-python/tests/numpy/test_argument_validation.py index b291fbd8af..b9ba13c8e6 100644 --- a/hypothesis-python/tests/numpy/test_argument_validation.py +++ b/hypothesis-python/tests/numpy/test_argument_validation.py @@ -267,6 +267,7 @@ def e(a, **kwargs): e(nps.basic_indices, shape=(0, 0), max_dims=-1), e(nps.basic_indices, shape=(0, 0), max_dims=1.0), e(nps.basic_indices, shape=(0, 0), min_dims=2, max_dims=1), + e(nps.basic_indices, shape=(0, 0), min_dims=50), e(nps.basic_indices, shape=(0, 0), max_dims=50), e(nps.integer_array_indices, shape=()), e(nps.integer_array_indices, shape=(2, 0)), diff --git a/hypothesis-python/tests/numpy/test_gufunc.py b/hypothesis-python/tests/numpy/test_gufunc.py index 6130688cb9..cd051176ee 100644 --- a/hypothesis-python/tests/numpy/test_gufunc.py +++ b/hypothesis-python/tests/numpy/test_gufunc.py @@ -20,6 +20,10 @@ from hypothesis import example, given, note, settings, strategies as st from hypothesis.errors import InvalidArgument from hypothesis.extra import numpy as nps +from hypothesis.extra._array_helpers import ( + _SIGNATURE, + _hypothesis_parse_gufunc_signature, +) from tests.common.debug import find_any, minimal @@ -55,7 +59,7 @@ def test_numpy_signature_parses(sig): np_sig = np.lib.function_base._parse_gufunc_signature(sig) try: - hy_sig = nps._hypothesis_parse_gufunc_signature(sig, all_checks=False) + hy_sig = _hypothesis_parse_gufunc_signature(sig, all_checks=False) assert np_sig == hy_sig_2_np_sig(hy_sig) except InvalidArgument: shape_too_long = any(len(s) > 32 for s in np_sig[0] + np_sig[1]) @@ -67,14 +71,14 @@ def test_numpy_signature_parses(sig): sig = in_ + "->" + out.split(",(")[0] np_sig = np.lib.function_base._parse_gufunc_signature(sig) if all(len(s) <= 32 for s in np_sig[0] + np_sig[1]): - hy_sig = nps._hypothesis_parse_gufunc_signature(sig, all_checks=False) + hy_sig = _hypothesis_parse_gufunc_signature(sig, all_checks=False) assert np_sig == hy_sig_2_np_sig(hy_sig) @use_signature_examples -@given(st.from_regex(nps._SIGNATURE)) +@given(st.from_regex(_SIGNATURE)) def test_hypothesis_signature_parses(sig): - hy_sig = nps._hypothesis_parse_gufunc_signature(sig, all_checks=False) + hy_sig = _hypothesis_parse_gufunc_signature(sig, all_checks=False) try: np_sig = np.lib.function_base._parse_gufunc_signature(sig) assert np_sig == hy_sig_2_np_sig(hy_sig) @@ -82,13 +86,13 @@ def test_hypothesis_signature_parses(sig): assert "?" in sig # We can always fix this up, and it should then always validate. sig = sig.replace("?", "") - hy_sig = nps._hypothesis_parse_gufunc_signature(sig, all_checks=False) + hy_sig = _hypothesis_parse_gufunc_signature(sig, all_checks=False) np_sig = np.lib.function_base._parse_gufunc_signature(sig) assert np_sig == hy_sig_2_np_sig(hy_sig) def test_frozen_dims_signature(): - nps._hypothesis_parse_gufunc_signature("(2),(3)->(4)") + _hypothesis_parse_gufunc_signature("(2),(3)->(4)") @st.composite @@ -183,7 +187,7 @@ def einlabels(labels): assert "x" not in labels, "we reserve x for fixed-dimensions" return "..." + "".join(i if not i.isdigit() else "x" for i in labels) - gufunc_sig = nps._hypothesis_parse_gufunc_signature(gufunc_sig) + gufunc_sig = _hypothesis_parse_gufunc_signature(gufunc_sig) input_sig = ",".join(map(einlabels, gufunc_sig.input_shapes)) return input_sig + "->" + einlabels(gufunc_sig.result_shape) From eb34163a0735fd208f9221a1d4a6ae5d27fc6297 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Fri, 27 Aug 2021 19:32:09 +0100 Subject: [PATCH 10/24] Explicitly import BroadcastableShapes and BasicIndex into namespace --- hypothesis-python/src/hypothesis/extra/numpy.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/hypothesis-python/src/hypothesis/extra/numpy.py b/hypothesis-python/src/hypothesis/extra/numpy.py index 478e8ba78f..5754169606 100644 --- a/hypothesis-python/src/hypothesis/extra/numpy.py +++ b/hypothesis-python/src/hypothesis/extra/numpy.py @@ -21,7 +21,13 @@ from hypothesis import strategies as st from hypothesis.errors import InvalidArgument from hypothesis.extra import _array_helpers -from hypothesis.extra._array_helpers import Shape, check_argument, order_check +from hypothesis.extra._array_helpers import ( + BasicIndex, + BroadcastableShapes, + Shape, + check_argument, + order_check, +) from hypothesis.internal.conjecture import utils as cu from hypothesis.internal.coverage import check_function from hypothesis.internal.reflection import proxies @@ -30,6 +36,10 @@ from hypothesis.strategies._internal.utils import defines_strategy __all__ = [ + "Shape", + "BroadcastableShapes", + "BasicIndex", + "TIME_RESOLUTIONS", "from_dtype", "arrays", "array_shapes", From 9bd21b4b507f88c4435586c01166d3965ac42ae8 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Fri, 27 Aug 2021 20:26:01 +0100 Subject: [PATCH 11/24] Update ghostwriter example for np.matmul --- hypothesis-python/tests/ghostwriter/recorded/magic_gufunc.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hypothesis-python/tests/ghostwriter/recorded/magic_gufunc.txt b/hypothesis-python/tests/ghostwriter/recorded/magic_gufunc.txt index e1bfa36e0b..71c9b6b8e1 100644 --- a/hypothesis-python/tests/ghostwriter/recorded/magic_gufunc.txt +++ b/hypothesis-python/tests/ghostwriter/recorded/magic_gufunc.txt @@ -3,7 +3,7 @@ import numpy from hypothesis import given, strategies as st -from hypothesis.extra.numpy import mutually_broadcastable_shapes +from hypothesis.extra._array_helpers import mutually_broadcastable_shapes @given( From 42e08c60c94be179747efba17ce163945ce59a2f Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Fri, 27 Aug 2021 21:25:46 +0100 Subject: [PATCH 12/24] Define basic_indices inside NumPy extra Prior factory pattern obfuscates method signature --- .../src/hypothesis/extra/_array_helpers.py | 82 ++----------------- .../src/hypothesis/extra/numpy.py | 64 ++++++++++++++- 2 files changed, 65 insertions(+), 81 deletions(-) diff --git a/hypothesis-python/src/hypothesis/extra/_array_helpers.py b/hypothesis-python/src/hypothesis/extra/_array_helpers.py index 2973cfa688..34a768d5ad 100644 --- a/hypothesis-python/src/hypothesis/extra/_array_helpers.py +++ b/hypothesis-python/src/hypothesis/extra/_array_helpers.py @@ -14,7 +14,7 @@ # END HEADER import re -from typing import Callable, NamedTuple, Optional, Tuple, Union +from typing import NamedTuple, Optional, Tuple, Union from hypothesis import assume, strategies as st from hypothesis.errors import InvalidArgument @@ -25,16 +25,19 @@ from hypothesis.utils.conventions import UniqueIdentifier, not_set __all__ = [ + "NDIM_MAX", "Shape", "BroadcastableShapes", "BasicIndex", "check_argument", "order_check", + "check_valid_dims", "array_shapes", "valid_tuple_axes", "broadcastable_shapes", "mutually_broadcastable_shapes", - "make_basic_indices", + "MutuallyBroadcastableShapesStrategy", + "BasicIndexStrategy", ] @@ -481,81 +484,6 @@ def mutually_broadcastable_shapes( ) -def make_basic_indices(allow_0d_index: bool) -> Callable: - if allow_0d_index: - min_index_dim = 0 - min_dims_note = ( - "When ``min_dims == 0``, indices for zero-dimensional arrays are generated." - ) - else: - min_index_dim = 1 - min_dims_note = ( - "Indices for zero-dimensional arrays cannot be generated, " - "so ``min_dims`` must be greater than zero." - ) - - @defines_strategy() - def basic_indices( - shape: Shape, - *, - min_dims: int = min_index_dim, - max_dims: Optional[int] = None, - allow_newaxis: bool = False, - allow_ellipsis: bool = True, - ) -> st.SearchStrategy[BasicIndex]: - # Arguments to exclude scalars, zero-dim arrays, and dims of size zero were - # all considered and rejected. We want users to explicitly consider those - # cases if they're dealing in general indexers, and while it's fiddly we can - # back-compatibly add them later (hence using kwonlyargs). - check_type(tuple, shape, "shape") - if not allow_0d_index and len(shape) == 0: - raise InvalidArgument("Indices for 0-dimensional arrays are not allowed") - check_type(bool, allow_ellipsis, "allow_ellipsis") - check_type(bool, allow_newaxis, "allow_newaxis") - check_type(int, min_dims, "min_dims") - check_valid_dims(min_dims, "min_dims") - - if max_dims is None: - max_dims = min(max(len(shape), min_dims) + 2, NDIM_MAX) - check_type(int, max_dims, "max_dims") - check_valid_dims(max_dims, "max_dims") - - order_check("dims", min_index_dim, min_dims, max_dims) - - if not all(isinstance(x, int) and x >= 0 for x in shape): - raise InvalidArgument( - f"shape={shape!r}, but all dimensions must be of integer size >= 0" - ) - - return BasicIndexStrategy( - shape, - min_dims=min_dims, - max_dims=max_dims, - allow_ellipsis=allow_ellipsis, - allow_newaxis=allow_newaxis, - ) - - basic_indices.__doc__ = f""" - It generates tuples containing some mix of integers, :obj:`python:slice` - objects, ``...`` (an ``Ellipsis``), and ``None``. When a length-one tuple - would be generated, this strategy may instead return the element which will - index the first axis, e.g. ``5`` instead of ``(5,)``. - - * ``shape`` is the shape of the array that will be indexed, as a tuple of - positive integers. This must be at least two-dimensional for a tuple to be a - valid index; for one-dimensional arrays use - :func:`~hypothesis.strategies.slices` instead. - * ``min_dims`` is the minimum dimensionality of the resulting array from use of - the generated index. {min_dims_note} - * ``max_dims`` is the the maximum dimensionality of the resulting array, - defaulting to ``max(len(shape), min_dims) + 2``. - * ``allow_newaxis`` specifies whether ``None`` is allowed in the index. - * ``allow_ellipsis`` specifies whether ``...`` is allowed in the index. - """ - - return basic_indices - - class MutuallyBroadcastableShapesStrategy(st.SearchStrategy): def __init__( self, diff --git a/hypothesis-python/src/hypothesis/extra/numpy.py b/hypothesis-python/src/hypothesis/extra/numpy.py index 5754169606..760126302f 100644 --- a/hypothesis-python/src/hypothesis/extra/numpy.py +++ b/hypothesis-python/src/hypothesis/extra/numpy.py @@ -23,7 +23,9 @@ from hypothesis.extra import _array_helpers from hypothesis.extra._array_helpers import ( BasicIndex, + BasicIndexStrategy, BroadcastableShapes, + MutuallyBroadcastableShapesStrategy, Shape, check_argument, order_check, @@ -58,7 +60,9 @@ "valid_tuple_axes", "broadcastable_shapes", "mutually_broadcastable_shapes", + "MutuallyBroadcastableShapesStrategy", "basic_indices", + "BasicIndexStrategy", "integer_array_indices", ] @@ -786,13 +790,65 @@ def nested_dtypes( """ -basic_indices = _array_helpers.make_basic_indices(allow_0d_index=True) -basic_indices.__doc__ = f""" - Return a strategy for :np-ref:`basic indexes ` of + +@defines_strategy() +def basic_indices( + shape: Shape, + *, + min_dims: int = 0, + max_dims: Optional[int] = None, + allow_newaxis: bool = False, + allow_ellipsis: bool = True, +) -> st.SearchStrategy[BasicIndex]: + """Return a strategy for :np-ref:`basic indexes ` of arrays with the specified shape, which may include dimensions of size zero. - {basic_indices.__doc__} + It generates tuples containing some mix of integers, :obj:`python:slice` + objects, ``...`` (an ``Ellipsis``), and ``None``. When a length-one tuple + would be generated, this strategy may instead return the element which will + index the first axis, e.g. ``5`` instead of ``(5,)``. + + * ``shape`` is the shape of the array that will be indexed, as a tuple of + positive integers. This must be at least two-dimensional for a tuple to be + a valid index; for one-dimensional arrays use + :func:`~hypothesis.strategies.slices` instead. + * ``min_dims`` is the minimum dimensionality of the resulting array from use + of the generated index. When ``min_dims == 0``, indices for zero-dimensional + arrays are generated. + * ``max_dims`` is the the maximum dimensionality of the resulting array, + defaulting to ``max(len(shape), min_dims) + 2``. + * ``allow_newaxis`` specifies whether ``None`` is allowed in the index. + * ``allow_ellipsis`` specifies whether ``...`` is allowed in the index. """ + # Arguments to exclude scalars, zero-dim arrays, and dims of size zero were + # all considered and rejected. We want users to explicitly consider those + # cases if they're dealing in general indexers, and while it's fiddly we can + # back-compatibly add them later (hence using kwonlyargs). + check_type(tuple, shape, "shape") + check_type(bool, allow_ellipsis, "allow_ellipsis") + check_type(bool, allow_newaxis, "allow_newaxis") + check_type(int, min_dims, "min_dims") + _array_helpers.check_valid_dims(min_dims, "min_dims") + + if max_dims is None: + max_dims = min(max(len(shape), min_dims) + 2, _array_helpers.NDIM_MAX) + check_type(int, max_dims, "max_dims") + _array_helpers.check_valid_dims(max_dims, "max_dims") + + order_check("dims", 0, min_dims, max_dims) + + if not all(isinstance(x, int) and x >= 0 for x in shape): + raise InvalidArgument( + f"shape={shape!r}, but all dimensions must be of integer size >= 0" + ) + + return BasicIndexStrategy( + shape, + min_dims=min_dims, + max_dims=max_dims, + allow_ellipsis=allow_ellipsis, + allow_newaxis=allow_newaxis, + ) @defines_strategy() From eadec239b6494542b0418f1143dde18362e9fa9f Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Sat, 28 Aug 2021 09:45:54 +0100 Subject: [PATCH 13/24] More succint RELEASE.rst Co-authored-by: Zac Hatfield-Dodds --- hypothesis-python/RELEASE.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst index 8a5d4b203e..8f5ed5e9f4 100644 --- a/hypothesis-python/RELEASE.rst +++ b/hypothesis-python/RELEASE.rst @@ -1,6 +1,4 @@ RELEASE_TYPE: patch -This patch changes the internal structure of some strategies in the NumPy extra -which were not dependent on NumPy. They are moved to a separate private module -so that in the future Hypothesis can re-use these strategies for other purposes -(i.e. Array API support in :issue:`3065`). +This patch moves some internal helper code in preparation for :issue:`3065`. +There is no user-visible change, unless you depended on undocumented internals. From 5ca1affbbe77056d4b7e759ed0d37ad23f7454ed Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Sat, 28 Aug 2021 09:57:08 +0100 Subject: [PATCH 14/24] Removed redundant test case Co-authored-by: Ryan Soklaski --- hypothesis-python/tests/numpy/test_argument_validation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/hypothesis-python/tests/numpy/test_argument_validation.py b/hypothesis-python/tests/numpy/test_argument_validation.py index b9ba13c8e6..b291fbd8af 100644 --- a/hypothesis-python/tests/numpy/test_argument_validation.py +++ b/hypothesis-python/tests/numpy/test_argument_validation.py @@ -267,7 +267,6 @@ def e(a, **kwargs): e(nps.basic_indices, shape=(0, 0), max_dims=-1), e(nps.basic_indices, shape=(0, 0), max_dims=1.0), e(nps.basic_indices, shape=(0, 0), min_dims=2, max_dims=1), - e(nps.basic_indices, shape=(0, 0), min_dims=50), e(nps.basic_indices, shape=(0, 0), max_dims=50), e(nps.integer_array_indices, shape=()), e(nps.integer_array_indices, shape=(2, 0)), From 8e7117a58d05292659dfa50cf204a4e9b05356e6 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Sat, 28 Aug 2021 10:15:22 +0100 Subject: [PATCH 15/24] Clarify public API, use of deepcopy --- .../src/hypothesis/extra/numpy.py | 31 +++++++------------ 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/hypothesis-python/src/hypothesis/extra/numpy.py b/hypothesis-python/src/hypothesis/extra/numpy.py index 760126302f..42f91d73dc 100644 --- a/hypothesis-python/src/hypothesis/extra/numpy.py +++ b/hypothesis-python/src/hypothesis/extra/numpy.py @@ -14,21 +14,25 @@ # END HEADER import math +from copy import deepcopy from typing import Any, Mapping, Optional, Sequence, Tuple, Union import numpy as np from hypothesis import strategies as st from hypothesis.errors import InvalidArgument -from hypothesis.extra import _array_helpers from hypothesis.extra._array_helpers import ( + NDIM_MAX, BasicIndex, BasicIndexStrategy, - BroadcastableShapes, - MutuallyBroadcastableShapesStrategy, Shape, + array_shapes, + broadcastable_shapes, check_argument, + check_valid_dims, + mutually_broadcastable_shapes, order_check, + valid_tuple_axes, ) from hypothesis.internal.conjecture import utils as cu from hypothesis.internal.coverage import check_function @@ -38,10 +42,6 @@ from hypothesis.strategies._internal.utils import defines_strategy __all__ = [ - "Shape", - "BroadcastableShapes", - "BasicIndex", - "TIME_RESOLUTIONS", "from_dtype", "arrays", "array_shapes", @@ -60,9 +60,7 @@ "valid_tuple_axes", "broadcastable_shapes", "mutually_broadcastable_shapes", - "MutuallyBroadcastableShapesStrategy", "basic_indices", - "BasicIndexStrategy", "integer_array_indices", ] @@ -466,9 +464,6 @@ def arrays( return ArrayStrategy(elements, shape, dtype, fill, unique) -array_shapes = _array_helpers.array_shapes - - @defines_strategy() def scalar_dtypes() -> st.SearchStrategy[np.dtype]: """Return a strategy that can return any non-flexible scalar dtype.""" @@ -737,7 +732,7 @@ def nested_dtypes( ).filter(lambda d: max_itemsize is None or d.itemsize <= max_itemsize) -valid_tuple_axes = _array_helpers.valid_tuple_axes +valid_tuple_axes = deepcopy(valid_tuple_axes) valid_tuple_axes.__doc__ = f""" Return a strategy for generating permissible tuple-values for the ``axis`` argument for a numpy sequential function (e.g. @@ -748,9 +743,7 @@ def nested_dtypes( """ -broadcastable_shapes = _array_helpers.broadcastable_shapes - -mutually_broadcastable_shapes = _array_helpers.mutually_broadcastable_shapes +mutually_broadcastable_shapes = deepcopy(mutually_broadcastable_shapes) mutually_broadcastable_shapes.__doc__ = f""" {mutually_broadcastable_shapes.__doc__} @@ -828,12 +821,12 @@ def basic_indices( check_type(bool, allow_ellipsis, "allow_ellipsis") check_type(bool, allow_newaxis, "allow_newaxis") check_type(int, min_dims, "min_dims") - _array_helpers.check_valid_dims(min_dims, "min_dims") + check_valid_dims(min_dims, "min_dims") if max_dims is None: - max_dims = min(max(len(shape), min_dims) + 2, _array_helpers.NDIM_MAX) + max_dims = min(max(len(shape), min_dims) + 2, NDIM_MAX) check_type(int, max_dims, "max_dims") - _array_helpers.check_valid_dims(max_dims, "max_dims") + check_valid_dims(max_dims, "max_dims") order_check("dims", 0, min_dims, max_dims) From 675842659c61e3be7d1365b1fb823315e800fb64 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Sat, 28 Aug 2021 10:16:49 +0100 Subject: [PATCH 16/24] Specifically predicate np.newaxis (not None) in indices tests Co-authored-by: Zac Hatfield-Dodds --- hypothesis-python/tests/numpy/test_gen_data.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hypothesis-python/tests/numpy/test_gen_data.py b/hypothesis-python/tests/numpy/test_gen_data.py index 57e22d9afb..b3d95a6484 100644 --- a/hypothesis-python/tests/numpy/test_gen_data.py +++ b/hypothesis-python/tests/numpy/test_gen_data.py @@ -1053,8 +1053,8 @@ def test_advanced_integer_index_can_generate_any_pattern(shape, data): [ lambda ix: Ellipsis in ix, lambda ix: Ellipsis not in ix, - lambda ix: None in ix, - lambda ix: None not in ix, + lambda ix: np.newaxis in ix, + lambda ix: np.newaxis not in ix, ], ) def test_basic_indices_options(condition): @@ -1121,7 +1121,7 @@ def test_basic_indices_generate_valid_indexers( assert 0 <= len(indexer) <= len(shape) + int(allow_ellipsis) else: assert 1 <= len(shape) + int(allow_ellipsis) - assert None not in shape + assert np.newaxis not in shape if not allow_ellipsis: assert Ellipsis not in shape From d6d2b09e30b5d083ddf29f7b26a836c67966c560 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Sat, 28 Aug 2021 10:46:09 +0100 Subject: [PATCH 17/24] Clarified how axes and mutshape strategies are used with underscores --- hypothesis-python/src/hypothesis/extra/numpy.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/hypothesis-python/src/hypothesis/extra/numpy.py b/hypothesis-python/src/hypothesis/extra/numpy.py index 42f91d73dc..14a12e6ec1 100644 --- a/hypothesis-python/src/hypothesis/extra/numpy.py +++ b/hypothesis-python/src/hypothesis/extra/numpy.py @@ -30,9 +30,9 @@ broadcastable_shapes, check_argument, check_valid_dims, - mutually_broadcastable_shapes, + mutually_broadcastable_shapes as _mutually_broadcastable_shapes, order_check, - valid_tuple_axes, + valid_tuple_axes as _valid_tuple_axes, ) from hypothesis.internal.conjecture import utils as cu from hypothesis.internal.coverage import check_function @@ -732,20 +732,20 @@ def nested_dtypes( ).filter(lambda d: max_itemsize is None or d.itemsize <= max_itemsize) -valid_tuple_axes = deepcopy(valid_tuple_axes) +valid_tuple_axes = deepcopy(_valid_tuple_axes) valid_tuple_axes.__doc__ = f""" Return a strategy for generating permissible tuple-values for the ``axis`` argument for a numpy sequential function (e.g. :func:`numpy:numpy.sum`), given an array of the specified dimensionality. - {valid_tuple_axes.__doc__} + {_valid_tuple_axes.__doc__} """ -mutually_broadcastable_shapes = deepcopy(mutually_broadcastable_shapes) +mutually_broadcastable_shapes = deepcopy(_mutually_broadcastable_shapes) mutually_broadcastable_shapes.__doc__ = f""" - {mutually_broadcastable_shapes.__doc__} + {_mutually_broadcastable_shapes.__doc__} **Use with Generalised Universal Function signatures** From 1151c095d771dd2e9baaf63e616594313e584089 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Sat, 28 Aug 2021 10:53:12 +0100 Subject: [PATCH 18/24] Hard coded evaluated np.lib.function_base._SIGNATURE --- .../src/hypothesis/extra/_array_helpers.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/hypothesis-python/src/hypothesis/extra/_array_helpers.py b/hypothesis-python/src/hypothesis/extra/_array_helpers.py index 34a768d5ad..c37912c0aa 100644 --- a/hypothesis-python/src/hypothesis/extra/_array_helpers.py +++ b/hypothesis-python/src/hypothesis/extra/_array_helpers.py @@ -300,10 +300,17 @@ def _hypothesis_parse_gufunc_signature(signature, all_checks=True): "anyone who uses them! Please get in touch with us to fix that." f"\n (signature={signature!r})" ) - if re.match("^{0:}->{0:}$".format(_ARGUMENT_LIST), signature): + if re.match( + ( + # Taken from np.lib.function_base._SIGNATURE + r"^\\((?:\\w+(?:,\\w+)*)?\\)(?:,\\((?:\\w+(?:,\\w+)*)?\\))*->" + r"\\((?:\\w+(?:,\\w+)*)?\\)(?:,\\((?:\\w+(?:,\\w+)*)?\\))*$" + ), + signature, + ): raise InvalidArgument( f"signature={signature!r} matches Numpy's regex for gufunc signatures, " - "but contains shapes with more than 32 dimensions and is thus invalid." + f"but contains shapes with more than {NDIM_MAX} dimensions and is thus invalid." ) raise InvalidArgument(f"{signature!r} is not a valid gufunc signature") input_shapes, output_shapes = ( From 198325796b334aea5160324e15b0c456926a7ae5 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Sat, 28 Aug 2021 11:24:28 +0100 Subject: [PATCH 19/24] Revert "Removed redundant test case" This reverts commit 5ca1affbbe77056d4b7e759ed0d37ad23f7454ed. --- hypothesis-python/tests/numpy/test_argument_validation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hypothesis-python/tests/numpy/test_argument_validation.py b/hypothesis-python/tests/numpy/test_argument_validation.py index b291fbd8af..b9ba13c8e6 100644 --- a/hypothesis-python/tests/numpy/test_argument_validation.py +++ b/hypothesis-python/tests/numpy/test_argument_validation.py @@ -267,6 +267,7 @@ def e(a, **kwargs): e(nps.basic_indices, shape=(0, 0), max_dims=-1), e(nps.basic_indices, shape=(0, 0), max_dims=1.0), e(nps.basic_indices, shape=(0, 0), min_dims=2, max_dims=1), + e(nps.basic_indices, shape=(0, 0), min_dims=50), e(nps.basic_indices, shape=(0, 0), max_dims=50), e(nps.integer_array_indices, shape=()), e(nps.integer_array_indices, shape=(2, 0)), From 3c082e2fd61a5b0c801fea54c26cfdbb3e3d1df0 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Sat, 28 Aug 2021 13:45:14 +0100 Subject: [PATCH 20/24] Shape, BroadcastableShapes and BasicIndex specified as public API --- hypothesis-python/src/hypothesis/extra/numpy.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hypothesis-python/src/hypothesis/extra/numpy.py b/hypothesis-python/src/hypothesis/extra/numpy.py index 14a12e6ec1..9b64978047 100644 --- a/hypothesis-python/src/hypothesis/extra/numpy.py +++ b/hypothesis-python/src/hypothesis/extra/numpy.py @@ -25,6 +25,7 @@ NDIM_MAX, BasicIndex, BasicIndexStrategy, + BroadcastableShapes, Shape, array_shapes, broadcastable_shapes, @@ -42,6 +43,9 @@ from hypothesis.strategies._internal.utils import defines_strategy __all__ = [ + "Shape", + "BroadcastableShapes", + "BasicIndex", "from_dtype", "arrays", "array_shapes", From 3415fca62ef6a50af5e1dcacada39e36baa71b1f Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Sat, 28 Aug 2021 22:04:38 +0100 Subject: [PATCH 21/24] Unescaped evaluated np.lib.function_base._SIGNATURE Co-authored-by: Ryan Soklaski --- hypothesis-python/src/hypothesis/extra/_array_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hypothesis-python/src/hypothesis/extra/_array_helpers.py b/hypothesis-python/src/hypothesis/extra/_array_helpers.py index c37912c0aa..3463df24e5 100644 --- a/hypothesis-python/src/hypothesis/extra/_array_helpers.py +++ b/hypothesis-python/src/hypothesis/extra/_array_helpers.py @@ -303,8 +303,8 @@ def _hypothesis_parse_gufunc_signature(signature, all_checks=True): if re.match( ( # Taken from np.lib.function_base._SIGNATURE - r"^\\((?:\\w+(?:,\\w+)*)?\\)(?:,\\((?:\\w+(?:,\\w+)*)?\\))*->" - r"\\((?:\\w+(?:,\\w+)*)?\\)(?:,\\((?:\\w+(?:,\\w+)*)?\\))*$" + r"^\((?:\w+(?:,\w+)*)?\)(?:,\((?:\w+(?:,\w+)*)?\))*->" + r"\((?:\w+(?:,\w+)*)?\)(?:,\((?:\w+(?:,\w+)*)?\))*$" ), signature, ): From 6bbccf4916d051f7cfe19ad04de06dbe5993f668 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Sat, 28 Aug 2021 22:22:38 +0100 Subject: [PATCH 22/24] Removed private objects Shape and BasicIndex from __all__ Co-authored-by: Ryan Soklaski --- hypothesis-python/src/hypothesis/extra/numpy.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/hypothesis-python/src/hypothesis/extra/numpy.py b/hypothesis-python/src/hypothesis/extra/numpy.py index 9b64978047..fce0ece2b2 100644 --- a/hypothesis-python/src/hypothesis/extra/numpy.py +++ b/hypothesis-python/src/hypothesis/extra/numpy.py @@ -43,9 +43,7 @@ from hypothesis.strategies._internal.utils import defines_strategy __all__ = [ - "Shape", "BroadcastableShapes", - "BasicIndex", "from_dtype", "arrays", "array_shapes", From a246ac8d92806c34609a54620276ce3fb237e9ac Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Sat, 28 Aug 2021 22:45:48 +0100 Subject: [PATCH 23/24] Test case to smoke wildcard import --- .../numpy/{test_lazy_import.py => test_import.py} | 14 ++++++++++++++ 1 file changed, 14 insertions(+) rename hypothesis-python/tests/numpy/{test_lazy_import.py => test_import.py} (76%) diff --git a/hypothesis-python/tests/numpy/test_lazy_import.py b/hypothesis-python/tests/numpy/test_import.py similarity index 76% rename from hypothesis-python/tests/numpy/test_lazy_import.py rename to hypothesis-python/tests/numpy/test_import.py index ef9fe16bb2..69f1ecec00 100644 --- a/hypothesis-python/tests/numpy/test_lazy_import.py +++ b/hypothesis-python/tests/numpy/test_import.py @@ -27,3 +27,17 @@ def test_hypothesis_is_not_the_first_to_import_numpy(testdir): # We only import numpy if the user did so first. result = testdir.runpytest(testdir.makepyfile(SHOULD_NOT_IMPORT_NUMPY)) result.assert_outcomes(passed=1, failed=0) + + +# We check the wildcard import works on the module level because that's the only +# place Python actually allows us to use them. +try: + from hypothesis.extra.numpy import * # noqa: F401, F403 + + star_import_works = True +except AttributeError: + star_import_works = False + + +def test_wildcard_import(): + assert star_import_works From 068fbd6ef9d17750a83cf275e6070670d3917ddf Mon Sep 17 00:00:00 2001 From: Zac-HD Date: Sun, 29 Aug 2021 16:32:24 +1000 Subject: [PATCH 24/24] Clarify error message --- hypothesis-python/src/hypothesis/extra/_array_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hypothesis-python/src/hypothesis/extra/_array_helpers.py b/hypothesis-python/src/hypothesis/extra/_array_helpers.py index 3463df24e5..a392e20038 100644 --- a/hypothesis-python/src/hypothesis/extra/_array_helpers.py +++ b/hypothesis-python/src/hypothesis/extra/_array_helpers.py @@ -78,7 +78,7 @@ def check_valid_dims(dims, name): if dims > NDIM_MAX: raise InvalidArgument( f"{name}={dims}, but Hypothesis does not support arrays with " - f"dimensions greater than {NDIM_MAX}" + f"more than {NDIM_MAX} dimensions" )