Allow optional fields in fixed_dictionaries() (#2127)

Allow `optional` fields in `fixed_dictionaries()`

.isort.cfg

@@ -1,6 +1,6 @@
 [settings]
-known_third_party = attr, click, dateutil, django, dpcontracts, flaky, lark, numpy, pandas, pytz, pytest, pyup, requests, scipy, unicodenazi, yaml
-known_first_party = hypothesis, tests
+default_section = THIRDPARTY
+known_first_party = hypothesis, hypothesistooling, tests
 add_imports = from __future__ import absolute_import, from __future__ import print_function, from __future__ import division
 multi_line_output = 3
 include_trailing_comma = True

guides/api-style.rst

@@ -68,6 +68,13 @@ We have a reasonably distinctive style when it comes to handling arguments:
 * It's worth thinking about the order of arguments: the first one or two
   arguments are likely to be passed positionally, so try to put values there
   where this is useful and not too confusing.
+* Arguments which have a default value should also be keyword-only.
+  For example, the ideal signature for lists would be
+  ``lists(elements, *, min_size=0, max_size=None, unique_by=None, unique=False)``.
+  We intend to `migrate to this style after dropping Python 2 support <https://github.com/HypothesisWorks/hypothesis/issues/2130>`__
+  in early 2020, with a gentle deprecation pathway.
+  New functions or argumemnts can implement a forward-compatible signature with
+  ``hypothesis.internal.reflection.reserved_to_kwonly``.
 * When adding arguments to strategies, think carefully about whether the user
   is likely to want that value to vary often. If so, make it a strategy instead
   of a value. In particular if it's likely to be common that they would want to

hypothesis-python/.coveragerc

@@ -23,3 +23,4 @@ exclude_lines =
     except ImportError:
     if PY2:
     assert all\(.+\)
+    if __reserved is not not_set:

hypothesis-python/RELEASE.rst

@@ -0,0 +1,4 @@
+RELEASE_TYPE: minor
+
+This release upgrades the :func:`~hypothesis.strategies.fixed_dictionaries`
+strategy to support ``optional`` keys (:issue:`1913`).

hypothesis-python/docs/strategies.rst

@@ -28,6 +28,8 @@ Some packages provide strategies directly:
 * :pypi:`hypothesis-ros` - strategies to generate messages and parameters for the `Robot Operating System <https://www.ros.org/>`_.
 * :pypi:`hypothesis-csv` - strategy to generate CSV files.
 * :pypi:`hypothesis-networkx` - strategy to generate :pypi:`networkx` graphs.
+* :pypi:`hypothesis-bio` - strategies for bioinformatics data, such as DNA, codons, FASTA, and FASTQ formats.
+* :pypi:`hypothesmith` - strategy to generate syntatically-valid Python code.
 
 Others provide a function to infer a strategy from some other schema:
 
@@ -56,6 +58,11 @@ that allows ``@given(...)`` to work with Trio-style async test functions, and
 :pypi:`hypothesis-trio` includes stateful testing extensions to support
 concurrent programs.
 
+:pypi:`pymtl3` is "an open-source Python-based hardware generation, simulation,
+and verification framework with multi-level hardware modeling support", which
+ships with Hypothesis integrations to check that all of those levels are
+eqivalent, from function-level to register-transfer level and even to hardware.
+
 :pypi:`libarchimedes` makes it easy to use Hypothesis in
 `the Hy language <https://github.com/hylang/hy>`_, a Lisp embedded in Python.
 

hypothesis-python/src/hypothesis/_strategies.py

@@ -69,6 +69,7 @@
     nicerepr,
     proxies,
     required_args,
+    reserved_means_kwonly_star,
 )
 from hypothesis.internal.validation import (
     check_type,
@@ -82,6 +83,7 @@
 )
 from hypothesis.searchstrategy import SearchStrategy, check_strategy
 from hypothesis.searchstrategy.collections import (
+    FixedAndOptionalKeysDictStrategy,
     FixedKeysDictStrategy,
     ListStrategy,
     TupleStrategy,
@@ -875,8 +877,13 @@ def iterables(
 
 
 @defines_strategy
-def fixed_dictionaries(mapping):
-    # type: (Dict[T, SearchStrategy[Ex]]) -> SearchStrategy[Dict[T, Ex]]
+@reserved_means_kwonly_star
+def fixed_dictionaries(
+    mapping,  # type: Dict[T, SearchStrategy[Ex]]
+    __reserved=not_set,  # type: Any
+    optional=None,  # type: Dict[T, SearchStrategy[Ex]]
+):
+    # type: (...) -> SearchStrategy[Dict[T, Ex]]
     """Generates a dictionary of the same type as mapping with a fixed set of
     keys mapping to strategies. mapping must be a dict subclass.
 
@@ -891,6 +898,23 @@ def fixed_dictionaries(mapping):
     check_type(dict, mapping, "mapping")
     for k, v in mapping.items():
         check_strategy(v, "mapping[%r]" % (k,))
+    if __reserved is not not_set:
+        raise InvalidArgument("Do not pass __reserved; got %r" % (__reserved,))
+    if optional is not None:
+        check_type(dict, optional, "optional")
+        for k, v in optional.items():
+            check_strategy(v, "optional[%r]" % (k,))
+        if type(mapping) != type(optional):
+            raise InvalidArgument(
+                "Got arguments of different types: mapping=%s, optional=%s"
+                % (nicerepr(type(mapping)), nicerepr(type(optional)))
+            )
+        if set(mapping) & set(optional):
+            raise InvalidArgument(
+                "The following keys were in both mapping and optional, "
+                "which is invalid: %r" % (set(mapping) & set(optional))
+            )
+        return FixedAndOptionalKeysDictStrategy(mapping, optional)
     return FixedKeysDictStrategy(mapping)
 
 

hypothesis-python/src/hypothesis/extra/numpy.py

@@ -28,7 +28,7 @@
 from hypothesis.errors import InvalidArgument
 from hypothesis.internal.compat import PY2, hrange, integer_types
 from hypothesis.internal.coverage import check_function
-from hypothesis.internal.reflection import proxies
+from hypothesis.internal.reflection import proxies, reserved_means_kwonly_star
 from hypothesis.internal.validation import check_type, check_valid_interval
 from hypothesis.reporting import current_verbosity
 from hypothesis.searchstrategy import SearchStrategy
@@ -923,6 +923,7 @@ def do_draw(self, data):
 
 
 @st.defines_strategy
+@reserved_means_kwonly_star
 def basic_indices(
     shape,
     __reserved=not_set,

hypothesis-python/src/hypothesis/internal/reflection.py

@@ -31,6 +31,7 @@
 
 from hypothesis.internal.compat import (
     ARG_NAME_ATTRIBUTE,
+    PY2,
     getfullargspec,
     hrange,
     isidentifier,
@@ -47,6 +48,11 @@
 except ImportError:  # pragma: no cover
     detect_encoding = None
 
+if False:
+    from typing import TypeVar  # noqa
+
+    C = TypeVar("C", bound=callable)
+
 
 def fully_qualified_name(f):
     """Returns a unique identifier for f pointing to the module it was defined
@@ -620,3 +626,38 @@ def accept(proxy):
         )
 
     return accept
+
+
+def reserved_means_kwonly_star(func):
+    # type: (C) -> C
+    """A decorator to implement Python-2-compatible kwonly args.
+
+    The following functions behave identically:
+        def f(a, __reserved=not_set, b=None): ...
+        def f(a, *, b=None): ...
+
+    Obviously this doesn't allow required kwonly args, but it's a nice way
+    of defining forward-compatible APIs given our plans to turn all args
+    with default values into kwonly args.
+    """
+    if PY2:
+        return func
+
+    signature = inspect.signature(func)
+    seen = False
+    parameters = []
+    for param in signature.parameters.values():
+        assert param.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD
+        if param.name == "__reserved":
+            seen = True
+        elif not seen:
+            parameters.append(param)
+        else:
+            parameters.append(param.replace(kind=inspect.Parameter.KEYWORD_ONLY))
+    assert seen, "function does not have `__reserved` argument"
+
+    func.__signature__ = signature.replace(parameters=parameters)
+    newsig = define_function_signature(
+        func.__name__, func.__doc__, getfullargspec(func)
+    )
+    return impersonate(func)(wraps(func)(newsig(func)))

hypothesis-python/src/hypothesis/searchstrategy/collections.py

@@ -234,3 +234,47 @@ def __repr__(self):
 
     def pack(self, value):
         return self.dict_type(zip(self.keys, value))
+
+
+class FixedAndOptionalKeysDictStrategy(SearchStrategy):
+    """A strategy which produces dicts with a fixed set of keys, given a
+    strategy for each of their equivalent values.
+
+    e.g. {'foo' : some_int_strategy} would generate dicts with the single
+    key 'foo' mapping to some integer.
+    """
+
+    def __init__(self, strategy_dict, optional):
+        self.required = strategy_dict
+        self.fixed = FixedKeysDictStrategy(strategy_dict)
+        self.optional = optional
+
+        if isinstance(self.optional, OrderedDict):
+            self.optional_keys = tuple(self.optional.keys())
+        else:
+            try:
+                self.optional_keys = tuple(sorted(self.optional.keys()))
+            except TypeError:
+                self.optional_keys = tuple(sorted(self.optional.keys(), key=repr))
+
+    def calc_is_empty(self, recur):
+        return recur(self.fixed)
+
+    def __repr__(self):
+        return "FixedAndOptionalKeysDictStrategy(%r, %r)" % (
+            self.required,
+            self.optional,
+        )
+
+    def do_draw(self, data):
+        result = data.draw(self.fixed)
+        remaining = [k for k in self.optional_keys if not self.optional[k].is_empty]
+        should_draw = cu.many(
+            data, min_size=0, max_size=len(remaining), average_size=len(remaining) / 2
+        )
+        while should_draw.more():
+            j = cu.integer_range(data, 0, len(remaining) - 1)
+            remaining[-1], remaining[j] = remaining[j], remaining[-1]
+            key = remaining.pop()
+            result[key] = data.draw(self.optional[key])
+        return result

hypothesis-python/tests/cover/test_direct_strategies.py

@@ -140,6 +140,10 @@ def fn_ktest(*fnkwargs):
     (ds.complex_numbers, {"max_magnitude": 2, "allow_nan": True}),
     (ds.fixed_dictionaries, {"mapping": "fish"}),
     (ds.fixed_dictionaries, {"mapping": {1: "fish"}}),
+    (ds.fixed_dictionaries, {"mapping": {}, "optional": "fish"}),
+    (ds.fixed_dictionaries, {"mapping": {}, "optional": {1: "fish"}}),
+    (ds.fixed_dictionaries, {"mapping": {}, "optional": collections.OrderedDict()}),
+    (ds.fixed_dictionaries, {"mapping": {1: ds.none()}, "optional": {1: ds.none()}}),
     (ds.dictionaries, {"keys": ds.integers(), "values": 1}),
     (ds.dictionaries, {"keys": 1, "values": ds.integers()}),
     (ds.text, {"alphabet": "", "min_size": 1}),

hypothesis-python/tests/cover/test_simple_collections.py

@@ -17,6 +17,7 @@
 
 from __future__ import absolute_import, division, print_function
 
+from collections import OrderedDict
 from random import Random
 
 import pytest
@@ -48,6 +49,11 @@
         (set(), sets(none(), max_size=0)),
         (frozenset(), frozensets(none(), max_size=0)),
         ({}, fixed_dictionaries({})),
+        ({}, fixed_dictionaries({}, optional={})),
+        (OrderedDict(), fixed_dictionaries(OrderedDict(), optional=OrderedDict())),
+        ({}, fixed_dictionaries({}, optional={1: booleans()})),
+        ({0: False}, fixed_dictionaries({0: booleans()}, optional={1: booleans()})),
+        ({}, fixed_dictionaries({}, optional={(): booleans(), 0: booleans()})),
         ([], lists(nothing())),
         ([], lists(nothing(), unique=True)),
     ],
@@ -90,6 +96,11 @@ def test_ordered_dictionaries_preserve_keys():
     assert list(x.keys()) == keys
 
 
+@given(fixed_dictionaries({}, optional={0: booleans(), 1: nothing(), 2: booleans()}))
+def test_fixed_dictionaries_with_optional_and_empty_keys(d):
+    assert 1 not in d
+
+
 @pytest.mark.parametrize(u"n", range(10))
 def test_lists_of_fixed_length(n):
     assert minimal(lists(integers(), min_size=n, max_size=n), lambda x: True) == [0] * n

hypothesis-python/tests/numpy/test_argument_validation.py

@@ -171,7 +171,7 @@ def test_unicode_string_dtype_len_0(data):
 
 
 def test_test_basic_indices_kwonly_emulation():
-    with pytest.raises(InvalidArgument):
+    with pytest.raises(TypeError):
         nps.basic_indices((), 0, 1).validate()
-    with pytest.raises(InvalidArgument):
+    with pytest.raises(TypeError):
         nps.basic_indices((), __reserved=None).validate()