diff --git a/README.md b/README.md index db98a14..8edf5ee 100644 --- a/README.md +++ b/README.md @@ -249,7 +249,7 @@ encode_account = rules.lookup(typ=Union[AccountA, AccountB, AccountC], ### Adding custom rules -See [the examples][] for details on custom rules, but generally a rule is just a +See [the extras][] for details on custom rules, but generally a rule is just a function. Say, for instance, your type has class methods that encode and decode, this would be sufficient for many cases: @@ -365,7 +365,7 @@ union. [↩](#a2) [poetry]: https://poetry.eustace.io/docs/#installation [gradual typing]: https://www.python.org/dev/peps/pep-0483/#summary-of-gradual-typing -[the examples]: https://github.com/UnitedIncome/json-syntax/tree/master/json_syntax/examples +[the extras]: https://github.com/UnitedIncome/json-syntax/tree/master/json_syntax/extras [typing]: https://docs.python.org/3/library/typing.html [types]: https://github.com/UnitedIncome/json-syntax/blob/master/TYPES.md [attrs]: https://attrs.readthedocs.io/en/stable/ diff --git a/json_syntax/action_v1.py b/json_syntax/action_v1.py index b41da98..537724a 100644 --- a/json_syntax/action_v1.py +++ b/json_syntax/action_v1.py @@ -178,14 +178,15 @@ def check_mapping(value, key, val, con): def convert_dict_to_attrs(value, pre_hook, inner_map, con): value = pre_hook(value) args = {} - for name, inner in inner_map: - with ErrorContext("[{!r}]".format(name)): + for attr in inner_map: + with ErrorContext("[{!r}]".format(attr.name)): try: - arg = value[name] + arg = value[attr.name] except KeyError: - pass + if attr.is_required: + raise KeyError("Missing key") from None else: - args[name] = inner(arg) + args[attr.name] = attr.inner(arg) return con(**args) @@ -193,27 +194,27 @@ def check_dict(value, inner_map, pre_hook): value = pre_hook(value) if not isinstance(value, dict): return False - for name, inner, required in inner_map: - with ErrorContext("[{!r}]".format(name)): + for attr in inner_map: + with ErrorContext("[{!r}]".format(attr.name)): try: - arg = value[name] + arg = value[attr.name] except KeyError: - if required: + if attr.is_required: return False else: - if not inner(arg): + if not attr.inner(arg): return False return True def convert_attrs_to_dict(value, post_hook, inner_map): out = {} - for name, inner, default in inner_map: - with ErrorContext("." + name): - field = getattr(value, name) - if field == default: + for attr in inner_map: + with ErrorContext("." + attr.name): + field = getattr(value, attr.name) + if field == attr.default: continue - out[name] = inner(field) + out[attr.name] = attr.inner(field) if post_hook is not None: out = getattr(value, post_hook)(out) return out diff --git a/json_syntax/attrs.py b/json_syntax/attrs.py index f741502..eefed1f 100644 --- a/json_syntax/attrs.py +++ b/json_syntax/attrs.py @@ -1,16 +1,4 @@ -from .helpers import ( - JSON2PY, - PY2JSON, - INSP_JSON, - INSP_PY, - PATTERN, - SENTINEL, - has_origin, - identity, - is_attrs_field_required, - issub_safe, - resolve_fwd_ref, -) +from .helpers import JSON2PY, PY2JSON, INSP_JSON, INSP_PY, PATTERN, has_origin, identity from .action_v1 import ( check_dict, check_isinst, @@ -20,9 +8,12 @@ convert_tuple_as_list, ) from . import pattern as pat +from .product import build_attribute_map, build_named_tuple_map, build_typed_dict_map from functools import partial +_SUPPORTED_VERBS = (JSON2PY, PY2JSON, INSP_PY, INSP_JSON, PATTERN) + def attrs_classes( verb, @@ -53,49 +44,26 @@ def attrs_classes( `__json_check__` may be used to completely override the `inspect_json` check generated for this class. """ - if verb not in (JSON2PY, PY2JSON, INSP_PY, INSP_JSON, PATTERN): + if verb not in _SUPPORTED_VERBS: + return + inner_map = build_attribute_map(verb, typ, ctx, read_all=verb == PY2JSON) + if inner_map is None: return - try: - fields = typ.__attrs_attrs__ - except AttributeError: - try: - fields = typ.__dataclass_fields__ - except AttributeError: - return - else: - fields = fields.values() if verb == INSP_PY: return partial(check_isinst, typ=typ) - inner_map = [] - for field in fields: - if field.init or verb == PY2JSON: - tup = ( - field.name, - ctx.lookup( - verb=verb, typ=resolve_fwd_ref(field.type, typ), accept_missing=True - ), - ) - if verb == PY2JSON: - tup += (field.default,) - elif verb in (INSP_JSON, PATTERN): - tup += (is_attrs_field_required(field),) - inner_map.append(tup) - if verb == JSON2PY: pre_hook_method = getattr(typ, pre_hook, identity) return partial( convert_dict_to_attrs, pre_hook=pre_hook_method, - inner_map=tuple(inner_map), + inner_map=inner_map, con=typ, ) elif verb == PY2JSON: post_hook = post_hook if hasattr(typ, post_hook) else None - return partial( - convert_attrs_to_dict, post_hook=post_hook, inner_map=tuple(inner_map) - ) + return partial(convert_attrs_to_dict, post_hook=post_hook, inner_map=inner_map) elif verb == INSP_JSON: check = getattr(typ, check, None) if check: @@ -104,9 +72,26 @@ def attrs_classes( return partial(check_dict, inner_map=inner_map, pre_hook=pre_hook_method) elif verb == PATTERN: return pat.Object.exact( - (pat.String.exact(name), inner or pat.Unkown) - for name, inner, req in inner_map - if req + (pat.String.exact(attr.name), attr.inner or pat.Unkown) + for attr in inner_map + if attr.is_required + ) + + +def _simple_product(inner_map, verb, typ, ctx): + if verb == JSON2PY: + return partial( + convert_dict_to_attrs, pre_hook=identity, inner_map=inner_map, con=typ + ) + elif verb == PY2JSON: + return partial(convert_attrs_to_dict, post_hook=None, inner_map=inner_map) + elif verb == INSP_JSON: + return partial(check_dict, pre_hook=identity, inner_map=inner_map) + elif verb == PATTERN: + return pat.Object.exact( + (pat.String.exact(attr.name), attr.inner) + for attr in inner_map + if attr.is_required ) @@ -115,67 +100,53 @@ def named_tuples(verb, typ, ctx): Handle a ``NamedTuple(name, [('field', type), ('field', type)])`` type. Also handles a ``collections.namedtuple`` if you have a fallback handler. + + Warning: there's no clear runtime marker that something is a namedtuple; it's just + a subclass of ``tuple`` that has some special fields. """ - if verb not in (JSON2PY, PY2JSON, INSP_PY, INSP_JSON, PATTERN) or not issub_safe( - typ, tuple - ): + if verb not in _SUPPORTED_VERBS: return - try: - fields = typ._field_types - except AttributeError: - try: - fields = typ._fields - except AttributeError: - return - fields = [(name, None) for name in fields] - else: - fields = fields.items() + + inner_map = build_named_tuple_map(verb, typ, ctx) + if inner_map is None: + return + if verb == INSP_PY: return partial(check_isinst, typ=typ) - defaults = {} - defaults.update(getattr(typ, "_fields_defaults", ())) - defaults.update(getattr(typ, "_field_defaults", ())) - inner_map = [] - for name, inner in fields: - tup = ( - name, - ctx.lookup(verb=verb, typ=resolve_fwd_ref(inner, typ), accept_missing=True), - ) - if verb == PY2JSON: - tup += (defaults.get(name, SENTINEL),) - elif verb in (INSP_JSON, PATTERN): - tup += (name not in defaults,) - inner_map.append(tup) + return _simple_product(inner_map, verb, typ, ctx) - if verb == JSON2PY: - return partial( - convert_dict_to_attrs, - pre_hook=identity, - inner_map=tuple(inner_map), - con=typ, - ) - elif verb == PY2JSON: - return partial( - convert_attrs_to_dict, post_hook=None, inner_map=tuple(inner_map) - ) - elif verb == INSP_JSON: - return partial(check_dict, pre_hook=identity, inner_map=tuple(inner_map)) - elif verb == PATTERN: - return pat.Object.exact( - (pat.String.exact(name), inner) for name, inner, req in inner_map if req - ) + +def typed_dicts(verb, typ, ctx): + """ + Handle the TypedDict product type. This allows you to construct a dict with specific (string) keys, which + is often how people really use dicts. + + Both the class form and the functional form, ``TypedDict('Name', {'field': type, 'field': type})`` are + supported. + """ + if verb not in _SUPPORTED_VERBS: + return + + inner_map = build_typed_dict_map(verb, typ, ctx) + if inner_map is None: + return + + if verb == INSP_PY: + return partial(check_dict, inner_map=inner_map, pre_hook=identity) + + # Note: we pass `dict` as the typ here because it's the correct constructor. + return _simple_product(inner_map, verb, dict, ctx) def tuples(verb, typ, ctx): """ Handle a ``Tuple[type, type, type]`` product type. Use a ``NamedTuple`` if you don't - want a list. + want a list. Though, if possible, prefer ``attrs`` or ``dataclass``. """ - if verb not in (JSON2PY, PY2JSON, INSP_PY, INSP_JSON, PATTERN) or not has_origin( - typ, tuple - ): + if verb not in _SUPPORTED_VERBS or not has_origin(typ, tuple): return + args = typ.__args__ if Ellipsis in args: # This is a homogeneous tuple, use the lists rule. diff --git a/json_syntax/cache.py b/json_syntax/cache.py index d35244b..5dc915f 100644 --- a/json_syntax/cache.py +++ b/json_syntax/cache.py @@ -26,6 +26,18 @@ class SimpleCache: def __init__(self): self.cache = {} + def access(self): + """Requests a context manager to access the cache.""" + return self + + def __enter__(self): + """Stub implementation; see subclasses.""" + return self + + def __exit__(self, e_typ, e_val, e_tb): + """Stub implementation; see subclasses.""" + return + def get(self, verb, typ): result = self._lookup(verb, typ) return result if result is not NotImplemented else None @@ -107,3 +119,22 @@ def cache(self): except AttributeError: _cache = local.cache = {} return _cache + + +class RLockCache(SimpleCache): + """ + Uses a re-entrant lock to ensure only one thread is touching rules at a time. + """ + + def __init__(self, timeout=-1): + self._rlock = threading.RLock() + self._timeout = -1 + self.cache = {} + + def __enter__(self): + if not self._rlock.acquire(timeout=self._timeout): + raise TypeError("acquire failed to acquire a lock") + return self + + def __exit__(self, e_typ, e_val, e_tb): + self._rlock.release() diff --git a/json_syntax/examples/README.md b/json_syntax/extras/README.md similarity index 100% rename from json_syntax/examples/README.md rename to json_syntax/extras/README.md diff --git a/json_syntax/examples/__init__.py b/json_syntax/extras/__init__.py similarity index 100% rename from json_syntax/examples/__init__.py rename to json_syntax/extras/__init__.py diff --git a/json_syntax/extras/dynamodb.py b/json_syntax/extras/dynamodb.py new file mode 100644 index 0000000..1dc4235 --- /dev/null +++ b/json_syntax/extras/dynamodb.py @@ -0,0 +1,402 @@ +""" +While the main suite is fairly complex, it's really not hard to construct a small, useful translation. + +AWS's DynamoDB decorates values to represent them in JSON. The rules for the decorations are fairly simple, and we'd +like to translate to and from Python objects. + +The a Dynamo values look like this: + + {"BOOL": true} + {"L": [{"N": "1.5"}, {"S": "apple"}]} + +We will generate rules to convert Python primitive types, lists and attrs classes into Dynamo types. + +This will special case the kinds of sets Dynamo handles. In keeping with the principle of least astonishment, +it won't convert, e.g. ``Set[MyType]`` into a Dynamo list. This will just fail because Dynamo doesn't actually +support that. You could add a rule if that's the correct semantics. + +For boto3 users: you must use the **client**, not the resource. + + ddb = boto3.client('dynamodb') + ddb.put_item(TableName='chair', Item=...) + +The ``boto3.resource('dynamodb').Table`` is already doing a conversion step we don't want. + +Ref: https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_AttributeValue.html#DDB-Type-AttributeValue-NS +""" + +from json_syntax.helpers import issub_safe, NoneType, has_origin, get_origin +from json_syntax.product import build_attribute_map +from json_syntax.ruleset import SimpleRuleSet + +import base64 as b64 +from decimal import Decimal +from enum import Enum +from functools import partial +from math import isfinite +from numbers import Real +from typing import Union + +DDB2PY = "dynamodb_to_python" +PY2DDB = "python_to_dynamodb" + + +def booleans(verb, typ, ctx): + """ + A rule to represent boolean values as Dynamo booleans. + """ + if typ != bool: + return + if verb == DDB2PY: + return decode_boolean + elif verb == PY2DDB: + return encode_boolean + + +def numbers(verb, typ, ctx): + """ + A rule to represent numeric values as Dynamo numbers. Any number type should work, however both Decimal and float + support NaN and infinity and I haven't tested these in Dynamo. + """ + if typ == bool or not issub_safe(typ, (Decimal, Real)): + return + if verb == DDB2PY: + return partial(decode_number, typ=typ) + elif verb == PY2DDB: + return encode_number + + +def strings(verb, typ, ctx): + """ + A rule to represent string values as Dynamo strings. + """ + if typ != str: + return + if verb == DDB2PY: + return decode_string + elif verb == PY2DDB: + return encode_string + + +def enums(verb, typ, ctx): + "Rule to convert between enumerated types and strings." + if issub_safe(typ, Enum): + if verb == PY2DDB: + return encode_enum + elif verb == DDB2PY: + return partial(decode_enum, typ=typ) + + +def binary(verb, typ, ctx): + """ + A rule to represent bytes as Dynamo binary values. + """ + if typ != bytes: + return + if verb == DDB2PY: + return decode_binary + elif verb == PY2DDB: + return encode_binary + + +def lists(verb, typ, ctx): + """ + A rule to represent lists as Dynamo list values. + """ + if has_origin(typ, list, num_args=1): + (inner,) = typ.__args__ + elif has_origin(typ, tuple, num_args=2): + (inner, ell) = typ.__args__ + if ell is not Ellipsis: + return + else: + return + inner = ctx.lookup(verb=verb, typ=inner) + if verb == DDB2PY: + return partial(decode_list, inner=inner, typ=get_origin(typ)) + elif verb == PY2DDB: + return partial(encode_list, inner=inner) + + +def sets(verb, typ, ctx): + """ + A rule to represent sets. Will only use specialized Dynamo sets, to abide by principle of least astonishment. + + Valid python types include Set[Decimal], Set[str], Set[bytes], or FrozenSet for any of these. Also, any number that + converts from Decimal and converts to a decimal if str is called should work. + """ + if not has_origin(typ, (set, frozenset), num_args=1): + return + (inner,) = typ.__args__ + con = get_origin(typ) + if inner == bytes: + if verb == DDB2PY: + return partial(decode_binary_set, con=con) + elif verb == PY2DDB: + return encode_binary_set + + if inner != bool and issub_safe(inner, (Decimal, Real)): + if verb == DDB2PY: + return partial(decode_number_set, elem=inner, con=con) + elif verb == PY2DDB: + return encode_number_set + + if inner == str: + if verb == DDB2PY: + return partial(decode_string_set, con=con) + elif verb == PY2DDB: + return encode_string_set + + +def attrs(verb, typ, ctx): + """ + A rule to represent attrs classes. This isn't trying to support hooks or any of that. + """ + inner_map = build_attribute_map(verb, typ, ctx, read_all=verb == PY2DDB) + if inner_map is None: + return + + if verb == DDB2PY: + return partial(decode_map, inner_map=inner_map, con=typ) + elif verb == PY2DDB: + return partial(encode_map, inner_map=inner_map) + + +def nulls(verb, typ, ctx): + """ + A rule to represent boolean values as Dynamo nulls. + """ + if typ != NoneType: + return + if verb == DDB2PY: + return decode_null + elif verb == PY2DDB: + return encode_null + + +def optionals(verb, typ, ctx): + """ + Handle an ``Optional[inner]`` by passing ``None`` through. + """ + if has_origin(typ, Union, num_args=2): + if NoneType not in typ.__args__: + return + inner = None + for arg in typ.__args__: + if arg is not NoneType: + inner = arg + if inner is None: + raise TypeError("Could not find inner type for Optional: " + str(typ)) + else: + return + inner = ctx.lookup(verb=verb, typ=inner) + if verb == DDB2PY: + return partial(decode_optional, inner=inner) + elif verb == PY2DDB: + return partial(encode_optional, inner=inner) + + +class DynamodbRuleSet(SimpleRuleSet): + def dynamodb_to_python(self, typ): + return self.lookup(verb=DDB2PY, typ=typ) + + def python_to_dynamodb(self, typ): + return self.lookup(verb=PY2DDB, typ=typ) + + +def dynamodb_ruleset( + strings=strings, + numbers=numbers, + booleans=booleans, + binary=binary, + lists=lists, + attrs=attrs, + enums=enums, + sets=sets, + optionals=optionals, + extras=(), + custom=DynamodbRuleSet, + cache=None, +): + """ + Constructs a RuleSet to migrate data to and from DynamoDB. + """ + return custom( + strings, + numbers, + booleans, + binary, + lists, + attrs, + enums, + sets, + optionals, + *extras, + cache=cache, + ) + + +def desigil(value, **sigils): + """ + Parse a ``{sigil: value}`` expression and unpack the value inside. + """ + if isinstance(value, dict) and len(value) == 1: + for sig, typ in sigils.items(): + try: + inner = value[sig] + except KeyError: + pass + else: + if not isinstance(inner, typ): + raise ValueError( + "This Dynamo value {} must have a member encoded as type {}".format( + sig, typ.__name__ + ) + ) + return sig, inner + for sig, typ in sigils.items(): + break + raise ValueError( + "This Dynamo value must be encoded as a single-item dict {%r: %s}" + % (sig, typ.__name__) + ) + + +def decode_optional(value, inner): + try: + desigil(value, NULL=bool) + except ValueError: + return inner(value) + else: + return None + + +def encode_optional(value, inner): + if value is None: + return {"NULL": True} + else: + return inner(value) + + +def decode_boolean(value): + _, value = desigil(value, BOOL=bool) + return value + + +def encode_boolean(value): + return {"BOOL": bool(value)} + + +b64decode = partial(b64.b64decode, validate=True) + + +def b64encode(value): + return b64.b64encode(value).decode("ASCII") + + +def decode_binary(value): + _, value = desigil(value, B=str) + return b64decode(value) + + +def encode_binary(value): + return {"B": b64encode(value)} + + +def decode_number(value, typ): + _, value = desigil(value, N=str, S=str) + return typ(value) + + +def _encode_number(value): + if not isfinite(value): + # We could employ a string type here, but this could put us in a corner if we + # try to use number sets... + raise ValueError("Can't encode non-finite values in Dynamodb") + if isinstance(value, (int, float, Decimal)): + return str(value) + else: + # This is all the Real interface guarantees us. It's a strech using Fraction in Dynamo. + return str(float(value)) + + +def encode_number(value): + return {"N": _encode_number(value)} + + +def decode_string(value): + _, value = desigil(value, S=str) + return value + + +def encode_string(value): + return {"S": str(value)} + + +def decode_enum(value, typ): + _, value = desigil(value, S=str) + return typ[value] + + +def encode_enum(value): + return {"S": value.name} + + +def decode_list(value, inner, typ): + _, value = desigil(value, L=list) + return typ(map(inner, value)) + + +def encode_list(value, inner): + return {"L": list(map(inner, value))} + + +def decode_map(value, inner_map, con): + _, value = desigil(value, M=dict) + args = {} + for attr in inner_map: + try: + arg = value[attr.name] + except KeyError: + if attr.is_required: + raise KeyError("Missing key") from None + else: + args[attr.name] = attr.inner(arg) + return con(**args) + + +def encode_map(value, inner_map): + out = {} + for attr in inner_map: + field = getattr(value, attr.name) + if field == attr.default: + continue + out[attr.name] = attr.inner(field) + return {"M": out} + + +def decode_binary_set(value, con): + _, value = desigil(value, BS=list) + return con(map(b64decode, value)) + + +def encode_binary_set(value): + return {"BS": list(map(b64encode, value))} + + +def decode_number_set(value, con, elem): + _, value = desigil(value, NS=list) + return con(map(elem, value)) + + +def encode_number_set(value): + return {"NS": list(map(_encode_number, value))} + + +def decode_string_set(value, con): + _, value = desigil(value, SS=list) + return con(map(str, value)) + + +def encode_string_set(value): + return {"SS": list(map(str, value))} diff --git a/json_syntax/examples/flags.py b/json_syntax/extras/flags.py similarity index 100% rename from json_syntax/examples/flags.py rename to json_syntax/extras/flags.py diff --git a/json_syntax/examples/loose_dates.py b/json_syntax/extras/loose_dates.py similarity index 87% rename from json_syntax/examples/loose_dates.py rename to json_syntax/extras/loose_dates.py index e53241d..7582705 100644 --- a/json_syntax/examples/loose_dates.py +++ b/json_syntax/extras/loose_dates.py @@ -7,9 +7,11 @@ """ This example is of working around common date issues. -The standard rules use the standard library's fromisoformat and isoformat methods, to abide by the principle of least surprise. +The standard rules use the standard library's fromisoformat and isoformat methods, to abide by the principle of least +surprise. -But it's pretty common to have to consume a datetime in a date field, and it may also be the case that you want to discard the timestamp. +But it's pretty common to have to consume a datetime in a date field, and it may also be the case that you want to +discard the timestamp. (Note: requires python3.7 or greater.) """ diff --git a/json_syntax/helpers.py b/json_syntax/helpers.py index 1f481d8..889701d 100644 --- a/json_syntax/helpers.py +++ b/json_syntax/helpers.py @@ -37,6 +37,9 @@ class and ``__args__`` are the type arguments. def get_origin(typ): + """ + Get the origin type of a generic type. For example, List has an "origin type" of list. + """ try: t_origin = typ.__origin__ except AttributeError: @@ -84,30 +87,34 @@ def is_generic(typ): for check in getattr(abc, name, None), getattr(c, name.lower(), None): if check: _map.append((generic, check)) - continue + break _pts = {prov: stable for prov, stable in _map} - _stp = {stable: prov for prov, stable in _map} + # _stp = {stable: prov for prov, stable in _map} def _origin_pts(origin, _pts=_pts): """ - Convert the __origin__ of a generic type returned by the provisional typing API (python3.5) to the stable version. + Convert the __origin__ of a generic type returned by the provisional typing API (python3.5) to the stable + version. + + Don't use this, just use get_origin. """ return _pts.get(origin, origin) - def _origin_stp(origin, _stp=_stp): - """ - Convert the __origin__ of a generic type in the stable typing API (python3.6+) to the provisional version. - """ - return _stp.get(origin, origin) + # def _origin_stp(origin, _stp=_stp): + # """ + # Convert the __origin__ of a generic type in the stable typing API (python3.6+) to the provisional version. + # """ + # return _stp.get(origin, origin) del _pts - del _stp + # del _stp del _map del seen del abc del c else: - _origin_pts = _origin_stp = identity + _origin_pts = identity + # _origin_stp = identity def issub_safe(sub, sup): @@ -149,57 +156,41 @@ def resolve_fwd_ref(typ, context_class): # noqa return typ -_missing_values = set() -try: - import attr +class _Context: + """ + Stash contextual information in an exception. As we don't know exactly when an exception is displayed + to a user, this class tries to keep it always up to date. - _missing_values.add(attr.NOTHING) -except ImportError: - pass -try: - import dataclasses + This class subclasses string (to be compatible) and tracks an insertion point. + """ - _missing_values.add(dataclasses.MISSING) -except ImportError: - pass + __slots__ = ("original", "context", "lead") + def __init__(self, original, lead, context): + self.original = original + self.lead = lead + self.context = [context] -def is_attrs_field_required(field): - """ - Determine if a field's default value is missing. - """ - if field.default not in _missing_values: - return False - try: - factory = field.default_factory - except AttributeError: - return True - else: - return factory in _missing_values + def __str__(self): + return "{}{}{}".format( + self.original, self.lead, "".join(reversed(self.context)) + ) + def __repr__(self): + return repr(self.__str__()) -def _add_context(context, exc): - try: + @classmethod + def add(cls, exc, context): + args = exc.args + if args and isinstance(args[0], cls): + args[0].context.append(context) + return args = list(exc.args) - arg_num, point = getattr(exc, "_context", (None, None)) - - if arg_num is None: - for arg_num, val in enumerate(args): - if isinstance(val, str): - args[arg_num] = args[arg_num] + "; at " if val else "At " - break - else: # This 'else' clause runs if we don't `break` - arg_num = len(args) - args.append("At ") - point = len(args[arg_num]) - - arg = args[arg_num] - args[arg_num] = arg[:point] + str(context) + arg[point:] + if args: + args[0] = cls(args[0], "; at ", context) + else: + args.append(cls("", "At ", context)) exc.args = tuple(args) - exc._context = (arg_num, point) - except Exception: - # Swallow exceptions to avoid adding confusion. - pass class ErrorContext: @@ -232,7 +223,7 @@ class ErrorContext: them with angle brackets, e.g. `` """ - def __init__(self, context): + def __init__(self, *context): self.context = context def __enter__(self): @@ -240,7 +231,7 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): if exc_value is not None: - _add_context(self.context, exc_value) + _Context.add(exc_value, "".join(self.context)) def err_ctx(context, func): @@ -255,5 +246,5 @@ def err_ctx(context, func): try: return func() except Exception as exc: - _add_context(context, exc) + _Context.add(exc, context) raise diff --git a/json_syntax/pattern.py b/json_syntax/pattern.py index d942848..ca2e65c 100644 --- a/json_syntax/pattern.py +++ b/json_syntax/pattern.py @@ -2,8 +2,8 @@ Patterns to represent roughly what syntax will look like, and also to investigate whether unions are potentially ambiguous. """ -from functools import partial, lru_cache, singledispatch -from itertools import chain, cycle, islice, product, zip_longest +from functools import partial, singledispatch +from itertools import cycle, islice, product, zip_longest from enum import IntEnum try: diff --git a/json_syntax/product.py b/json_syntax/product.py new file mode 100644 index 0000000..e3defc5 --- /dev/null +++ b/json_syntax/product.py @@ -0,0 +1,187 @@ +""" +A module to help with product types in Python. +""" + +from .helpers import issub_safe, resolve_fwd_ref, SENTINEL + +_TypedDictMeta = None +try: + from typing import _TypedDictMeta +except ImportError: + try: + from typing_extensions import _TypeDictMeta # noqa + except ImportError: + pass + + +_attrs_missing_values = set() +try: + import attr + + _attrs_missing_values.add(attr.NOTHING) +except ImportError: + pass +try: + import dataclasses + + _attrs_missing_values.add(dataclasses.MISSING) +except ImportError: + pass + + +class Attribute: + """ + Generic class to describe an attribute for a product type that can be represented as, e.g., a JSON map. + + An Attribute is associated with an action, specifically, its "inner" field directs how to process the inside type, + not necessarily what the inside type is. + + See the various build_* commands to generate attribute maps. (These are really just lists of Attribute instances.) + + Fields: + name: the attribute name + inner: the action to take given the verb and the attribute's type + default: a static default Python value + is_required: a boolean indicating if the attribute is required + """ + + __slots__ = ("name", "typ", "inner", "default", "is_required") + + def __init__(self, name, typ, is_required, default=SENTINEL, inner=None): + self.name = name + self.typ = typ + self.inner = inner + self.default = default + self.is_required = is_required + + def __repr__(self): + return ''.format(self.name, 'required' if self.is_required else 'optional') + + +def is_attrs_field_required(field): + """ + Determine if a field can calculate its default value. + """ + if field.default not in _attrs_missing_values: + return False + try: + factory = field.default_factory + except AttributeError: + return True + else: + return factory in _attrs_missing_values + + +def attr_map(verb, outer, ctx, gen): + result = [] + failed = [] + for attr in gen: + if attr.typ is not None: + try: + attr.typ = resolve_fwd_ref(attr.typ, outer) + except TypeError: + failed.append('resolve fwd ref {} for {}'.format(attr.typ, attr.name)) + if attr.inner is None: + attr.inner = ctx.lookup(verb=verb, typ=resolve_fwd_ref(attr.typ, outer), accept_missing=True) + if attr.inner is None: + if attr.typ is None: + failed.append('get fallback for {}'.format(attr.name)) + else: + failed.append('get {} for {}'.format(attr.typ, attr.name)) + result.append(attr) + + if failed: + raise TypeError("{}({}) failed while trying to: {}".format(verb, outer, ', '.join(failed))) + return tuple(result) + + +def build_attribute_map(verb, typ, ctx, read_all): + """ + Examine an attrs or dataclass type and construct a list of attributes. + + Returns a list of Attribute instances, or None if the type is not an attrs or dataclass type. + """ + try: + fields = typ.__attrs_attrs__ + except AttributeError: + try: + fields = typ.__dataclass_fields__ + except AttributeError: + return + else: + fields = fields.values() + + return attr_map(verb, typ, ctx, ( + Attribute( + name=field.name, + typ=field.type, + is_required=is_attrs_field_required(field), + default=field.default, + ) + for field in fields + if read_all or field.init + )) + + +def build_named_tuple_map(verb, typ, ctx): + """ + Examine a named tuple type and construct a list of attributes. + + Returns a list of Attribute instances, or None if the type is not a named tuple. + """ + if not issub_safe(typ, tuple): + return + try: + fields = typ._field_types + except AttributeError: + try: + fields = typ._fields + except AttributeError: + return + fields = [(name, None) for name in fields] + else: + fields = fields.items() + + defaults = {} + try: + defaults.update(typ._fields_defaults) + except AttributeError: + pass + try: + defaults.update(typ._field_defaults) + except AttributeError: + pass + + return attr_map(verb, typ, ctx, ( + Attribute( + name=name, + typ=inner, + is_required=name not in defaults, + default=defaults.get(name, SENTINEL), + ) + for name, inner in fields + )) + + +def build_typed_dict_map(verb, typ, ctx): + """ + Examine a TypedDict class and construct a list of attributes. + + Returns a list of Attribute instances, or None if the type is not a typed dict. + """ + if ( + _TypedDictMeta is None + or not issub_safe(typ, dict) + or typ.__class__ is not _TypedDictMeta + ): + return + + return attr_map(verb, typ, ctx, ( + Attribute( + name=name, + typ=inner, + is_required=True, + default=SENTINEL, + ) + for name, inner in typ.__annotations__.items() + )) diff --git a/json_syntax/ruleset.py b/json_syntax/ruleset.py index 01e7951..706dd71 100644 --- a/json_syntax/ruleset.py +++ b/json_syntax/ruleset.py @@ -22,13 +22,13 @@ class SimpleRuleSet: """ This is the base of RuleSet and doesn't know anything about the standard verbs. - A ruleset contains a series of rules that will be evaluated, in order, against types to attempt - to construct encoders and decoders. + A ruleset contains a series of rules that will be evaluated, in order, against types to attempt to construct + encoders and decoders. It takes a list of rules; functions that accept a verb and type and return actions. - The keyword argument `cache` can specify a custom rule cache. `json_syntax.cache.ThreadLocalCache` - may be helpful if you are loading rules in a multi-threaded environment. + The keyword argument `cache` can specify a custom rule cache. `json_syntax.cache.ThreadLocalCache` may be helpful + if you are loading rules in a multi-threaded environment. """ def __init__(self, *rules, cache=None): @@ -38,33 +38,36 @@ def __init__(self, *rules, cache=None): def lookup(self, verb, typ, accept_missing=False): trace("lookup({!s}, {!r}): start", verb, typ) if typ is None: - if not accept_missing: + if accept_missing: + trace("lookup({!s}, {!r}): attempt fallabck", verb, typ) + typ = self.fallback(verb=verb, typ=typ) + if typ is None: raise TypeError("Attempted to find {} for 'None'".format(verb)) - return self.fallback(verb=verb, typ=typ) - action = self.cache.get(verb=verb, typ=typ) - if action is not None: - trace("lookup({!s}, {!r}): cached", verb, typ) - return action + with self.cache.access() as cache: + action = cache.get(verb=verb, typ=typ) + if action is not None: + trace("lookup({!s}, {!r}): cached", verb, typ) + return action + + forward = cache.in_flight(verb=verb, typ=typ) - forward = self.cache.in_flight(verb=verb, typ=typ) + try: + for rule in self.rules: + action = rule(verb=verb, typ=typ, ctx=self) + if action is not None: + cache.complete(verb=verb, typ=typ, action=action) + trace("lookup({!s}, {!r}): computed", verb, typ) + return action - try: - for rule in self.rules: - action = rule(verb=verb, typ=typ, ctx=self) + trace("lookup({!s}, {!r}): fallback", verb, typ) + action = self.fallback(verb=verb, typ=typ) if action is not None: - self.cache.complete(verb=verb, typ=typ, action=action) - trace("lookup({!s}, {!r}): computed", verb, typ) + cache.complete(verb=verb, typ=typ, action=action) + trace("lookup({!s}, {!r}): computed by fallback", verb, typ) return action - - trace("lookup({!s}, {!r}): fallback", verb, typ) - action = self.fallback(verb=verb, typ=typ) - if action is not None: - self.cache.complete(verb=verb, typ=typ, action=action) - trace("lookup({!s}, {!r}): computed by fallback", verb, typ) - return action - finally: - self.cache.de_flight(verb=verb, typ=typ, forward=forward) + finally: + cache.de_flight(verb=verb, typ=typ, forward=forward) if action is None and not accept_missing: raise TypeError("Failed: lookup({!s}, {!r})".format(verb, typ)) diff --git a/pyproject.toml b/pyproject.toml index fa37e6b..d7dfc4c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "json-syntax" -version = "1.0.1" +version = "2.0.0" description = "Generates functions to convert Python classes to JSON dumpable objects." authors = ["Ben Samuel "] license = "MIT" @@ -37,8 +37,6 @@ skipsdist = true [testenv] deps = poetry -whitelist_externals = - /bin/rm commands = poetry install poetry run pytest {posargs} diff --git a/setup.cfg b/setup.cfg index 0edafcc..718e59c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,9 +1,16 @@ [flake8] -ignore = D203, W503, E501 +# D203: one blank line before class docstring, this or D211 +# W503: line break before binary operator, have to pick this or W504 +# Everything after the first line is fixed with black. +ignore = D203, W503, + E111, E121, E122, E123, E124, E125, E126, E201, E202, E203, + E221, E222, E225, E226, E227, E231, E241, E251, E261, E262, + E265, E271, E272, E302, E303, E306, E502, E701, E702, E703, + E704, W291, W292, W293, W391 exclude = .tox __pycache__ .git htmlcov -max-line-length = 88 +max-line-length = 120 disable-noqa = False diff --git a/tests/examples/__init__.py b/tests/extras/__init__.py similarity index 100% rename from tests/examples/__init__.py rename to tests/extras/__init__.py diff --git a/tests/extras/test_dynamodb.py b/tests/extras/test_dynamodb.py new file mode 100644 index 0000000..8f2ab0e --- /dev/null +++ b/tests/extras/test_dynamodb.py @@ -0,0 +1,137 @@ +import pytest + +from json_syntax.extras.dynamodb import dynamodb_ruleset +from json_syntax.helpers import NoneType + +from fractions import Fraction +from decimal import Decimal +from typing import List, Set, Optional + +try: + import attr +except ImportError: + attr = None + + +def encode(value, typ): + return dynamodb_ruleset().python_to_dynamodb(typ)(value) + + +def decode(value, typ): + return dynamodb_ruleset().dynamodb_to_python(typ)(value) + + +def test_optional(): + assert encode(None, Optional[int]) == {"NULL": True} + assert encode(5, Optional[int]) == {"N": "5"} + assert decode({"NULL": True}, Optional[str]) is None + assert decode({"S": "wat"}, Optional[str]) == "wat" + + +def test_bool(): + assert encode(True, bool) == {"BOOL": True} + assert decode({"BOOL": False}, bool) is False + + +def test_binary(): + assert encode(b"foobar", bytes) == {"B": "Zm9vYmFy"} + assert decode({"B": "Zm9vYmFy"}, bytes) == b"foobar" + + +def test_number1(): + assert encode(55.125, float) == {"N": "55.125"} + assert decode({"N": "-55.125"}, float) == -55.125 + + +def test_number2(): + with pytest.raises(ValueError): + encode(float("nan"), float) + + +def test_number3(): + assert encode(Fraction(441, 8), Fraction) == {"N": "55.125"} + assert decode({"N": "55.125"}, Fraction) == Fraction(441, 8) + + +def test_number4(): + assert encode(Decimal("55.125"), Decimal) == {"N": "55.125"} + assert decode({"N": "-55.125"}, Decimal) == Decimal("-55.125") + + +def test_string(): + assert encode("foobar", str) == {"S": "foobar"} + assert decode({"S": "foobar"}, str) == "foobar" + + +def test_list(): + assert encode([1, 2, 4, 5], List[int]) == { + "L": [{"N": str(x)} for x in [1, 2, 4, 5]] + } + assert decode({"L": [{"S": "apple"}, {"S": "banana"}]}, List[str]) == [ + "apple", + "banana", + ] + + +def cheat(value): + for val in value.values(): + val.sort() + return value + + +def test_str_set(): + assert cheat(encode({"foo", "bar", "qux"}, Set[str])) == { + "SS": ["bar", "foo", "qux"] + } + assert decode({"SS": ["foo", "bar", "qux"]}, Set[str]) == {"foo", "bar", "qux"} + + +def test_num_set(): + assert cheat(encode({-33.5, 11.25, 1.75}, Set[float])) == { + "NS": ["-33.5", "1.75", "11.25"] + } + assert decode({"NS": [11.25, 1.75, -33.5]}, Set[float]) == {-33.5, 11.25, 1.75} + + +def test_bin_set(): + assert cheat(encode({b"foo", b"bar", b"qux"}, Set[bytes])) == { + "BS": ["YmFy", "Zm9v", "cXV4"] + } + assert decode({"BS": ["YmFy", "Zm9v", "cXV4"]}, Set[bytes]) == { + b"foo", + b"bar", + b"qux", + } + + +@attr.s +class Inner: + name = attr.ib(type=str) + + +@attr.s +class Outer: + stuff = attr.ib(type=Inner) + count = attr.ib(type=int, default=7) + + +def test_map1(): + subj = Outer(count=3, stuff=Inner(name="bob")) + expected = {"M": {"count": {"N": "3"}, "stuff": {"M": {"name": {"S": "bob"}}}}} + assert encode(subj, Outer) == expected + + subj = {"M": {"count": {"N": "3"}, "stuff": {"M": {"name": {"S": "bob"}}}}} + expected = Outer(count=3, stuff=Inner(name="bob")) + assert decode(subj, Outer) == expected + + +def test_map2(): + subj = Outer(stuff=Inner(name="bob")) + expected = {"M": {"stuff": {"M": {"name": {"S": "bob"}}}}} + assert encode(subj, Outer) == expected + + subj = { + "M": {"stuff": {"M": {"name": {"S": "bob"}}}, "other_key": {"S": "ignored"}} + } + expected = Outer(stuff=Inner(name="bob")) + assert decode(subj, Outer) == expected diff --git a/tests/examples/test_loose_dates.py b/tests/extras/test_loose_dates.py similarity index 86% rename from tests/examples/test_loose_dates.py rename to tests/extras/test_loose_dates.py index c6c0547..873605c 100644 --- a/tests/examples/test_loose_dates.py +++ b/tests/extras/test_loose_dates.py @@ -1,6 +1,6 @@ import pytest -from json_syntax.examples import loose_dates as exam +from json_syntax.extras import loose_dates as exam from json_syntax.helpers import JSON2PY, PY2JSON, INSP_PY, INSP_JSON, python_minor from datetime import date, datetime @@ -15,7 +15,7 @@ def Fail(): python_minor < (3, 7), reason="datetime.isoformat not supported before python 3.7" ) def test_iso_dates_loose(): - "Test the iso_dates_loose rule will generate encoders and decoders for dates using ISO8601, accepting datetimes as input to dates." + "Test the iso_dates_loose handles dates using ISO8601, accepting datetimes as input to dates." decoder = exam.iso_dates_loose(verb=JSON2PY, typ=date, ctx=Fail()) assert decoder("1776-07-04") == date(1776, 7, 4) diff --git a/tests/test_attrs.py b/tests/test_attrs.py index 91cd13b..3dc823b 100644 --- a/tests/test_attrs.py +++ b/tests/test_attrs.py @@ -1,17 +1,13 @@ import pytest -from unittest.mock import Mock from json_syntax import attrs as at from json_syntax.helpers import JSON2PY, PY2JSON, INSP_PY, INSP_JSON -import attr -from collections import namedtuple - try: from dataclasses import dataclass except ImportError: - dataclass = lambda cls: None -from typing import NamedTuple, Tuple + dataclass = lambda cls: None # noqa +from typing import Tuple try: from tests.types_attrs_ann import flat_types, hook_types, Named1, Named2, Named3 diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 84313ea..8c72b45 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -1,5 +1,6 @@ from json_syntax import helpers as hlp +import traceback as tb import typing as t @@ -29,7 +30,7 @@ def test_has_origin_num_args(): assert hlp.has_origin(t.Tuple[int, str, float], tuple, num_args=3) -def test_issub_safe_normal_type(): +def test_issub_safe_normal_type1(): "Test that issub_safe behaves like issubclass for normal types." assert hlp.issub_safe(bool, int) @@ -37,7 +38,7 @@ def test_issub_safe_normal_type(): assert not hlp.issub_safe(int, str) -def test_issub_safe_normal_type(): +def test_issub_safe_normal_type2(): "Test that issub_safe returns False for generic types." assert not hlp.issub_safe(t.List[int], list) @@ -81,3 +82,27 @@ def test_resolve_fwd_ref_bad_context(): actual = hlp.resolve_fwd_ref(subj, "dummy") assert actual is subj + + +def outside(*msg): + with hlp.ErrorContext(".", "alpha"): + return inside(*msg) + + +def inside(*msg): + with hlp.ErrorContext(".", "beta"): + raise ValueError(*msg) + + +def run_func(*args): + try: + outside(*args) + except ValueError as exc: + return "".join(tb.format_exception_only(type(exc), exc)) + + +def test_error_contexts(): + "Test that error contexts add information correctly." + assert run_func() == "ValueError: At .alpha.beta\n" + assert run_func("message") == "ValueError: message; at .alpha.beta\n" + assert run_func("two", "parts") == "ValueError: ('two; at .alpha.beta', 'parts')\n" diff --git a/tests/test_std.py b/tests/test_std.py index 461b69b..29af511 100644 --- a/tests/test_std.py +++ b/tests/test_std.py @@ -279,7 +279,7 @@ def test_iso_dates_disregard(): def test_iso_dates(): - "Test the iso_dates rule will generate encoders and decoders for dates using ISO8601, rejecting datetimes as input to dates." + "Test the iso_dates rule handles dates using ISO8601, rejecting datetimes as input to dates." decoder = std.iso_dates(verb=JSON2PY, typ=date, ctx=Fail()) assert decoder("1776-07-04") == date(1776, 7, 4) diff --git a/tests/test_std_ruleset.py b/tests/test_std_ruleset.py index 84f8e40..20c5b19 100644 --- a/tests/test_std_ruleset.py +++ b/tests/test_std_ruleset.py @@ -2,8 +2,6 @@ from datetime import date from decimal import Decimal -from enum import Enum -from typing import Optional, List import json_syntax as syn try: diff --git a/tests/test_union_prop.py b/tests/test_union_prop.py index 79a0729..52e9c16 100644 --- a/tests/test_union_prop.py +++ b/tests/test_union_prop.py @@ -1,17 +1,16 @@ -import pytest -from hypothesis import given, settings, HealthCheck, reproduce_failure +from hypothesis import given, settings, HealthCheck from . import type_strategies as ts -import attr -from datetime import date, datetime -from decimal import Decimal -from enum import Enum -from itertools import product -from typing import Union, List, Tuple, Set, FrozenSet, Dict +# import attr +# from datetime import date, datetime +# from decimal import Decimal +# from enum import Enum +# from itertools import product +# from typing import Union, List, Tuple, Set, FrozenSet, Dict from json_syntax import std_ruleset -from json_syntax.helpers import PY2JSON, JSON2PY, INSP_PY, INSP_JSON, NoneType +from json_syntax.helpers import PY2JSON, JSON2PY # INSP_PY, INSP_JSON, NoneType from json_syntax.pattern import Matches diff --git a/tests/types_attrs_ann.py b/tests/types_attrs_ann.py index 53c8452..11d5227 100644 --- a/tests/types_attrs_ann.py +++ b/tests/types_attrs_ann.py @@ -1,8 +1,9 @@ import attr -from collections import namedtuple from typing import NamedTuple -from tests.types_attrs_noann import * +from tests.types_attrs_noann import flat_types, hook_types, Hooks +# Import for re-export +from tests.types_attrs_noann import Named1, Named2 # noqa try: from dataclasses import dataclass diff --git a/tests/types_std_ann.py b/tests/types_std_ann.py index b57bb40..89da4bd 100644 --- a/tests/types_std_ann.py +++ b/tests/types_std_ann.py @@ -2,7 +2,6 @@ from dataclasses import dataclass except ImportError: from attr import dataclass -import attr from datetime import date from decimal import Decimal from enum import Enum