From 6fe29c9c70a09e1e4948a3301b693ec935371d39 Mon Sep 17 00:00:00 2001 From: Ben Samuel Date: Tue, 2 Apr 2019 20:59:04 -0400 Subject: [PATCH 1/6] Add property tests for non-union values. - Also adds timedelta to iso_dates and *very* basic round trip support. - Passes in all supported python versions! --- .gitignore | 1 + json_syntax/__init__.py | 4 +- json_syntax/action_v1.py | 59 ++++++++++++++++- json_syntax/std.py | 15 +++-- poetry.lock | 42 +++++++----- pyproject.toml | 3 +- tests/__init__.py | 0 tests/_strategies.py | 131 +++++++++++++++++++++++++++++++++++++ tests/test_union_prop.py | 26 ++++++++ tests/type_strategies.py | 135 +++++++++++++++++++++++++++++++++++++++ 10 files changed, 392 insertions(+), 24 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/_strategies.py create mode 100644 tests/test_union_prop.py create mode 100644 tests/type_strategies.py diff --git a/.gitignore b/.gitignore index e1eced7..2b645a9 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ /dist/ /.python-version /.coverage +/.hypothesis /htmlcov /pip-wheel-metadata setup.py diff --git a/json_syntax/__init__.py b/json_syntax/__init__.py index 6347835..b840df2 100644 --- a/json_syntax/__init__.py +++ b/json_syntax/__init__.py @@ -43,18 +43,18 @@ def std_ruleset( For example, to replace ``decimals`` with ``decimals_as_str`` just call ``std_ruleset(decimals=decimals_as_str)`` """ return custom( + enums, atoms, floats, decimals, dates, optional, - enums, lists, attrs_classes, sets, - dicts, named_tuples, tuples, + dicts, unions, *extras, cache=cache, diff --git a/json_syntax/action_v1.py b/json_syntax/action_v1.py index 2cb7d5c..c15e7e7 100644 --- a/json_syntax/action_v1.py +++ b/json_syntax/action_v1.py @@ -1,7 +1,9 @@ from .helpers import ErrorContext, err_ctx -from datetime import date, datetime, time +from datetime import date, datetime, time, timedelta +from decimal import InvalidOperation import math +import re def check_parse_error(value, parser, error): @@ -21,6 +23,13 @@ def check_has_type(value, typ): return type(value) == typ +def convert_decimal_str(value): + result = str(value) + if result == 'sNaN': + raise InvalidOperation("Won't save signalling NaN") + return result + + def convert_float(value): value = float(value) if math.isfinite(value): @@ -74,6 +83,54 @@ def convert_str_enum(value, mapping): del instance +def convert_timedelta_str(dur): + "Barebones support for storing a timedelta as an ISO8601 duration." + micro = '.{:06d}'.format(dur.microseconds) if dur.microseconds else '' + return 'P{:d}DT{:d}{}S'.format(dur.days, dur.seconds, micro) + + +_iso8601_duration = re.compile( + r"^P(?!$)([-+]?\d+(?:[.,]\d+)?Y)?" + r"([-+]?\d+(?:[.,]\d+)?M)?" + r"([-+]?\d+(?:[.,]\d+)?W)?" + r"([-+]?\d+(?:[.,]\d+)?D)?" + r"(?:(T)(?=[0-9+-])" + r"([-+]?\d+(?:[.,]\d+)?H)?" + r"([-+]?\d+(?:[.,]\d+)?M)?" + r"([-+]?\d+(?:[.,]\d+)?S)?)?$", +) +_duration_args = { + 'PW': 'weeks', + 'PD': 'days', + 'TH': 'hours', + 'TM': 'minutes', + 'TS': 'seconds' +} + + +def convert_str_timedelta(dur): + match = _iso8601_duration.match(dur.upper().replace(',', '.')) + section = 'P' + if not match: + raise ValueError('Value was not an ISO8601 duration.') + args = {} + for elem in match.groups(): + if elem is None: + continue + if elem == 'T': + section = 'T' + continue + part = section + elem[-1] + value = float(elem[:-1]) + if not value: + continue + + if part in ('PY', 'PM'): + raise ValueError('Year and month durations not supported') + args[_duration_args[part]] = value + return timedelta(**args) + + def convert_optional(value, inner): if value is None: return None diff --git a/json_syntax/std.py b/json_syntax/std.py index 4ce5214..9640385 100644 --- a/json_syntax/std.py +++ b/json_syntax/std.py @@ -20,17 +20,20 @@ convert_collection, convert_date, convert_datetime, + convert_decimal_str, convert_enum_str, convert_float, convert_mapping, convert_none, convert_optional, convert_str_enum, + convert_str_timedelta, convert_time, + convert_timedelta_str, ) from collections import OrderedDict -from datetime import datetime, date, time +from datetime import datetime, date, time, timedelta from decimal import Decimal from enum import Enum from functools import partial @@ -132,7 +135,7 @@ def decimals_as_str(verb, typ, ctx): if verb == JSON2PY: return Decimal elif verb == PY2JSON: - return str + return convert_decimal_str elif verb == INSP_PY: return partial(check_isinst, typ=Decimal) elif verb == INSP_JSON: @@ -143,15 +146,15 @@ def iso_dates(verb, typ, ctx): """ Rule to handle iso formatted datetimes and dates. - This is the strict variant that simply uses the `fromisoformat` and `isoformat` methods of `date` and `datetime`. + This simply uses the `fromisoformat` and `isoformat` methods of `date` and `datetime`. There is a loose variant in the examples that will accept a datetime in a date. A datetime always accepts both dates and datetimes. """ - if typ not in (date, datetime, time): + if typ not in (date, datetime, time, timedelta): return if verb == PY2JSON: - return typ.isoformat + return convert_timedelta_str if typ == timedelta else typ.isoformat elif verb == INSP_PY: return partial(check_has_type, typ=typ) elif verb in (JSON2PY, INSP_JSON): @@ -161,6 +164,8 @@ def iso_dates(verb, typ, ctx): parse = convert_datetime elif typ == time: parse = convert_time + elif typ == timedelta: + parse = convert_str_timedelta else: return if verb == JSON2PY: diff --git a/poetry.lock b/poetry.lock index 98ad268..a299a97 100644 --- a/poetry.lock +++ b/poetry.lock @@ -30,11 +30,11 @@ marker = "python_version >= \"3.7\"" name = "black" optional = false python-versions = ">=3.6" -version = "18.9b0" +version = "19.3b0" [package.dependencies] appdirs = "*" -attrs = ">=17.4.0" +attrs = ">=18.1.0" click = ">=6.5" toml = ">=0.9.4" @@ -62,7 +62,18 @@ description = "Code coverage measurement for Python" name = "coverage" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, <4" -version = "4.5.2" +version = "4.5.3" + +[[package]] +category = "dev" +description = "A library for property based testing" +name = "hypothesis" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "4.14.2" + +[package.dependencies] +attrs = ">=16.0.0" [[package]] category = "dev" @@ -71,7 +82,7 @@ marker = "python_version > \"2.7\"" name = "more-itertools" optional = false python-versions = ">=3.4" -version = "6.0.0" +version = "7.0.0" [[package]] category = "dev" @@ -91,7 +102,7 @@ description = "plugin and hook calling mechanisms for python" name = "pluggy" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "0.8.1" +version = "0.9.0" [[package]] category = "dev" @@ -99,7 +110,7 @@ description = "library with cross-python path, ini-parsing, io, code, log facili name = "py" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "1.7.0" +version = "1.8.0" [[package]] category = "dev" @@ -107,13 +118,13 @@ description = "pytest: simple powerful testing with Python" name = "pytest" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "4.2.1" +version = "4.4.0" [package.dependencies] atomicwrites = ">=1.0" attrs = ">=17.4.0" colorama = "*" -pluggy = ">=0.7" +pluggy = ">=0.9" py = ">=1.5.0" setuptools = "*" six = ">=1.10.0" @@ -168,22 +179,23 @@ python-versions = "*" version = "0.10.0" [metadata] -content-hash = "4c0e31907eb58992b7d6ee19b1f5b30233cffbfd056d7208bc6f5278f2ea1b6b" +content-hash = "30ade8462629ad243bf4029ae6c7d9fe784af2117aa85c903aadc3b6f6e2d4f9" python-versions = "^3.5" [metadata.hashes] appdirs = ["9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", "d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e"] atomicwrites = ["03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", "75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6"] attrs = ["10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", "ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb"] -black = ["817243426042db1d36617910df579a54f1afd659adb96fc5032fcf4b36209739", "e030a9a28f542debc08acceb273f228ac422798e5215ba2a791a6ddeaaca22a5"] +black = ["09a9dcb7c46ed496a9850b76e4e825d6049ecd38b611f1224857a79bd985a8cf", "68950ffd4d9169716bcb8719a56c07a2f4485354fec061cdd5910aa07369731c"] click = ["2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", "5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7"] colorama = ["05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d", "f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48"] -coverage = ["06123b58a1410873e22134ca2d88bd36680479fe354955b3579fb8ff150e4d27", "09e47c529ff77bf042ecfe858fb55c3e3eb97aac2c87f0349ab5a7efd6b3939f", "0a1f9b0eb3aa15c990c328535655847b3420231af299386cfe5efc98f9c250fe", "0cc941b37b8c2ececfed341444a456912e740ecf515d560de58b9a76562d966d", "0d34245f824cc3140150ab7848d08b7e2ba67ada959d77619c986f2062e1f0e8", "10e8af18d1315de936d67775d3a814cc81d0747a1a0312d84e27ae5610e313b0", "1b4276550b86caa60606bd3572b52769860a81a70754a54acc8ba789ce74d607", "1e8a2627c48266c7b813975335cfdea58c706fe36f607c97d9392e61502dc79d", "258b21c5cafb0c3768861a6df3ab0cfb4d8b495eee5ec660e16f928bf7385390", "2b224052bfd801beb7478b03e8a66f3f25ea56ea488922e98903914ac9ac930b", "3ad59c84c502cd134b0088ca9038d100e8fb5081bbd5ccca4863f3804d81f61d", "447c450a093766744ab53bf1e7063ec82866f27bcb4f4c907da25ad293bba7e3", "46101fc20c6f6568561cdd15a54018bb42980954b79aa46da8ae6f008066a30e", "4710dc676bb4b779c4361b54eb308bc84d64a2fa3d78e5f7228921eccce5d815", "510986f9a280cd05189b42eee2b69fecdf5bf9651d4cd315ea21d24a964a3c36", "5535dda5739257effef56e49a1c51c71f1d37a6e5607bb25a5eee507c59580d1", "5a7524042014642b39b1fcae85fb37556c200e64ec90824ae9ecf7b667ccfc14", "5f55028169ef85e1fa8e4b8b1b91c0b3b0fa3297c4fb22990d46ff01d22c2d6c", "6694d5573e7790a0e8d3d177d7a416ca5f5c150742ee703f3c18df76260de794", "6831e1ac20ac52634da606b658b0b2712d26984999c9d93f0c6e59fe62ca741b", "71afc1f5cd72ab97330126b566bbf4e8661aab7449f08895d21a5d08c6b051ff", "7349c27128334f787ae63ab49d90bf6d47c7288c63a0a5dfaa319d4b4541dd2c", "77f0d9fa5e10d03aa4528436e33423bfa3718b86c646615f04616294c935f840", "828ad813c7cdc2e71dcf141912c685bfe4b548c0e6d9540db6418b807c345ddd", "859714036274a75e6e57c7bab0c47a4602d2a8cfaaa33bbdb68c8359b2ed4f5c", "85a06c61598b14b015d4df233d249cd5abfa61084ef5b9f64a48e997fd829a82", "869ef4a19f6e4c6987e18b315721b8b971f7048e6eaea29c066854242b4e98d9", "8cb4febad0f0b26c6f62e1628f2053954ad2c555d67660f28dfb1b0496711952", "977e2d9a646773cc7428cdd9a34b069d6ee254fadfb4d09b3f430e95472f3cf3", "99bd767c49c775b79fdcd2eabff405f1063d9d959039c0bdd720527a7738748a", "a5c58664b23b248b16b96253880b2868fb34358911400a7ba39d7f6399935389", "aaa0f296e503cda4bc07566f592cd7a28779d433f3a23c48082af425d6d5a78f", "ab235d9fe64833f12d1334d29b558aacedfbca2356dfb9691f2d0d38a8a7bfb4", "b3b0c8f660fae65eac74fbf003f3103769b90012ae7a460863010539bb7a80da", "bab8e6d510d2ea0f1d14f12642e3f35cefa47a9b2e4c7cea1852b52bc9c49647", "c45297bbdbc8bb79b02cf41417d63352b70bcb76f1bbb1ee7d47b3e89e42f95d", "d19bca47c8a01b92640c614a9147b081a1974f69168ecd494687c827109e8f42", "d64b4340a0c488a9e79b66ec9f9d77d02b99b772c8b8afd46c1294c1d39ca478", "da969da069a82bbb5300b59161d8d7c8d423bc4ccd3b410a9b4d8932aeefc14b", "ed02c7539705696ecb7dc9d476d861f3904a8d2b7e894bd418994920935d36bb", "ee5b8abc35b549012e03a7b1e86c09491457dba6c94112a2482b18589cc2bdb9"] -more-itertools = ["0125e8f60e9e031347105eb1682cef932f5e97d7b9a1a28d9bf00c22a5daef40", "590044e3942351a1bdb1de960b739ff4ce277960f2425ad4509446dbace8d9d1"] +coverage = ["0c5fe441b9cfdab64719f24e9684502a59432df7570521563d7b1aff27ac755f", "2b412abc4c7d6e019ce7c27cbc229783035eef6d5401695dccba80f481be4eb3", "3684fabf6b87a369017756b551cef29e505cb155ddb892a7a29277b978da88b9", "39e088da9b284f1bd17c750ac672103779f7954ce6125fd4382134ac8d152d74", "3c205bc11cc4fcc57b761c2da73b9b72a59f8d5ca89979afb0c1c6f9e53c7390", "42692db854d13c6c5e9541b6ffe0fe921fe16c9c446358d642ccae1462582d3b", "465ce53a8c0f3a7950dfb836438442f833cf6663d407f37d8c52fe7b6e56d7e8", "48020e343fc40f72a442c8a1334284620f81295256a6b6ca6d8aa1350c763bbe", "4ec30ade438d1711562f3786bea33a9da6107414aed60a5daa974d50a8c2c351", "5296fc86ab612ec12394565c500b412a43b328b3907c0d14358950d06fd83baf", "5f61bed2f7d9b6a9ab935150a6b23d7f84b8055524e7be7715b6513f3328138e", "6899797ac384b239ce1926f3cb86ffc19996f6fa3a1efbb23cb49e0c12d8c18c", "68a43a9f9f83693ce0414d17e019daee7ab3f7113a70c79a3dd4c2f704e4d741", "6b8033d47fe22506856fe450470ccb1d8ba1ffb8463494a15cfc96392a288c09", "7ad7536066b28863e5835e8cfeaa794b7fe352d99a8cded9f43d1161be8e9fbd", "7bacb89ccf4bedb30b277e96e4cc68cd1369ca6841bde7b005191b54d3dd1034", "839dc7c36501254e14331bcb98b27002aa415e4af7ea039d9009409b9d2d5420", "8e679d1bde5e2de4a909efb071f14b472a678b788904440779d2c449c0355b27", "8f9a95b66969cdea53ec992ecea5406c5bd99c9221f539bca1e8406b200ae98c", "932c03d2d565f75961ba1d3cec41ddde00e162c5b46d03f7423edcb807734eab", "93f965415cc51604f571e491f280cff0f5be35895b4eb5e55b47ae90c02a497b", "988529edadc49039d205e0aa6ce049c5ccda4acb2d6c3c5c550c17e8c02c05ba", "998d7e73548fe395eeb294495a04d38942edb66d1fa61eb70418871bc621227e", "9de60893fb447d1e797f6bf08fdf0dbcda0c1e34c1b06c92bd3a363c0ea8c609", "9e80d45d0c7fcee54e22771db7f1b0b126fb4a6c0a2e5afa72f66827207ff2f2", "a545a3dfe5082dc8e8c3eb7f8a2cf4f2870902ff1860bd99b6198cfd1f9d1f49", "a5d8f29e5ec661143621a8f4de51adfb300d7a476224156a39a392254f70687b", "a9abc8c480e103dc05d9b332c6cc9fb1586330356fc14f1aa9c0ca5745097d19", "aca06bfba4759bbdb09bf52ebb15ae20268ee1f6747417837926fae990ebc41d", "bb23b7a6fd666e551a3094ab896a57809e010059540ad20acbeec03a154224ce", "bfd1d0ae7e292105f29d7deaa9d8f2916ed8553ab9d5f39ec65bcf5deadff3f9", "c22ab9f96cbaff05c6a84e20ec856383d27eae09e511d3e6ac4479489195861d", "c62ca0a38958f541a73cf86acdab020c2091631c137bd359c4f5bddde7b75fd4", "c709d8bda72cf4cd348ccec2a4881f2c5848fd72903c185f363d361b2737f773", "c968a6aa7e0b56ecbd28531ddf439c2ec103610d3e2bf3b75b813304f8cb7723", "ca58eba39c68010d7e87a823f22a081b5290e3e3c64714aac3c91481d8b34d22", "df785d8cb80539d0b55fd47183264b7002077859028dfe3070cf6359bf8b2d9c", "f406628ca51e0ae90ae76ea8398677a921b36f0bd71aab2099dfed08abd0322f", "f46087bbd95ebae244a0eda01a618aff11ec7a069b15a3ef8f6b520db523dcf1", "f8019c5279eb32360ca03e9fac40a12667715546eed5c5eb59eb381f2f501260", "fc5f4d209733750afd2714e9109816a29500718b32dd9a5db01c0cb3a019b96a"] +hypothesis = ["232f5e19de11e5da6ec78091a4c6cd8fe7b9454a15ebf9df38284ebc2435db78", "c3fd7aa8d22f4179089a42fd5f759b439de0a9dd02a228935d3c85e22780cdf2", "d23b44e711fcef554eda08328b88c7bd4143d4d0028c74118160643248916094"] +more-itertools = ["2112d2ca570bb7c3e53ea1a35cd5df42bb0fd10c45f0fb97178679c3c03d64c7", "c3e4748ba1aad8dba30a4886b0b1a2004f9a863837b8654e7059eebf727afa5a"] pathlib2 = ["25199318e8cc3c25dcb45cbe084cc061051336d5a9ea2a12448d3d8cb748f742", "5887121d7f7df3603bca2f710e7219f3eca0eb69e0b7cc6e0a022e155ac931a7"] -pluggy = ["8ddc32f03971bfdf900a81961a48ccf2fb677cf7715108f85295c67405798616", "980710797ff6a041e9a73a5787804f848996ecaa6f8a1b1e08224a5894f2074a"] -py = ["bf92637198836372b520efcba9e020c330123be8ce527e535d185ed4b6f45694", "e76826342cefe3c3d5f7e8ee4316b80d1dd8a300781612ddbc765c17ba25a6c6"] -pytest = ["80cfd9c8b9e93f419abcc0400e9f595974a98e44b6863a77d3e1039961bfc9c4", "c2396a15726218a2dfef480861c4ba37bd3952ebaaa5b0fede3fc23fddcd7f8c"] +pluggy = ["19ecf9ce9db2fce065a7a0586e07cfb4ac8614fe96edf628a264b1c70116cf8f", "84d306a647cc805219916e62aab89caa97a33a1dd8c342e87a37f91073cd4746"] +py = ["64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa", "dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53"] +pytest = ["13c5e9fb5ec5179995e9357111ab089af350d788cbc944c628f3cde72285809b", "f21d2f1fb8200830dcbb5d8ec466a9c9120e20d8b53c7585d180125cce1d297a"] pytest-cov = ["0ab664b25c6aa9716cbf203b17ddb301932383046082c081b9848a0edf5add33", "230ef817450ab0699c6cc3c9c8f7a829c34674456f2ed8df1fe1d39780f7c87f"] python-dateutil = ["7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", "c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e"] six = ["3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", "d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"] diff --git a/pyproject.toml b/pyproject.toml index e5c6f55..099bf81 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,8 @@ python-dateutil = {version="^2.7", python = "<3.7"} pytest = "^4.1" attrs = "^18.2" pytest-cov = "^2.6" -black = {version = "^18.3-alpha.0",allows-prereleases = true,python = ">=3.7"} +black = {version = "^19.3-beta.0",allows-prereleases = true,python = ">=3.7"} +hypothesis = "^4.14" [tool.black] line-length = 88 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/_strategies.py b/tests/_strategies.py new file mode 100644 index 0000000..300850f --- /dev/null +++ b/tests/_strategies.py @@ -0,0 +1,131 @@ +''' +Some miscellany to keep the type_strategies module a bit more readable. +''' +from hypothesis import strategies as st + +import attr +try: + import dataclasses as dc +except ImportError: + dc = None +from datetime import date +from enum import IntEnum +from keyword import iskeyword +import os +import typing + + +_max_cp = None if os.environ.get('UNICODE_NAMES') else 0x7f +_any_char = st.characters(min_codepoint=1, max_codepoint=_max_cp) +_ident_start = st.characters(whitelist_categories=['Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nl'], + max_codepoint=_max_cp) +_ident_tail = st.characters(whitelist_categories=['Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nl', 'Mn', 'Mc', 'Nd', 'Pc'], + whitelist_characters="_", + max_codepoint=_max_cp) + + +@st.composite +def _idents(draw, lengths=st.integers(min_value=0, max_value=80)): + chars = [draw(_ident_start)] + chars.extend(draw(_ident_tail) for _ in range(draw(lengths))) + chars = ''.join(chars) + if iskeyword(chars): + chars += draw(_ident_tail) + return chars + + +def _make_enum(name, elems): + # print(f'IntEnum(enum_{name}, {elems!r})') + return IntEnum('enum_' + name, elems) + + +idents = _idents() +enums = st.builds(_make_enum, idents, st.lists(idents, min_size=1, max_size=12, unique=True)) + + +def fields_idents(types): + return st.dictionaries(idents, types, dict_class=list, min_size=0, max_size=6) + + +class _Faux(attr.validators._InstanceOfValidator): + def __call__(self, inst, attr, value): + pass + + +def attrs(types, frozen): + def _make(name, fields, **kw): + def _attrib(typ): + # Add a bogus validator because from_type reads that, not `type` + # Can't use the real one because of generic types! + return attr.ib(type=typ, validator=_Faux(typ)) + # print(f'attrs({name}, {fields}, **{kw})') + return attr.make_class( + 'attrs_' + name, {field: _attrib(typ) for field, typ in fields}, frozen=frozen, **kw + ) + + return st.builds( + _make, + idents, + fields_idents(types), + slots=st.booleans() + ) + + +if dc is not None: + def dataclasses(types, frozen): + def _make(name, fields, order): + # print(f'dataclass({name}, {fields}, frozen={frozen}, order={order}') + return dc.make_dataclass('dc_' + name, fields, frozen=frozen, eq=True, order=order) + + return st.builds(_make, idents, fields_idents(types), + order=st.booleans()) +else: + def dataclasses(types, frozen): + return None + + +try: + _NamedTuple = typing.NamedTuple +except AttributeError: + def namedtuples(types): + return None +else: + def namedtuples(types): + def _make(name, fields): + # print(f'namedtuple({name}, {fields})') + return _NamedTuple('nt_' + name, fields) + return st.builds(_make, idents, fields_idents(types)) + + +def lists(types): + return st.builds(lambda a: typing.List[a], types) + + +def hmg_tuples(types): + return st.builds(lambda a: typing.Tuple[a, ...], types) + + +def sets(types): + return st.builds(lambda a: typing.Set[a], types) + + +def frozensets(types): + return st.builds(lambda a: typing.FrozenSet[a], types) + + +_dict_keys = atoms = st.one_of([ + st.sampled_from([int, str, date]), + enums +]) + + +def dicts(val_types): + return st.builds(lambda k, v: typing.Dict[k, v], _dict_keys, val_types) + + +def prod_tuples(types): + return st.builds(lambda a: typing.Tuple[tuple(a)], st.lists(types)) + + +def unions(types, max_size=None): + return st.builds(lambda a: typing.Union[tuple(a)], st.lists(types, min_size=1, max_size=max_size)) diff --git a/tests/test_union_prop.py b/tests/test_union_prop.py new file mode 100644 index 0000000..de9cd63 --- /dev/null +++ b/tests/test_union_prop.py @@ -0,0 +1,26 @@ +import pytest +from hypothesis import given, settings, HealthCheck, reproduce_failure + +from . import type_strategies as ts + +import attr +from datetime import date, datetime +from decimal import Decimal +from enum import Enum +from itertools import product +from typing import Union, List, Tuple, Set, FrozenSet, Dict + +from json_syntax import std_ruleset +from json_syntax.helpers import PY2JSON, JSON2PY, INSP_PY, INSP_JSON, NoneType + + +@settings(suppress_health_check=[HealthCheck.too_slow], max_examples=100, deadline=None) +@given(ts.type_value_pairs(ts.complex_no_unions)) +def test_roundtrip(pair): + typ, py_value = pair + rs = std_ruleset() + act = rs.lookup(verb=PY2JSON, typ=typ) + json_value = act(py_value) + act2 = rs.lookup(verb=JSON2PY, typ=typ) + rt_py_value = act2(json_value) + assert py_value == rt_py_value diff --git a/tests/type_strategies.py b/tests/type_strategies.py new file mode 100644 index 0000000..2ade981 --- /dev/null +++ b/tests/type_strategies.py @@ -0,0 +1,135 @@ +from hypothesis import strategies as st + +from decimal import Decimal +import datetime as dt +from enum import Enum + +from . import _strategies as _st + + +# Tests often want to compare for equality, and there's no good way to do this with NaNs breaking it. :-( +st.register_type_strategy(Decimal, st.decimals(allow_nan=False)) +st.register_type_strategy(float, st.floats(allow_nan=False)) + + +def type_value_pairs(base): + @st.composite + def tv_pairs(draw): + typ = draw(base) + try: + val = draw(st.from_type(typ)) + except Exception as exc: + exc.args += (typ,) + raise + return (typ, val) + + return tv_pairs() + + +atoms = st.sampled_from([ + type(None), + bool, + int, + float, + Decimal, + str, + dt.date, + dt.datetime, + dt.time, + dt.timedelta, +]) + + +class Head(Enum): + def __init__(self, disposition): + self.disposition = disposition + self.atomic = disposition == 'atom' + self.hashable = disposition in ('atom', 'immut') + self.is_union = disposition == 'union' + + atoms = 'atom' + enums = 'atom' + lists = 'mut' + sets = 'mut' + dicts = 'mut' + mut_attrs = 'mut' + mut_dataclasses = 'mut' + hmg_tuples = 'immut' + frozensets = 'immut' + prod_tuples = 'immut' + frz_attrs = 'immut' + frz_dataclasses = 'immut' + namedtuples = 'immut' + unions = 'union' + + @classmethod + def short(cls, elems): + if isinstance(elems, (cls, str)): + elems = [elems] + out = set() + for elem in elems: + if isinstance(elem, cls): + out.add(elem) + elif isinstance(elem, str): + out.update(head for head in cls if head.disposition == elem) + return out + + +# Need to add: +# 1. default values to all of these +# 2. typeless variants +# 3. our own subclasses? + +def map_heads(types, frz_types): + H = Head + yield H.atoms, atoms + yield H.enums, _st.enums + if types: + yield H.lists, _st.lists(types) + yield H.unions, _st.unions(types) + yield H.mut_attrs, _st.attrs(types, frozen=False) + yield H.mut_dataclasses, _st.dataclasses(types, frozen=False) + yield H.dicts, _st.dicts(types) + if frz_types: + yield H.hmg_tuples, _st.hmg_tuples(frz_types) + yield H.sets, _st.sets(frz_types) + yield H.frozensets, _st.frozensets(frz_types) + yield H.prod_tuples, _st.prod_tuples(frz_types) + yield H.frz_attrs, _st.attrs(frz_types, frozen=True) + yield H.frz_dataclasses, _st.dataclasses(frz_types, frozen=True) + yield H.namedtuples, _st.namedtuples(frz_types) + + +def type_tree(*levels): + ''' + Constructs a type tree of a fixed maximum height based on the heads provided. + The last level must be leaves that can be contained by the levels above. + ''' + types, frz_types = None, None + + for level in map(Head.short, reversed(levels)): + tt = [] + frz_tt = [] + for head, typ in map_heads(types, frz_types): + if typ is None: + continue + if head in level: + tt.append(typ) + if head.hashable: + frz_tt.append(typ) + types = st.one_of(tt) if tt else None + frz_types = st.one_of(frz_tt) if frz_tt else None + + if types is None: + raise ValueError("No types for {}".format(levels)) + return types + + +complex_no_unions = type_tree( + {'atom', 'mut', 'immut'}, + {'atom', 'mut', 'immut'}, + {'atom', 'mut', 'immut'}, + {'atom'} +) + +unions_of_simple = type_tree({Head.unions}, {'atom', 'mut', 'immut'}, {'atom'}) From b6f2939ca6783394ab9f46891ae9e3fb8bb28b52 Mon Sep 17 00:00:00 2001 From: Ben Samuel Date: Thu, 4 Apr 2019 21:08:50 -0400 Subject: [PATCH 2/6] Add initial cut of pattern sort. - show_pattern verb - show_pattern implementation in standard rules - Pattern objects and matching logic --- json_syntax/attrs.py | 18 +++- json_syntax/helpers.py | 1 + json_syntax/pattern.py | 224 +++++++++++++++++++++++++++++++++++++++ json_syntax/std.py | 72 ++++++++----- json_syntax/unions.py | 27 +++-- tests/_strategies.py | 7 +- tests/test_union_prop.py | 13 +++ 7 files changed, 321 insertions(+), 41 deletions(-) create mode 100644 json_syntax/pattern.py diff --git a/json_syntax/attrs.py b/json_syntax/attrs.py index 8249db3..f9b6b04 100644 --- a/json_syntax/attrs.py +++ b/json_syntax/attrs.py @@ -3,6 +3,7 @@ PY2JSON, INSP_JSON, INSP_PY, + PATTERN, SENTINEL, has_origin, identity, @@ -18,6 +19,7 @@ convert_dict_to_attrs, convert_tuple_as_list, ) +from . import pattern as pat from functools import partial @@ -33,7 +35,7 @@ def attrs_classes( """ Handle an ``@attr.s`` or ``@dataclass`` decorated class. """ - if verb not in (JSON2PY, PY2JSON, INSP_PY, INSP_JSON): + if verb not in (JSON2PY, PY2JSON, INSP_PY, INSP_JSON, PATTERN): return try: fields = typ.__attrs_attrs__ @@ -59,7 +61,7 @@ def attrs_classes( ) if verb == PY2JSON: tup += (field.default,) - elif verb == INSP_JSON: + elif verb in (INSP_JSON, PATTERN): tup += (is_attrs_field_required(field),) inner_map.append(tup) @@ -82,6 +84,8 @@ def attrs_classes( return check pre_hook_method = getattr(typ, pre_hook, identity) return partial(check_dict, inner_map=inner_map, pre_hook=pre_hook_method) + elif verb == PATTERN: + return pat.Object.exact( (name, inner) for name, inner, req in inner_map if req ) def named_tuples(verb, typ, ctx): @@ -90,7 +94,7 @@ def named_tuples(verb, typ, ctx): Also handles a ``collections.namedtuple`` if you have a fallback handler. """ - if verb not in (JSON2PY, PY2JSON, INSP_PY, INSP_JSON) or not issub_safe(typ, tuple): + if verb not in (JSON2PY, PY2JSON, INSP_PY, INSP_JSON, PATTERN) or not issub_safe(typ, tuple): return try: fields = typ._field_types @@ -116,7 +120,7 @@ def named_tuples(verb, typ, ctx): ) if verb == PY2JSON: tup += (defaults.get(name, SENTINEL),) - elif verb == INSP_JSON: + elif verb in (INSP_JSON, PATTERN): tup += (name not in defaults,) inner_map.append(tup) @@ -133,6 +137,8 @@ def named_tuples(verb, typ, ctx): ) elif verb == INSP_JSON: return partial(check_dict, pre_hook=identity, inner_map=tuple(inner_map)) + elif verb == PATTERN: + return pat.Object.exact( (name, inner) for name, inner, req in inner_map if req ) def tuples(verb, typ, ctx): @@ -140,7 +146,7 @@ def tuples(verb, typ, ctx): Handle a ``Tuple[type, type, type]`` product type. Use a ``NamedTuple`` if you don't want a list. """ - if verb not in (JSON2PY, PY2JSON, INSP_PY, INSP_JSON) or not has_origin(typ, tuple): + if verb not in (JSON2PY, PY2JSON, INSP_PY, INSP_JSON, PATTERN) or not has_origin(typ, tuple): return args = typ.__args__ if Ellipsis in args: @@ -155,3 +161,5 @@ def tuples(verb, typ, ctx): return partial(check_tuple_as_list, inner=inner, con=tuple) elif verb == INSP_JSON: return partial(check_tuple_as_list, inner=inner, con=list) + elif verb == PATTERN: + return pat.Array.exact(inner) diff --git a/json_syntax/helpers.py b/json_syntax/helpers.py index 48b29ce..e1f1a05 100644 --- a/json_syntax/helpers.py +++ b/json_syntax/helpers.py @@ -9,6 +9,7 @@ PY2JSON = "python_to_json" INSP_JSON = "inspect_json" INSP_PY = "inspect_python" +PATTERN = "show_pattern" NoneType = type(None) SENTINEL = object() python_minor = sys.version_info[:2] diff --git a/json_syntax/pattern.py b/json_syntax/pattern.py new file mode 100644 index 0000000..883aa9f --- /dev/null +++ b/json_syntax/pattern.py @@ -0,0 +1,224 @@ +""" +Patterns to represent roughly what syntax will look like, and also to investigate whether +unions are potentially ambiguous. +""" +from functools import partial, lru_cache +from enum import IntEnum + +try: + import simplejson as json +except ImportError: + import json + + def _def(obj): + return obj.for_json() + _args = {'default': lambda obj: obj.for_json()} +else: + _args = {'for_json': True} + +dump = partial(json.dump, **_args) +dumps = partial(json.dumps, **_args) + + +class Matches(IntEnum): + ''' + This determines the degree to which one pattern can shadow another causing potential ambiguity. + ''' + always = 0 # Will always match + sometimes = 1 # It will sometimes match + potential = 2 # Can't prove it won't match + never = 3 # Provably won't match + + +@lru_cache(32) +def _reduce(left, right, reverse): + consider = (left, right) + seq = Matches + if reverse: + seq = reversed(seq) + for val in seq: + if val in consider: + return val + raise ValueError("Can't reduce against unknown type") + + +def _match_many(source, *, pos): + "Combine matches where all elements must match." + result = Matches.always if pos else Matches.never + stop = Matches.never if pos else Matches.always + for match in source: + result = _reduce(result, match, reverse=pos) + if result == stop: + break + return result + + +match_all = partial(_match_many, pos=True) +match_any = partial(_match_many, pos=False) + + +def matches(left, right, ctx=None): + if ctx is None: + ctx = (set(), set()) + else: + if left in ctx[0] or right in ctx[1]: + return Matches.never + ctx[0].add(left) + ctx[1].add(right) + return match_any(left._matches(right, ctx) for left, right + in product(left._unpack(), right._unpack())) + + +class Pattern: + def _matches(self, other, ctx): + raise NotImplementedError() + + def _unpack(self): + return [self] + + def __repr__(self): + return dumps(self, indent=2) + + +class Atom: + def __init__(self, value): + self.value = value + + def _matches(self, other, ctx): + return isinstance(other, Atom) and self.value == other.value + + +class String(Pattern): + """ + Rather than try to analyze regular expressions, we just name common string patterns, + and have a list of known ambiguities. + + We're deliberately not trying to analyze regexes here as we assume you would want to + use specialize logic to make such fine distinctions. + """ + def __init__(self, name, arg=None): + self.name = name + self.arg = arg + + def for_json(self): + if name == 'exact': + return '=' + arg + else: + return name + + def _matches(self, other, ctx): + "Check whether this pattern will match the other." + if not isinstance(other, StringPattern): + return Matches.never + if self.name == 'str': + return Matches.always # Strings always overshadow + elif other.name == 'str': + return Matches.sometimes # Strings are sometimes shadowed + if self.name == 'exact': + if other.name == 'exact': + return Matches.always if self.arg == other.arg else Matches.never + elif other.arg is None: + return Matches.potential + else: + return Matches.always if other.arg(self.arg) else Matches.never + return Matches.always if self.name == other.name else Matches.potential + + +class _Missing(Pattern): + def _matches(self, other, ctx): + return Matches.never + + def __repr__(self): + return '' + + +String.any = String('str') +Number = Atom(0) +Null = Atom(None) +Bool = Atom(False) +Missing = _Missing() + + +class Alternatives(Pattern): + """ + Used by the `show_pattern` verb to represent alternative patterns in unions. + """ + + def __init__(self, alts): + self.alts = tuple(alts) + + def _unpack(self): + yield from self.alts + + def _matches(self, other, ctx): + raise NotImplementedError("Didn't call unpack") # Should be bypassed by _unpack. + + def for_json(self): + out = ['alts'] + out.extend(self.alts) + return out + + +class Array(Pattern): + def __init__(self, elems, *, homog): + self.elems = tuple(elems) + self.homog = homog + + @classmethod + def homog(cls, elem): + return cls((elem,), homog=True) + + @classmethod + def exact(cls, elems): + return cls(elems, homog=False) + + def _matches(self, other, ctx): + if not isinstance(other, Array): + return Matches.never + left = self.elems + right = other.elems + if self.homog and not other.homog: + left = cycle(left) + elif not self.homog and other.homog: + right = cycle(right) + + return matches_all( + matches(l, r, ctx) for l, r + in zip_longest(left, right, fillvalue=Missing) + ) + + def for_json(self): + out = ['...'] if self.homog else ['exact'] + out.extend(self.elems) + return out + + +class Object(Pattern): + def __init__(self, items, *, homog): + self.items = tuple(items) + self.homog = homog + + @classmethod + def homog(cls, key, val): + return cls(((key, val),), homog=True) + + @classmethod + def exact(cls, items): + return cls(items, homog=False) + + def _matches(self, other, ctx): + if not isinstance(other, Object): + return Matches.never + + return matches_all( + matches_any( + matches(l, r, ctx) + for r in right + ) for l in left + ) + + def for_json(self): + out = dict(self.items) + if self.homog: + out['...'] = '...' + return out diff --git a/json_syntax/std.py b/json_syntax/std.py index 9640385..4ec6290 100644 --- a/json_syntax/std.py +++ b/json_syntax/std.py @@ -7,6 +7,7 @@ PY2JSON, INSP_JSON, INSP_PY, + PATTERN, ) from .action_v1 import ( check_collection, @@ -31,6 +32,7 @@ convert_time, convert_timedelta_str, ) +from . import pattern as pat from collections import OrderedDict from datetime import datetime, date, time, timedelta @@ -64,6 +66,10 @@ def atoms(verb, typ, ctx): for base in (NoneType, str, bool, int): if issubclass(typ, base): return partial(check_isinst, typ=base) + elif verb == PATTERN: + for base, pat in [(NoneType, pat.Null), (str, pat.String.any), (bool, pat.Bool), (int, pat.Number)]: + if issubclass(typ, base): + return pat def floats(verb, typ, ctx): @@ -85,6 +91,8 @@ def floats(verb, typ, ctx): return partial(check_isinst, typ=float) elif verb == INSP_JSON: return partial(check_isinst, typ=(int, float)) + elif verb == PATTERN: + return pat.Number def floats_nan_str(verb, typ, ctx): @@ -104,6 +112,8 @@ def floats_nan_str(verb, typ, ctx): return partial(check_isinst, typ=float) elif verb == INSP_JSON: return check_float + elif verb == PATTERN: + return pat.Number def decimals(verb, typ, ctx): @@ -121,6 +131,8 @@ def decimals(verb, typ, ctx): return Decimal elif verb in (INSP_JSON, INSP_PY): return partial(check_isinst, typ=Decimal) + elif verb == PATTERN: + return pat.Number def decimals_as_str(verb, typ, ctx): @@ -138,8 +150,9 @@ def decimals_as_str(verb, typ, ctx): return convert_decimal_str elif verb == INSP_PY: return partial(check_isinst, typ=Decimal) - elif verb == INSP_JSON: - return partial(check_parse_error, parser=Decimal, error=ArithmeticError) + elif verb in (INSP_JSON, PATTERN): + inspect = partial(check_parse_error, parser=Decimal, error=ArithmeticError) + return pat.String('number', inspect) if verb == PATTERN else inspect def iso_dates(verb, typ, ctx): @@ -157,7 +170,7 @@ def iso_dates(verb, typ, ctx): return convert_timedelta_str if typ == timedelta else typ.isoformat elif verb == INSP_PY: return partial(check_has_type, typ=typ) - elif verb in (JSON2PY, INSP_JSON): + elif verb in (JSON2PY, INSP_JSON, PATTERN): if typ == date: parse = convert_date elif typ == datetime: @@ -170,10 +183,10 @@ def iso_dates(verb, typ, ctx): return if verb == JSON2PY: return parse - else: - return partial( - check_parse_error, parser=parse, error=(TypeError, ValueError) - ) + inspect = partial( + check_parse_error, parser=parse, error=(TypeError, ValueError) + ) + return pat.String(typ.__name__, inspect) if verb == PATTERN else inspect def enums(verb, typ, ctx): @@ -185,8 +198,9 @@ def enums(verb, typ, ctx): return partial(convert_str_enum, mapping=dict(typ.__members__)) elif verb == INSP_PY: return partial(check_isinst, typ=typ) - elif verb == INSP_JSON: - return partial(check_str_enum, mapping=frozenset(typ.__members__.keys())) + elif verb in (INSP_JSON, PATTERN): + inspect = partial(check_str_enum, mapping=frozenset(typ.__members__.keys())) + return pat.String(typ.__name__, inspect) if verb == PATTERN else inspect def faux_enums(verb, typ, ctx): @@ -195,15 +209,16 @@ def faux_enums(verb, typ, ctx): if verb in (JSON2PY, PY2JSON): mapping = {name: name for name in typ.__members__} return partial(convert_str_enum, mapping=mapping) - elif verb in (INSP_JSON, INSP_PY): - return partial(check_str_enum, mapping=frozenset(typ.__members__.keys())) + elif verb in (INSP_JSON, INSP_PY, PATTERN): + inspect = partial(check_str_enum, mapping=frozenset(typ.__members__.keys())) + return pat.String(typ.__name__, inspect) if verb == PATTERN else inspect def optional(verb, typ, ctx): """ Handle an ``Optional[inner]`` by passing ``None`` through. """ - if verb not in (JSON2PY, PY2JSON, INSP_PY, INSP_JSON): + if verb not in (JSON2PY, PY2JSON, INSP_PY, INSP_JSON, PATTERN): return if has_origin(typ, Union, num_args=2): if NoneType not in typ.__args__: @@ -221,6 +236,8 @@ def optional(verb, typ, ctx): return partial(convert_optional, inner=inner) elif verb in (INSP_JSON, INSP_PY): return partial(check_optional, inner=inner) + elif verb == PATTERN: + return pat.Alternatives([None, inner]) def lists(verb, typ, ctx): @@ -230,7 +247,7 @@ def lists(verb, typ, ctx): Trivia: the ellipsis indicates a homogenous tuple; ``Tuple[A, B, C]`` is a product type that contains exactly those elements. """ - if verb not in (JSON2PY, PY2JSON, INSP_PY, INSP_JSON): + if verb not in (JSON2PY, PY2JSON, INSP_PY, INSP_JSON, PATTERN): return if has_origin(typ, list, num_args=1): (inner,) = typ.__args__ @@ -241,28 +258,32 @@ def lists(verb, typ, ctx): else: return inner = ctx.lookup(verb=verb, typ=inner) - con = list if verb in (PY2JSON, INSP_JSON) else get_origin(typ) + con = list if verb in (PY2JSON, INSP_JSON, PATTERN) else get_origin(typ) if verb in (JSON2PY, PY2JSON): return partial(convert_collection, inner=inner, con=con) elif verb in (INSP_JSON, INSP_PY): return partial(check_collection, inner=inner, con=con) + elif verb == PATTERN: + return pat.Array.homog(inner) def sets(verb, typ, ctx): """ Handle a ``Set[type]`` or ``FrozenSet[type]``. """ - if verb not in (JSON2PY, PY2JSON, INSP_PY, INSP_JSON): + if verb not in (JSON2PY, PY2JSON, INSP_PY, INSP_JSON, PATTERN): return if not has_origin(typ, (set, frozenset), num_args=1): return (inner,) = typ.__args__ - con = list if verb in (PY2JSON, INSP_JSON) else get_origin(typ) + con = list if verb in (PY2JSON, INSP_JSON, PATTERN) else get_origin(typ) inner = ctx.lookup(verb=verb, typ=inner) if verb in (JSON2PY, PY2JSON): return partial(convert_collection, inner=inner, con=con) elif verb in (INSP_JSON, INSP_PY): return partial(check_collection, inner=inner, con=con) + elif verb == PATTERN: + return pat.Array.homog(inner) def _stringly(verb, typ, ctx): @@ -271,16 +292,19 @@ def _stringly(verb, typ, ctx): This is used internally by dicts. """ - if verb not in (JSON2PY, PY2JSON, INSP_PY, INSP_JSON) or not issub_safe( - typ, (int, str, date, Enum) - ): - return for base in str, int: if issubclass(typ, base): + if verb == PATTERN and base == str: + return pat.String.any if verb in (JSON2PY, PY2JSON): return base - elif verb in (INSP_JSON, INSP_PY): + elif verb == INSP_PY: return partial(check_isinst, typ=base) + elif verb in (INSP_JSON, PATTERN): + inspect = partial(check_parse_error, parser=base, error=ValueError) + return pat.String(typ.__name__, inspect) if verb == PATTERN else inspect + if issubclass(typ, (datetime, time)): + return for rule in enums, iso_dates: action = rule(verb=verb, typ=typ, ctx=ctx) if action is not None: @@ -289,10 +313,8 @@ def _stringly(verb, typ, ctx): def dicts(verb, typ, ctx): """ - Handle a ``Dict[key, value]`` where key is a string, integer or enum type. + Handle a ``Dict[key, value]`` where key is a string, integer, date or enum type. """ - if verb not in (JSON2PY, PY2JSON, INSP_PY, INSP_JSON): - return if not has_origin(typ, (dict, OrderedDict), num_args=2): return (key_type, val_type) = typ.__args__ @@ -304,3 +326,5 @@ def dicts(verb, typ, ctx): return partial(convert_mapping, key=key_type, val=val_type, con=get_origin(typ)) elif verb in (INSP_JSON, INSP_PY): return partial(check_mapping, key=key_type, val=val_type, con=get_origin(typ)) + elif verb == PATTERN: + return pat.Object.homog(key_type, val_type) diff --git a/json_syntax/unions.py b/json_syntax/unions.py index 9034bca..2caf4ba 100644 --- a/json_syntax/unions.py +++ b/json_syntax/unions.py @@ -1,5 +1,6 @@ from .helpers import has_origin, JSON2PY, PY2JSON, INSP_JSON, INSP_PY from .action_v1 import convert_union, check_union +from . import pattern as pat from functools import partial from typing import Union @@ -30,15 +31,23 @@ def unions(verb, typ, ctx): check_verb = INSP_JSON else: return - steps = [] - for arg in typ.__args__: - check = ctx.lookup(verb=check_verb, typ=arg) - convert = ctx.lookup(verb=verb, typ=arg) - steps.append((check, convert, "<{!s}>".format(arg))) + steps = [ + (ctx.lookup(verb=check_verb, typ=arg), + ctx.lookup(verb=verb, typ=arg), + "<{!s}>".format(arg)) + for arg in typ.__args__ + ] return partial(convert_union, steps=steps, typename=repr(typ)) elif verb in (INSP_JSON, INSP_PY): - steps = [] - for arg in typ.__args__: - check = ctx.lookup(verb=verb, typ=arg) - steps.append((check, "<{!s}>".format(arg))) + steps = [ + (ctx.lookup(verb=verb, typ=arg), + "<{!s}>".format(arg)) + for arg in typ.__args__ + ] return partial(check_union, steps=steps) + elif verb == PATTERN: + alts = [ + ctx.lookup(verb=verb, typ=arg) + for arg in typ.__args__ + ] + return pat.Alternatives(alts) diff --git a/tests/_strategies.py b/tests/_strategies.py index 300850f..25f78cc 100644 --- a/tests/_strategies.py +++ b/tests/_strategies.py @@ -15,6 +15,7 @@ import typing +MAX_FIELDS = 8 _max_cp = None if os.environ.get('UNICODE_NAMES') else 0x7f _any_char = st.characters(min_codepoint=1, max_codepoint=_max_cp) _ident_start = st.characters(whitelist_categories=['Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nl'], @@ -40,11 +41,11 @@ def _make_enum(name, elems): idents = _idents() -enums = st.builds(_make_enum, idents, st.lists(idents, min_size=1, max_size=12, unique=True)) +enums = st.builds(_make_enum, idents, st.lists(idents, min_size=1, max_size=MAX_FIELDS, unique=True)) def fields_idents(types): - return st.dictionaries(idents, types, dict_class=list, min_size=0, max_size=6) + return st.dictionaries(idents, types, dict_class=list, min_size=0, max_size=MAX_FIELDS) class _Faux(attr.validators._InstanceOfValidator): @@ -124,7 +125,7 @@ def dicts(val_types): def prod_tuples(types): - return st.builds(lambda a: typing.Tuple[tuple(a)], st.lists(types)) + return st.builds(lambda a: typing.Tuple[tuple(a)], st.lists(types, min_size=1, max_size=MAX_FIELDS)) def unions(types, max_size=None): diff --git a/tests/test_union_prop.py b/tests/test_union_prop.py index de9cd63..16c4b7c 100644 --- a/tests/test_union_prop.py +++ b/tests/test_union_prop.py @@ -24,3 +24,16 @@ def test_roundtrip(pair): act2 = rs.lookup(verb=JSON2PY, typ=typ) rt_py_value = act2(json_value) assert py_value == rt_py_value + + +# @settings(suppress_health_check=[HealthCheck.too_slow], max_examples=100, deadline=None) +# @given(ts.type_value_pairs(ts.unions_of_simple)) +# def test_roundtrip_union_simple(pair): +# typ, py_value = pair +# rs = std_ruleset() +# act = rs.lookup(verb=PY2JSON, typ=typ) +# json_value = act(py_value) +# act2 = rs.lookup(verb=JSON2PY, typ=typ) +# rt_py_value = act2(json_value) +# rt_json_value = act(rt_py_value) +# assert py_value == rt_py_value or json_value == rt_json_value From d49700effadaa124bee23075fd5a4986e9245a3d Mon Sep 17 00:00:00 2001 From: Ben Samuel Date: Thu, 4 Apr 2019 21:21:00 -0400 Subject: [PATCH 3/6] Reformat with black. - Use new target-version argument in pyproject.toml. --- README.md | 2 +- json_syntax/action_v1.py | 32 ++++++++--------- json_syntax/attrs.py | 12 ++++--- json_syntax/pattern.py | 53 +++++++++++++-------------- json_syntax/std.py | 9 +++-- json_syntax/unions.py | 16 ++++----- pyproject.toml | 2 +- tests/_strategies.py | 77 +++++++++++++++++++++++++--------------- tests/type_strategies.py | 77 +++++++++++++++++++++------------------- 9 files changed, 155 insertions(+), 125 deletions(-) diff --git a/README.md b/README.md index 24c0723..66830c6 100644 --- a/README.md +++ b/README.md @@ -306,7 +306,7 @@ This package is maintained via the [poetry][] tool. Some useful commands: 1. Setup: `poetry install` 2. Run tests: `poetry run pytest tests/` - 3. Reformat: `poetry run black -N json_syntax/ tests/` + 3. Reformat: `poetry run black json_syntax/ tests/` ### Setting up tox diff --git a/json_syntax/action_v1.py b/json_syntax/action_v1.py index c15e7e7..7e9adf4 100644 --- a/json_syntax/action_v1.py +++ b/json_syntax/action_v1.py @@ -25,7 +25,7 @@ def check_has_type(value, typ): def convert_decimal_str(value): result = str(value) - if result == 'sNaN': + if result == "sNaN": raise InvalidOperation("Won't save signalling NaN") return result @@ -85,8 +85,8 @@ def convert_str_enum(value, mapping): def convert_timedelta_str(dur): "Barebones support for storing a timedelta as an ISO8601 duration." - micro = '.{:06d}'.format(dur.microseconds) if dur.microseconds else '' - return 'P{:d}DT{:d}{}S'.format(dur.days, dur.seconds, micro) + micro = ".{:06d}".format(dur.microseconds) if dur.microseconds else "" + return "P{:d}DT{:d}{}S".format(dur.days, dur.seconds, micro) _iso8601_duration = re.compile( @@ -97,36 +97,36 @@ def convert_timedelta_str(dur): r"(?:(T)(?=[0-9+-])" r"([-+]?\d+(?:[.,]\d+)?H)?" r"([-+]?\d+(?:[.,]\d+)?M)?" - r"([-+]?\d+(?:[.,]\d+)?S)?)?$", + r"([-+]?\d+(?:[.,]\d+)?S)?)?$" ) _duration_args = { - 'PW': 'weeks', - 'PD': 'days', - 'TH': 'hours', - 'TM': 'minutes', - 'TS': 'seconds' + "PW": "weeks", + "PD": "days", + "TH": "hours", + "TM": "minutes", + "TS": "seconds", } def convert_str_timedelta(dur): - match = _iso8601_duration.match(dur.upper().replace(',', '.')) - section = 'P' + match = _iso8601_duration.match(dur.upper().replace(",", ".")) + section = "P" if not match: - raise ValueError('Value was not an ISO8601 duration.') + raise ValueError("Value was not an ISO8601 duration.") args = {} for elem in match.groups(): if elem is None: continue - if elem == 'T': - section = 'T' + if elem == "T": + section = "T" continue part = section + elem[-1] value = float(elem[:-1]) if not value: continue - if part in ('PY', 'PM'): - raise ValueError('Year and month durations not supported') + if part in ("PY", "PM"): + raise ValueError("Year and month durations not supported") args[_duration_args[part]] = value return timedelta(**args) diff --git a/json_syntax/attrs.py b/json_syntax/attrs.py index f9b6b04..810f508 100644 --- a/json_syntax/attrs.py +++ b/json_syntax/attrs.py @@ -85,7 +85,7 @@ def attrs_classes( pre_hook_method = getattr(typ, pre_hook, identity) return partial(check_dict, inner_map=inner_map, pre_hook=pre_hook_method) elif verb == PATTERN: - return pat.Object.exact( (name, inner) for name, inner, req in inner_map if req ) + return pat.Object.exact((name, inner) for name, inner, req in inner_map if req) def named_tuples(verb, typ, ctx): @@ -94,7 +94,9 @@ def named_tuples(verb, typ, ctx): Also handles a ``collections.namedtuple`` if you have a fallback handler. """ - if verb not in (JSON2PY, PY2JSON, INSP_PY, INSP_JSON, PATTERN) or not issub_safe(typ, tuple): + if verb not in (JSON2PY, PY2JSON, INSP_PY, INSP_JSON, PATTERN) or not issub_safe( + typ, tuple + ): return try: fields = typ._field_types @@ -138,7 +140,7 @@ def named_tuples(verb, typ, ctx): elif verb == INSP_JSON: return partial(check_dict, pre_hook=identity, inner_map=tuple(inner_map)) elif verb == PATTERN: - return pat.Object.exact( (name, inner) for name, inner, req in inner_map if req ) + return pat.Object.exact((name, inner) for name, inner, req in inner_map if req) def tuples(verb, typ, ctx): @@ -146,7 +148,9 @@ def tuples(verb, typ, ctx): Handle a ``Tuple[type, type, type]`` product type. Use a ``NamedTuple`` if you don't want a list. """ - if verb not in (JSON2PY, PY2JSON, INSP_PY, INSP_JSON, PATTERN) or not has_origin(typ, tuple): + if verb not in (JSON2PY, PY2JSON, INSP_PY, INSP_JSON, PATTERN) or not has_origin( + typ, tuple + ): return args = typ.__args__ if Ellipsis in args: diff --git a/json_syntax/pattern.py b/json_syntax/pattern.py index 883aa9f..e3bdfaf 100644 --- a/json_syntax/pattern.py +++ b/json_syntax/pattern.py @@ -12,18 +12,20 @@ def _def(obj): return obj.for_json() - _args = {'default': lambda obj: obj.for_json()} + + _args = {"default": lambda obj: obj.for_json()} else: - _args = {'for_json': True} + _args = {"for_json": True} dump = partial(json.dump, **_args) dumps = partial(json.dumps, **_args) class Matches(IntEnum): - ''' + """ This determines the degree to which one pattern can shadow another causing potential ambiguity. - ''' + """ + always = 0 # Will always match sometimes = 1 # It will sometimes match potential = 2 # Can't prove it won't match @@ -65,8 +67,10 @@ def matches(left, right, ctx=None): return Matches.never ctx[0].add(left) ctx[1].add(right) - return match_any(left._matches(right, ctx) for left, right - in product(left._unpack(), right._unpack())) + return match_any( + left._matches(right, ctx) + for left, right in product(left._unpack(), right._unpack()) + ) class Pattern: @@ -96,13 +100,14 @@ class String(Pattern): We're deliberately not trying to analyze regexes here as we assume you would want to use specialize logic to make such fine distinctions. """ + def __init__(self, name, arg=None): self.name = name self.arg = arg def for_json(self): - if name == 'exact': - return '=' + arg + if name == "exact": + return "=" + arg else: return name @@ -110,12 +115,12 @@ def _matches(self, other, ctx): "Check whether this pattern will match the other." if not isinstance(other, StringPattern): return Matches.never - if self.name == 'str': + if self.name == "str": return Matches.always # Strings always overshadow - elif other.name == 'str': + elif other.name == "str": return Matches.sometimes # Strings are sometimes shadowed - if self.name == 'exact': - if other.name == 'exact': + if self.name == "exact": + if other.name == "exact": return Matches.always if self.arg == other.arg else Matches.never elif other.arg is None: return Matches.potential @@ -129,10 +134,10 @@ def _matches(self, other, ctx): return Matches.never def __repr__(self): - return '' + return "" -String.any = String('str') +String.any = String("str") Number = Atom(0) Null = Atom(None) Bool = Atom(False) @@ -151,10 +156,12 @@ def _unpack(self): yield from self.alts def _matches(self, other, ctx): - raise NotImplementedError("Didn't call unpack") # Should be bypassed by _unpack. + raise NotImplementedError( + "Didn't call unpack" + ) # Should be bypassed by _unpack. def for_json(self): - out = ['alts'] + out = ["alts"] out.extend(self.alts) return out @@ -183,12 +190,11 @@ def _matches(self, other, ctx): right = cycle(right) return matches_all( - matches(l, r, ctx) for l, r - in zip_longest(left, right, fillvalue=Missing) + matches(l, r, ctx) for l, r in zip_longest(left, right, fillvalue=Missing) ) def for_json(self): - out = ['...'] if self.homog else ['exact'] + out = ["..."] if self.homog else ["exact"] out.extend(self.elems) return out @@ -210,15 +216,10 @@ def _matches(self, other, ctx): if not isinstance(other, Object): return Matches.never - return matches_all( - matches_any( - matches(l, r, ctx) - for r in right - ) for l in left - ) + return matches_all(matches_any(matches(l, r, ctx) for r in right) for l in left) def for_json(self): out = dict(self.items) if self.homog: - out['...'] = '...' + out["..."] = "..." return out diff --git a/json_syntax/std.py b/json_syntax/std.py index 4ec6290..42dbaab 100644 --- a/json_syntax/std.py +++ b/json_syntax/std.py @@ -67,7 +67,12 @@ def atoms(verb, typ, ctx): if issubclass(typ, base): return partial(check_isinst, typ=base) elif verb == PATTERN: - for base, pat in [(NoneType, pat.Null), (str, pat.String.any), (bool, pat.Bool), (int, pat.Number)]: + for base, pat in [ + (NoneType, pat.Null), + (str, pat.String.any), + (bool, pat.Bool), + (int, pat.Number), + ]: if issubclass(typ, base): return pat @@ -152,7 +157,7 @@ def decimals_as_str(verb, typ, ctx): return partial(check_isinst, typ=Decimal) elif verb in (INSP_JSON, PATTERN): inspect = partial(check_parse_error, parser=Decimal, error=ArithmeticError) - return pat.String('number', inspect) if verb == PATTERN else inspect + return pat.String("number", inspect) if verb == PATTERN else inspect def iso_dates(verb, typ, ctx): diff --git a/json_syntax/unions.py b/json_syntax/unions.py index 2caf4ba..50c7cec 100644 --- a/json_syntax/unions.py +++ b/json_syntax/unions.py @@ -32,22 +32,20 @@ def unions(verb, typ, ctx): else: return steps = [ - (ctx.lookup(verb=check_verb, typ=arg), - ctx.lookup(verb=verb, typ=arg), - "<{!s}>".format(arg)) + ( + ctx.lookup(verb=check_verb, typ=arg), + ctx.lookup(verb=verb, typ=arg), + "<{!s}>".format(arg), + ) for arg in typ.__args__ ] return partial(convert_union, steps=steps, typename=repr(typ)) elif verb in (INSP_JSON, INSP_PY): steps = [ - (ctx.lookup(verb=verb, typ=arg), - "<{!s}>".format(arg)) + (ctx.lookup(verb=verb, typ=arg), "<{!s}>".format(arg)) for arg in typ.__args__ ] return partial(check_union, steps=steps) elif verb == PATTERN: - alts = [ - ctx.lookup(verb=verb, typ=arg) - for arg in typ.__args__ - ] + alts = [ctx.lookup(verb=verb, typ=arg) for arg in typ.__args__] return pat.Alternatives(alts) diff --git a/pyproject.toml b/pyproject.toml index 099bf81..862284a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ hypothesis = "^4.14" [tool.black] line-length = 88 -py36 = true +target-version = ["py35"] [tool.tox] legacy_tox_ini = """ diff --git a/tests/_strategies.py b/tests/_strategies.py index 25f78cc..743dc4e 100644 --- a/tests/_strategies.py +++ b/tests/_strategies.py @@ -1,9 +1,10 @@ -''' +""" Some miscellany to keep the type_strategies module a bit more readable. -''' +""" from hypothesis import strategies as st import attr + try: import dataclasses as dc except ImportError: @@ -16,20 +17,23 @@ MAX_FIELDS = 8 -_max_cp = None if os.environ.get('UNICODE_NAMES') else 0x7f +_max_cp = None if os.environ.get("UNICODE_NAMES") else 0x7F _any_char = st.characters(min_codepoint=1, max_codepoint=_max_cp) -_ident_start = st.characters(whitelist_categories=['Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nl'], - max_codepoint=_max_cp) -_ident_tail = st.characters(whitelist_categories=['Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nl', 'Mn', 'Mc', 'Nd', 'Pc'], - whitelist_characters="_", - max_codepoint=_max_cp) +_ident_start = st.characters( + whitelist_categories=["Lu", "Ll", "Lt", "Lm", "Lo", "Nl"], max_codepoint=_max_cp +) +_ident_tail = st.characters( + whitelist_categories=["Lu", "Ll", "Lt", "Lm", "Lo", "Nl", "Mn", "Mc", "Nd", "Pc"], + whitelist_characters="_", + max_codepoint=_max_cp, +) @st.composite def _idents(draw, lengths=st.integers(min_value=0, max_value=80)): chars = [draw(_ident_start)] chars.extend(draw(_ident_tail) for _ in range(draw(lengths))) - chars = ''.join(chars) + chars = "".join(chars) if iskeyword(chars): chars += draw(_ident_tail) return chars @@ -37,15 +41,19 @@ def _idents(draw, lengths=st.integers(min_value=0, max_value=80)): def _make_enum(name, elems): # print(f'IntEnum(enum_{name}, {elems!r})') - return IntEnum('enum_' + name, elems) + return IntEnum("enum_" + name, elems) idents = _idents() -enums = st.builds(_make_enum, idents, st.lists(idents, min_size=1, max_size=MAX_FIELDS, unique=True)) +enums = st.builds( + _make_enum, idents, st.lists(idents, min_size=1, max_size=MAX_FIELDS, unique=True) +) def fields_idents(types): - return st.dictionaries(idents, types, dict_class=list, min_size=0, max_size=MAX_FIELDS) + return st.dictionaries( + idents, types, dict_class=list, min_size=0, max_size=MAX_FIELDS + ) class _Faux(attr.validators._InstanceOfValidator): @@ -59,28 +67,32 @@ def _attrib(typ): # Add a bogus validator because from_type reads that, not `type` # Can't use the real one because of generic types! return attr.ib(type=typ, validator=_Faux(typ)) + # print(f'attrs({name}, {fields}, **{kw})') return attr.make_class( - 'attrs_' + name, {field: _attrib(typ) for field, typ in fields}, frozen=frozen, **kw + "attrs_" + name, + {field: _attrib(typ) for field, typ in fields}, + frozen=frozen, + **kw, ) - return st.builds( - _make, - idents, - fields_idents(types), - slots=st.booleans() - ) + return st.builds(_make, idents, fields_idents(types), slots=st.booleans()) if dc is not None: + def dataclasses(types, frozen): def _make(name, fields, order): # print(f'dataclass({name}, {fields}, frozen={frozen}, order={order}') - return dc.make_dataclass('dc_' + name, fields, frozen=frozen, eq=True, order=order) + return dc.make_dataclass( + "dc_" + name, fields, frozen=frozen, eq=True, order=order + ) + + return st.builds(_make, idents, fields_idents(types), order=st.booleans()) + - return st.builds(_make, idents, fields_idents(types), - order=st.booleans()) else: + def dataclasses(types, frozen): return None @@ -88,13 +100,18 @@ def dataclasses(types, frozen): try: _NamedTuple = typing.NamedTuple except AttributeError: + def namedtuples(types): return None + + else: + def namedtuples(types): def _make(name, fields): # print(f'namedtuple({name}, {fields})') - return _NamedTuple('nt_' + name, fields) + return _NamedTuple("nt_" + name, fields) + return st.builds(_make, idents, fields_idents(types)) @@ -114,10 +131,7 @@ def frozensets(types): return st.builds(lambda a: typing.FrozenSet[a], types) -_dict_keys = atoms = st.one_of([ - st.sampled_from([int, str, date]), - enums -]) +_dict_keys = atoms = st.one_of([st.sampled_from([int, str, date]), enums]) def dicts(val_types): @@ -125,8 +139,13 @@ def dicts(val_types): def prod_tuples(types): - return st.builds(lambda a: typing.Tuple[tuple(a)], st.lists(types, min_size=1, max_size=MAX_FIELDS)) + return st.builds( + lambda a: typing.Tuple[tuple(a)], + st.lists(types, min_size=1, max_size=MAX_FIELDS), + ) def unions(types, max_size=None): - return st.builds(lambda a: typing.Union[tuple(a)], st.lists(types, min_size=1, max_size=max_size)) + return st.builds( + lambda a: typing.Union[tuple(a)], st.lists(types, min_size=1, max_size=max_size) + ) diff --git a/tests/type_strategies.py b/tests/type_strategies.py index 2ade981..e0705bd 100644 --- a/tests/type_strategies.py +++ b/tests/type_strategies.py @@ -26,41 +26,43 @@ def tv_pairs(draw): return tv_pairs() -atoms = st.sampled_from([ - type(None), - bool, - int, - float, - Decimal, - str, - dt.date, - dt.datetime, - dt.time, - dt.timedelta, -]) +atoms = st.sampled_from( + [ + type(None), + bool, + int, + float, + Decimal, + str, + dt.date, + dt.datetime, + dt.time, + dt.timedelta, + ] +) class Head(Enum): def __init__(self, disposition): self.disposition = disposition - self.atomic = disposition == 'atom' - self.hashable = disposition in ('atom', 'immut') - self.is_union = disposition == 'union' - - atoms = 'atom' - enums = 'atom' - lists = 'mut' - sets = 'mut' - dicts = 'mut' - mut_attrs = 'mut' - mut_dataclasses = 'mut' - hmg_tuples = 'immut' - frozensets = 'immut' - prod_tuples = 'immut' - frz_attrs = 'immut' - frz_dataclasses = 'immut' - namedtuples = 'immut' - unions = 'union' + self.atomic = disposition == "atom" + self.hashable = disposition in ("atom", "immut") + self.is_union = disposition == "union" + + atoms = "atom" + enums = "atom" + lists = "mut" + sets = "mut" + dicts = "mut" + mut_attrs = "mut" + mut_dataclasses = "mut" + hmg_tuples = "immut" + frozensets = "immut" + prod_tuples = "immut" + frz_attrs = "immut" + frz_dataclasses = "immut" + namedtuples = "immut" + unions = "union" @classmethod def short(cls, elems): @@ -80,6 +82,7 @@ def short(cls, elems): # 2. typeless variants # 3. our own subclasses? + def map_heads(types, frz_types): H = Head yield H.atoms, atoms @@ -101,10 +104,10 @@ def map_heads(types, frz_types): def type_tree(*levels): - ''' + """ Constructs a type tree of a fixed maximum height based on the heads provided. The last level must be leaves that can be contained by the levels above. - ''' + """ types, frz_types = None, None for level in map(Head.short, reversed(levels)): @@ -126,10 +129,10 @@ def type_tree(*levels): complex_no_unions = type_tree( - {'atom', 'mut', 'immut'}, - {'atom', 'mut', 'immut'}, - {'atom', 'mut', 'immut'}, - {'atom'} + {"atom", "mut", "immut"}, + {"atom", "mut", "immut"}, + {"atom", "mut", "immut"}, + {"atom"}, ) -unions_of_simple = type_tree({Head.unions}, {'atom', 'mut', 'immut'}, {'atom'}) +unions_of_simple = type_tree({Head.unions}, {"atom", "mut", "immut"}, {"atom"}) From f16b286ed83e5442b90e4d555923cf2da09407f1 Mon Sep 17 00:00:00 2001 From: Ben Samuel Date: Fri, 12 Apr 2019 12:54:46 -0400 Subject: [PATCH 4/6] Add ambiguity testing for unions. --- json_syntax/__init__.py | 2 +- json_syntax/action_v1.py | 2 + json_syntax/attrs.py | 4 +- json_syntax/pattern.py | 155 ++++++++++++++++++++++++++------------- json_syntax/ruleset.py | 14 +++- json_syntax/std.py | 6 +- json_syntax/unions.py | 2 +- tests/test_union_prop.py | 27 ++++--- 8 files changed, 144 insertions(+), 68 deletions(-) diff --git a/json_syntax/__init__.py b/json_syntax/__init__.py index b840df2..a33227d 100644 --- a/json_syntax/__init__.py +++ b/json_syntax/__init__.py @@ -22,7 +22,7 @@ ) from .attrs import attrs_classes, named_tuples, tuples from .unions import unions -from .helpers import JSON2PY, PY2JSON, INSP_PY, INSP_JSON # noqa +from .helpers import JSON2PY, PY2JSON, INSP_PY, INSP_JSON, PATTERN # noqa def std_ruleset( diff --git a/json_syntax/action_v1.py b/json_syntax/action_v1.py index 7e9adf4..d80a23b 100644 --- a/json_syntax/action_v1.py +++ b/json_syntax/action_v1.py @@ -109,6 +109,8 @@ def convert_timedelta_str(dur): def convert_str_timedelta(dur): + if not isinstance(dur, str): + raise ValueError("Value was not a string.") match = _iso8601_duration.match(dur.upper().replace(",", ".")) section = "P" if not match: diff --git a/json_syntax/attrs.py b/json_syntax/attrs.py index 810f508..0c5d940 100644 --- a/json_syntax/attrs.py +++ b/json_syntax/attrs.py @@ -85,7 +85,7 @@ def attrs_classes( pre_hook_method = getattr(typ, pre_hook, identity) return partial(check_dict, inner_map=inner_map, pre_hook=pre_hook_method) elif verb == PATTERN: - return pat.Object.exact((name, inner) for name, inner, req in inner_map if req) + return pat.Object.exact((pat.String.exact(name), inner or pat.Unkown) for name, inner, req in inner_map if req) def named_tuples(verb, typ, ctx): @@ -140,7 +140,7 @@ def named_tuples(verb, typ, ctx): elif verb == INSP_JSON: return partial(check_dict, pre_hook=identity, inner_map=tuple(inner_map)) elif verb == PATTERN: - return pat.Object.exact((name, inner) for name, inner, req in inner_map if req) + return pat.Object.exact((pat.String.exact(name), inner) for name, inner, req in inner_map if req) def tuples(verb, typ, ctx): diff --git a/json_syntax/pattern.py b/json_syntax/pattern.py index e3bdfaf..6402d80 100644 --- a/json_syntax/pattern.py +++ b/json_syntax/pattern.py @@ -2,7 +2,8 @@ Patterns to represent roughly what syntax will look like, and also to investigate whether unions are potentially ambiguous. """ -from functools import partial, lru_cache +from functools import partial, lru_cache, singledispatch +from itertools import chain, cycle, islice, product, zip_longest from enum import IntEnum try: @@ -32,45 +33,22 @@ class Matches(IntEnum): never = 3 # Provably won't match -@lru_cache(32) -def _reduce(left, right, reverse): - consider = (left, right) - seq = Matches - if reverse: - seq = reversed(seq) - for val in seq: - if val in consider: - return val - raise ValueError("Can't reduce against unknown type") - - -def _match_many(source, *, pos): - "Combine matches where all elements must match." - result = Matches.always if pos else Matches.never - stop = Matches.never if pos else Matches.always - for match in source: - result = _reduce(result, match, reverse=pos) - if result == stop: - break - return result - - -match_all = partial(_match_many, pos=True) -match_any = partial(_match_many, pos=False) +matches_all = partial(max, default=Matches.always) +matches_any = partial(min, default=Matches.never) def matches(left, right, ctx=None): if ctx is None: - ctx = (set(), set()) + ctx = set() else: - if left in ctx[0] or right in ctx[1]: - return Matches.never - ctx[0].add(left) - ctx[1].add(right) - return match_any( + if (left, right) in ctx: + return Matches.potential + ctx.add((left, right)) + result = matches_any( left._matches(right, ctx) for left, right in product(left._unpack(), right._unpack()) ) + return result class Pattern: @@ -84,12 +62,15 @@ def __repr__(self): return dumps(self, indent=2) -class Atom: +class Atom(Pattern): def __init__(self, value): self.value = value + def for_json(self): + return self.value + def _matches(self, other, ctx): - return isinstance(other, Atom) and self.value == other.value + return Matches.always if isinstance(other, Atom) and self.value == other.value else Matches.never class String(Pattern): @@ -106,14 +87,19 @@ def __init__(self, name, arg=None): self.arg = arg def for_json(self): - if name == "exact": - return "=" + arg + if self.name == "exact": + return "=" + self.arg else: - return name + return self.name + + @classmethod + def exact(cls, string): + assert isinstance(string, str) + return cls("exact", string) def _matches(self, other, ctx): "Check whether this pattern will match the other." - if not isinstance(other, StringPattern): + if not isinstance(other, String): return Matches.never if self.name == "str": return Matches.always # Strings always overshadow @@ -129,19 +115,24 @@ def _matches(self, other, ctx): return Matches.always if self.name == other.name else Matches.potential -class _Missing(Pattern): +class _Unknown(Pattern): + def __init__(self, name, match): + self._name = name + self._match = match + def _matches(self, other, ctx): - return Matches.never + return self._match def __repr__(self): - return "" + return self._name String.any = String("str") Number = Atom(0) Null = Atom(None) Bool = Atom(False) -Missing = _Missing() +Missing = _Unknown('', Matches.never) +Unknown = _Unknown('', Matches.potential) class Alternatives(Pattern): @@ -151,6 +142,7 @@ class Alternatives(Pattern): def __init__(self, alts): self.alts = tuple(alts) + assert all(isinstance(alt, Pattern) for alt in self.alts) def _unpack(self): yield from self.alts @@ -169,6 +161,7 @@ def for_json(self): class Array(Pattern): def __init__(self, elems, *, homog): self.elems = tuple(elems) + assert all(isinstance(elem, Pattern) for elem in self.elems) self.homog = homog @classmethod @@ -184,24 +177,36 @@ def _matches(self, other, ctx): return Matches.never left = self.elems right = other.elems + zero_match = False if self.homog and not other.homog: - left = cycle(left) + left = islice(cycle(left), len(right)) elif not self.homog and other.homog: - right = cycle(right) + right = islice(cycle(right), len(left)) + elif self.homog and other.homog: + zero_match = True - return matches_all( + possible = matches_all( matches(l, r, ctx) for l, r in zip_longest(left, right, fillvalue=Missing) ) + if zero_match: + possible = matches_any([Matches.sometimes, possible]) + return possible def for_json(self): - out = ["..."] if self.homog else ["exact"] - out.extend(self.elems) + out = list(self.elems) + if self.homog: + out.append("...") return out class Object(Pattern): def __init__(self, items, *, homog): self.items = tuple(items) + valid = all(isinstance(key, Pattern) and isinstance(val, Pattern) for key, val in self.items) + if not valid: + # for key, val in self.items: + # print(f"{key!r}: {type(key)} / {val!r}: {type(val)}") + raise TypeError("Keys and values must be patterns") self.homog = homog @classmethod @@ -216,10 +221,62 @@ def _matches(self, other, ctx): if not isinstance(other, Object): return Matches.never - return matches_all(matches_any(matches(l, r, ctx) for r in right) for l in left) + return matches_all( + matches_any(matches(lk, rk, ctx) and matches(lv, rv, ctx) for rk, rv in other.items) + for lk, lv in self.items + ) def for_json(self): - out = dict(self.items) + def jsonify(key): + try: + for_json = key.for_json + except AttributeError: + return key + else: + return for_json() + out = {jsonify(k): v for k, v in self.items} if self.homog: out["..."] = "..." return out + + +@singledispatch +def is_ambiguous(pattern, threshold=Matches.always, _path=()): + raise TypeError("pattern must be a recognized subclass of Pattern.") + +@is_ambiguous.register(Atom) +@is_ambiguous.register(String) +def _(pattern, threshold=Matches.always, _path=()): + return () + + +@is_ambiguous.register(_Unknown) +def _(pattern, threshold=Matches.always, _path=()): + return (str(pattern),) if pattern._match <= threshold else () + + +def _any(iterable): + for item in iterable: + if bool(item): + return item + return () + +@is_ambiguous.register(Array) +def _(pattern, threshold=Matches.always, _path=()): + _path += ('[]',) + return _any(is_ambiguous(elem, threshold, _path) for elem in pattern.elems) + +@is_ambiguous.register(Object) +def _(pattern, threshold=Matches.always, _path=()): + return _any(is_ambiguous(val, threshold, _path + (str(key),)) for key, val in pattern.items) + +@is_ambiguous.register(Alternatives) +def _(pattern, threshold=Matches.always, _path=()): + # An ambiguous pattern is one where an earlier pattern shadows a later pattern. + alts = pattern.alts + for i, early in enumerate(alts[:-1]): + for late in alts[i + 1:]: + if matches(early, late) <= threshold: + return _path + (f'alternative {i}',) + + return () diff --git a/json_syntax/ruleset.py b/json_syntax/ruleset.py index f241c25..695e808 100644 --- a/json_syntax/ruleset.py +++ b/json_syntax/ruleset.py @@ -1,4 +1,6 @@ from .cache import SimpleCache +from .helpers import PATTERN +from . import pattern import logging @@ -52,5 +54,15 @@ def lookup(self, verb, typ, accept_missing=False): finally: self.cache.de_flight(verb=verb, typ=typ, forward=forward) + if action is None and not accept_missing: + raise ValueError("Failed: lookup({!s}, {!r}".format(verb, typ)) + def fallback(self, verb, typ): - pass + if verb == PATTERN: + return pattern.Unknown + else: + return None + + def is_ambiguous(self, typ, threshold=pattern.Matches.always): + pat = self.lookup(verb=PATTERN, typ=typ) + return pattern.is_ambiguous(pat, threshold=threshold) diff --git a/json_syntax/std.py b/json_syntax/std.py index 42dbaab..dd071ac 100644 --- a/json_syntax/std.py +++ b/json_syntax/std.py @@ -67,14 +67,14 @@ def atoms(verb, typ, ctx): if issubclass(typ, base): return partial(check_isinst, typ=base) elif verb == PATTERN: - for base, pat in [ + for base, node in [ (NoneType, pat.Null), (str, pat.String.any), (bool, pat.Bool), (int, pat.Number), ]: if issubclass(typ, base): - return pat + return node def floats(verb, typ, ctx): @@ -242,7 +242,7 @@ def optional(verb, typ, ctx): elif verb in (INSP_JSON, INSP_PY): return partial(check_optional, inner=inner) elif verb == PATTERN: - return pat.Alternatives([None, inner]) + return pat.Alternatives([pat.Null, inner]) def lists(verb, typ, ctx): diff --git a/json_syntax/unions.py b/json_syntax/unions.py index 50c7cec..2279e3c 100644 --- a/json_syntax/unions.py +++ b/json_syntax/unions.py @@ -1,4 +1,4 @@ -from .helpers import has_origin, JSON2PY, PY2JSON, INSP_JSON, INSP_PY +from .helpers import has_origin, JSON2PY, PY2JSON, INSP_JSON, INSP_PY, PATTERN from .action_v1 import convert_union, check_union from . import pattern as pat diff --git a/tests/test_union_prop.py b/tests/test_union_prop.py index 16c4b7c..eed4092 100644 --- a/tests/test_union_prop.py +++ b/tests/test_union_prop.py @@ -12,6 +12,7 @@ from json_syntax import std_ruleset from json_syntax.helpers import PY2JSON, JSON2PY, INSP_PY, INSP_JSON, NoneType +from json_syntax.pattern import Matches @settings(suppress_health_check=[HealthCheck.too_slow], max_examples=100, deadline=None) @@ -26,14 +27,18 @@ def test_roundtrip(pair): assert py_value == rt_py_value -# @settings(suppress_health_check=[HealthCheck.too_slow], max_examples=100, deadline=None) -# @given(ts.type_value_pairs(ts.unions_of_simple)) -# def test_roundtrip_union_simple(pair): -# typ, py_value = pair -# rs = std_ruleset() -# act = rs.lookup(verb=PY2JSON, typ=typ) -# json_value = act(py_value) -# act2 = rs.lookup(verb=JSON2PY, typ=typ) -# rt_py_value = act2(json_value) -# rt_json_value = act(rt_py_value) -# assert py_value == rt_py_value or json_value == rt_json_value +@settings(suppress_health_check=[HealthCheck.too_slow], max_examples=1000, deadline=None) +@given(ts.type_value_pairs(ts.unions_of_simple)) +def test_roundtrip_union_simple(pair): + typ, py_value = pair + rs = std_ruleset() + print() + print() + print(f'{py_value!r} : {typ}') + print(rs.lookup(verb="show_pattern", typ=typ)) + act = rs.lookup(verb=PY2JSON, typ=typ) + json_value = act(py_value) + act2 = rs.lookup(verb=JSON2PY, typ=typ) + rt_py_value = act2(json_value) + if not rs.is_ambiguous(typ=typ, threshold=Matches.sometimes): + assert py_value == rt_py_value From b744897f746272c71642112d088909f24c5ee3b6 Mon Sep 17 00:00:00 2001 From: Ben Samuel Date: Fri, 12 Apr 2019 15:54:19 -0400 Subject: [PATCH 5/6] Get union property tests working. --- json_syntax/attrs.py | 10 +++++-- json_syntax/pattern.py | 59 +++++++++++++++++++++++++++------------- tests/test_union_prop.py | 19 +++++++++---- tests/type_strategies.py | 7 +++++ 4 files changed, 69 insertions(+), 26 deletions(-) diff --git a/json_syntax/attrs.py b/json_syntax/attrs.py index 0c5d940..555086a 100644 --- a/json_syntax/attrs.py +++ b/json_syntax/attrs.py @@ -85,7 +85,11 @@ def attrs_classes( pre_hook_method = getattr(typ, pre_hook, identity) return partial(check_dict, inner_map=inner_map, pre_hook=pre_hook_method) elif verb == PATTERN: - return pat.Object.exact((pat.String.exact(name), inner or pat.Unkown) for name, inner, req in inner_map if req) + return pat.Object.exact( + (pat.String.exact(name), inner or pat.Unkown) + for name, inner, req in inner_map + if req + ) def named_tuples(verb, typ, ctx): @@ -140,7 +144,9 @@ def named_tuples(verb, typ, ctx): elif verb == INSP_JSON: return partial(check_dict, pre_hook=identity, inner_map=tuple(inner_map)) elif verb == PATTERN: - return pat.Object.exact((pat.String.exact(name), inner) for name, inner, req in inner_map if req) + return pat.Object.exact( + (pat.String.exact(name), inner) for name, inner, req in inner_map if req + ) def tuples(verb, typ, ctx): diff --git a/json_syntax/pattern.py b/json_syntax/pattern.py index 6402d80..f98b54d 100644 --- a/json_syntax/pattern.py +++ b/json_syntax/pattern.py @@ -70,7 +70,11 @@ def for_json(self): return self.value def _matches(self, other, ctx): - return Matches.always if isinstance(other, Atom) and self.value == other.value else Matches.never + return ( + Matches.always + if isinstance(other, Atom) and self.value == other.value + else Matches.never + ) class String(Pattern): @@ -131,8 +135,8 @@ def __repr__(self): Number = Atom(0) Null = Atom(None) Bool = Atom(False) -Missing = _Unknown('', Matches.never) -Unknown = _Unknown('', Matches.potential) +Missing = _Unknown("", Matches.never) +Unknown = _Unknown("", Matches.potential) class Alternatives(Pattern): @@ -162,7 +166,7 @@ class Array(Pattern): def __init__(self, elems, *, homog): self.elems = tuple(elems) assert all(isinstance(elem, Pattern) for elem in self.elems) - self.homog = homog + self._homog = homog @classmethod def homog(cls, elem): @@ -175,20 +179,20 @@ def exact(cls, elems): def _matches(self, other, ctx): if not isinstance(other, Array): return Matches.never + if self._homog and not other.elems: + return Matches.always left = self.elems right = other.elems - zero_match = False - if self.homog and not other.homog: + if self._homog and not other._homog: left = islice(cycle(left), len(right)) - elif not self.homog and other.homog: + elif not self._homog and other._homog: right = islice(cycle(right), len(left)) - elif self.homog and other.homog: - zero_match = True possible = matches_all( matches(l, r, ctx) for l, r in zip_longest(left, right, fillvalue=Missing) ) - if zero_match: + if self._homog and other._homog: + # Zero cases can't be distinguished match. possible = matches_any([Matches.sometimes, possible]) return possible @@ -202,12 +206,15 @@ def for_json(self): class Object(Pattern): def __init__(self, items, *, homog): self.items = tuple(items) - valid = all(isinstance(key, Pattern) and isinstance(val, Pattern) for key, val in self.items) + valid = all( + isinstance(key, Pattern) and isinstance(val, Pattern) + for key, val in self.items + ) if not valid: # for key, val in self.items: # print(f"{key!r}: {type(key)} / {val!r}: {type(val)}") raise TypeError("Keys and values must be patterns") - self.homog = homog + self._homog = homog @classmethod def homog(cls, key, val): @@ -220,11 +227,18 @@ def exact(cls, items): def _matches(self, other, ctx): if not isinstance(other, Object): return Matches.never + if self._homog and not other.items: + return Matches.always - return matches_all( - matches_any(matches(lk, rk, ctx) and matches(lv, rv, ctx) for rk, rv in other.items) + possible = matches_all( + matches_any( + matches(lk, rk, ctx) and matches(lv, rv, ctx) for rk, rv in other.items + ) for lk, lv in self.items ) + if self._homog and other._homog: + possible = matches_any([Matches.sometimes, possible]) + return possible def for_json(self): def jsonify(key): @@ -234,8 +248,9 @@ def jsonify(key): return key else: return for_json() + out = {jsonify(k): v for k, v in self.items} - if self.homog: + if self._homog: out["..."] = "..." return out @@ -244,6 +259,7 @@ def jsonify(key): def is_ambiguous(pattern, threshold=Matches.always, _path=()): raise TypeError("pattern must be a recognized subclass of Pattern.") + @is_ambiguous.register(Atom) @is_ambiguous.register(String) def _(pattern, threshold=Matches.always, _path=()): @@ -261,22 +277,27 @@ def _any(iterable): return item return () + @is_ambiguous.register(Array) def _(pattern, threshold=Matches.always, _path=()): - _path += ('[]',) + _path += ("[]",) return _any(is_ambiguous(elem, threshold, _path) for elem in pattern.elems) + @is_ambiguous.register(Object) def _(pattern, threshold=Matches.always, _path=()): - return _any(is_ambiguous(val, threshold, _path + (str(key),)) for key, val in pattern.items) + return _any( + is_ambiguous(val, threshold, _path + (str(key),)) for key, val in pattern.items + ) + @is_ambiguous.register(Alternatives) def _(pattern, threshold=Matches.always, _path=()): # An ambiguous pattern is one where an earlier pattern shadows a later pattern. alts = pattern.alts for i, early in enumerate(alts[:-1]): - for late in alts[i + 1:]: + for late in alts[i + 1 :]: if matches(early, late) <= threshold: - return _path + (f'alternative {i}',) + return _path + (f"alternative {i}",) return () diff --git a/tests/test_union_prop.py b/tests/test_union_prop.py index eed4092..ad99fb4 100644 --- a/tests/test_union_prop.py +++ b/tests/test_union_prop.py @@ -27,15 +27,24 @@ def test_roundtrip(pair): assert py_value == rt_py_value -@settings(suppress_health_check=[HealthCheck.too_slow], max_examples=1000, deadline=None) +@settings(suppress_health_check=[HealthCheck.too_slow], max_examples=100, deadline=None) @given(ts.type_value_pairs(ts.unions_of_simple)) def test_roundtrip_union_simple(pair): typ, py_value = pair rs = std_ruleset() - print() - print() - print(f'{py_value!r} : {typ}') - print(rs.lookup(verb="show_pattern", typ=typ)) + act = rs.lookup(verb=PY2JSON, typ=typ) + json_value = act(py_value) + act2 = rs.lookup(verb=JSON2PY, typ=typ) + rt_py_value = act2(json_value) + if not rs.is_ambiguous(typ=typ, threshold=Matches.sometimes): + assert py_value == rt_py_value + + +@settings(suppress_health_check=[HealthCheck.too_slow], max_examples=100, deadline=None) +@given(ts.type_value_pairs(ts.complex_anything)) +def test_roundtrip_arbitrary_complex(pair): + typ, py_value = pair + rs = std_ruleset() act = rs.lookup(verb=PY2JSON, typ=typ) json_value = act(py_value) act2 = rs.lookup(verb=JSON2PY, typ=typ) diff --git a/tests/type_strategies.py b/tests/type_strategies.py index e0705bd..84dcdf8 100644 --- a/tests/type_strategies.py +++ b/tests/type_strategies.py @@ -136,3 +136,10 @@ def type_tree(*levels): ) unions_of_simple = type_tree({Head.unions}, {"atom", "mut", "immut"}, {"atom"}) + +complex_anything = type_tree( + {"atom", "mut", "immut", "unions"}, + {"atom", "mut", "immut", "unions"}, + {"atom", "mut", "immut", "unions"}, + {"atom"}, +) From e361001a47acd901b1129f4e5f107cb585ac0fd7 Mon Sep 17 00:00:00 2001 From: Ben Samuel Date: Fri, 12 Apr 2019 16:51:09 -0400 Subject: [PATCH 6/6] Clean up some issues breaking tox. Version bump; now beta. - Simplify test_union; still adds value as the instances are more readable. - Update README. --- README.md | 67 ++++++++++++++++++++++++++---------------- json_syntax/pattern.py | 2 +- pyproject.toml | 8 +++-- tests/test_union.py | 63 ++++++++++++++++----------------------- 4 files changed, 72 insertions(+), 68 deletions(-) diff --git a/README.md b/README.md index 66830c6..1fac5d4 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,8 @@ structure using libraries like [attrs][]. * The library has no dependencies of its own * It does not actually read or write JSON -At the time of writing, the library is in **alpha** and the API may move around or be -renamed. +At the time of writing, the library is in **beta** and the API is relatively stable but +may change. ### Supported types @@ -219,7 +219,7 @@ During encoding, the reverse sequence takes place: #### JSON type check hook -Type checks are only used in `json-syntax` to support `typing.Union`; in a nutshell, the +Type checks are only used in _json-syntax_ to support `typing.Union`; in a nutshell, the `unions` rule will inspect some JSON to see which variant is present. If a type-check hook is not defined, `__json_pre_decode__` will be called before the @@ -249,9 +249,40 @@ encode_account = rules.lookup(typ=Union[AccountA, AccountB, AccountC], See [the examples][] for details on custom rules. +### Debugging amibguous structures + +(May need more docs and some test cases.) + +As _json-syntax_ tries to directly translate your Python types to JSON, it is possible +to write ambiguous structures. To avoid this, there is a handy `is_ambiguous` method: + +```python +# This is true because both are represented as an array of numbers in JSON. +rules.is_ambiguous(typ=Union[List[int], Set[int]]) + +@dataclass +class Account: + user: str + address: str + +# This is true because such a dictionary would always match the contents of the account. +rules.is_ambiguous(typ=Union[Dict[str, str], Account]) +``` + +The aim of this is to let you put a check in your unit tests to make sure data can be +reliably expressed given your particular case. + +Internally, this is using the `PATTERN` verb to represent the JSON pattern, so this may +be helpful in understanding how _json-syntax_ is trying to represent your data: + +```python +print(rules.lookup(typ=MyAmbiguousClass, verb='show_pattern')) +``` + ### Sharp edges -_Alpha release status._ This API may change, there are probably bugs! +_Beta release status._ This API may change, there are probably bugs! In particular, the +status of rules accepting subclasses is likely to change. _The RuleSet caches encoders._ Construct a new ruleset if you want to change settings. @@ -265,28 +296,8 @@ _Everything to do with typing._ It's a bit magical and sort of wasn't designed f [We have a guide to it to try and help][types]. _Union types._ You can use `typing.Union` to allow a member to be one of some number of -alternates, but there are some caveats. These are documented in code in `test_unions`, -but in plain English: - -When encoding Python to JSON: - - * `Union[Super, Sub]` will never match Sub when converting from Python to JSON. - -When decoding JSON to Python: - - * `Union[str, Stringly]` will never construct an instance that is represented as a - string in JSON. - * This includes enums, dates and special float values (`Nan`, `-inf`, etc.) may be - represented as strings. - * `Union[datetime, date]` will never construct a date because `YYYY-MM-DD` is a valid - datetime according to ISO8601. - * `Union[Dict[str, Value], MyAttrs]` will never construct `MyAttrs` if all its - attributes are `Value`. - * `Union[List[X], Set[X], FrozenSet[X], Tuple[X, ...]]` will only ever construct - `List[X]` because all the others are also represented as JSON lists. - * `Union[MyClassA, MyClassB, MyClassC]` can be ambiguous if these classes all share - common fields. Consider using the `__json_check__` hook to differentiate. Simply - adding a field named `class` or something can be unambiguous and fast. +alternates, but there are some caveats. You should use the `.is_ambiguous()` method of +RuleSet to warn you of these. _Rules accept subclasses._ If you subclass `int`, the atoms rule will match it, and then the converter will call `int` against your instance. I haven't taken the time to examine @@ -322,6 +333,10 @@ You'll want pyenv, then install the pythons: Once you install `tox` in your preferred python, running it is just `tox`. +(Caveat: `poetry install` is now breaking in `tox` because `pip` has changed: it now +tries to create a dist in _pip-wheel-metadata_ each time. I'm nuking that directory, but +most likely there's some new config variable to hunt down.) + ### Notes 1: Writing the encoder is deceptively easy because the instances in diff --git a/json_syntax/pattern.py b/json_syntax/pattern.py index f98b54d..81027a2 100644 --- a/json_syntax/pattern.py +++ b/json_syntax/pattern.py @@ -298,6 +298,6 @@ def _(pattern, threshold=Matches.always, _path=()): for i, early in enumerate(alts[:-1]): for late in alts[i + 1 :]: if matches(early, late) <= threshold: - return _path + (f"alternative {i}",) + return _path + ("alternative {}".format(i),) return () diff --git a/pyproject.toml b/pyproject.toml index 862284a..27ff470 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,13 @@ [tool.poetry] name = "json-syntax" -version = "0.2.0" +version = "0.2.1" description = "Generates functions to convert Python classes to JSON dumpable objects." authors = ["Ben Samuel "] license = "MIT" readme = "README.md" repository = "https://github.com/UnitedIncome/json-syntax" classifiers = [ - "Development Status :: 3 - Alpha", + "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Topic :: Software Development :: Libraries" ] @@ -37,8 +37,10 @@ skipsdist = true [testenv] deps = poetry +whitelist_externals = + /bin/rm commands = - python -V + /bin/rm -r pip-wheel-metadata poetry install poetry run pytest {posargs} """ diff --git a/tests/test_union.py b/tests/test_union.py index 5f1ae9e..ff72b09 100644 --- a/tests/test_union.py +++ b/tests/test_union.py @@ -22,27 +22,41 @@ class Dir(Enum): DOWN = 2 -cases = [ +atoms = [ (NoneType, None, None), (bool, True, True), +] + +nums = [ (int, 5, 5), (float, 3.3, 3.3), (Decimal, Decimal("5.5"), Decimal("5.5")), +] + +strings = [ (str, "str", "str"), (date, date(2010, 10, 10), "2010-10-10"), (datetime, datetime(2011, 11, 11, 11, 11, 11), "2011-11-11T11:11:11"), - (Point, Point(x=4.5, y=6.6), {"x": 4.5, "y": 6.6}), - (Dir, Dir.UP, "UP"), + (Dir, Dir.UP, "UP") +] + +arrays = [ (List[Point], [Point(x=4.5, y=6.6)], [{"x": 4.5, "y": 6.6}]), (Tuple[Point, ...], (Point(x=4.5, y=6.6),), [{"x": 4.5, "y": 6.6}]), (Set[Point], {Point(x=4.5, y=6.6)}, [{"x": 4.5, "y": 6.6}]), (FrozenSet[Point], frozenset([Point(x=4.5, y=6.6)]), [{"x": 4.5, "y": 6.6}]), +] + +dicts = [ + (Point, Point(x=4.5, y=6.6), {"x": 4.5, "y": 6.6}), (Dict[Dir, Decimal], {Dir.UP: Decimal("7.7")}, {"UP": Decimal("7.7")}), (Dict[str, float], {"a": 2.3, "b": 3.4}, {"a": 2.3, "b": 3.4}), ] +cats = [atoms, nums, strings, arrays, dicts] -@pytest.mark.parametrize("typ,py,js", cases) + +@pytest.mark.parametrize("typ,py,js", [trip for cat in cats for trip in cat]) def test_simple(typ, py, js): rs = std_ruleset() act = rs.lookup(verb=PY2JSON, typ=typ) @@ -51,36 +65,18 @@ def test_simple(typ, py, js): assert act(js) == py -def ambiguous(left, right): - if left == str and right in {Dir, date, datetime}: - return "str prevents {} matching".format(right) - if left == date and right == datetime: - return "supertype date prevents subtype datetime matching" - if left == datetime and right == date: - return "dates in iso format are valid datetimes" - if left == Dict[str, float] and right == Point: - # Note that this is the case where the attrs class has homogenous fields - return "dict prevents attrs class matching" - ambiguous = {List[Point], Tuple[Point, ...], Set[Point], FrozenSet[Point]} - if left in ambiguous and right in ambiguous: - return "collections are all represented as json arrays" - return +def _pairs(): + for i in range(0, len(cats)): + lefts = cats[i] + rights = cats[(i + 2) % len(cats)] + yield from product(lefts, rights) def cvt_map(): - for left, right in product(cases, cases): - if left is right: - continue + for left, right in _pairs(): left_type, left_python, left_json = left right_type, right_python, right_json = right - if ( - left_json == right_json - or left_python == right_python - or ambiguous(left_type, right_type) - ): - continue - typ = Union[left_type, right_type] yield (PY2JSON, typ, left_python, left_json) yield (PY2JSON, typ, right_python, right_json) @@ -98,19 +94,10 @@ def test_convert_unions(verb, typ, subj, expect): def check_map(): - for left, right in product(cases, cases): - if left is right: - continue + for left, right in _pairs(): left_type, left_python, left_json = left right_type, right_python, right_json = right - if ( - left_json == right_json - or left_python == right_python - or ambiguous(left_type, right_type) - ): - continue - typ = Union[left_type, right_type] yield (INSP_PY, typ, left_python) yield (INSP_PY, typ, right_python)