From 03f4bf04842321a5a3d441437bec52d121f653fa Mon Sep 17 00:00:00 2001 From: Jace Browning Date: Wed, 1 Apr 2020 18:30:14 -0400 Subject: [PATCH] Fix serialization of optional nested dataclasses --- CHANGELOG.md | 4 ++ datafiles/converters/_bases.py | 2 +- datafiles/converters/containers.py | 11 +++-- datafiles/mapper.py | 8 ---- datafiles/tests/test_converters.py | 5 --- poetry.lock | 69 ++++++++++++++++++++---------- pyproject.toml | 2 +- tests/test_saving.py | 23 ++++++++++ 8 files changed, 82 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6dd37bc2..34e01456 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# 0.9 (beta) + +- Fixed serialization of optional nested dataclasses with a value of `None`. + # 0.8.1 (2020-03-30) - Fixed loading of `Missing` nested dataclasses attributes. diff --git a/datafiles/converters/_bases.py b/datafiles/converters/_bases.py index ba80a2c4..5879f5be 100644 --- a/datafiles/converters/_bases.py +++ b/datafiles/converters/_bases.py @@ -7,7 +7,7 @@ class Converter: """Base class for immutable attribute conversion.""" TYPE: type = object - DEFAULT: Any = None + DEFAULT: Any = NotImplemented @classmethod def as_optional(cls): diff --git a/datafiles/converters/containers.py b/datafiles/converters/containers.py index 1e6099e2..5c858f72 100644 --- a/datafiles/converters/containers.py +++ b/datafiles/converters/containers.py @@ -171,19 +171,22 @@ def to_python_value(cls, deserialized_data, *, target_object): def to_preserialization_data(cls, python_value, *, default_to_skip=None): data = {} + if python_value is None and cls.DEFAULT is None: + return None + for name, converter in cls.CONVERTERS.items(): if isinstance(python_value, dict): try: value = python_value[name] - except KeyError as e: - log.debug(e) + except KeyError: + log.debug(f'Added missing nested attribute: {name}') value = None else: try: value = getattr(python_value, name) - except AttributeError as e: - log.debug(e) + except AttributeError: + log.debug(f'Added missing nested attribute: {name}') value = None with suppress(AttributeError): diff --git a/datafiles/mapper.py b/datafiles/mapper.py index e8d19bd4..ea92774d 100644 --- a/datafiles/mapper.py +++ b/datafiles/mapper.py @@ -138,14 +138,6 @@ def _get_data(self, include_default_values: Trilean = None) -> Dict: if getattr(converter, 'DATACLASS', None): log.debug(f"Converting '{name}' dataclass with {converter}") - if value is None: - value = {} - - for field in dataclasses.fields(converter.DATACLASS): - if field.name not in value: - log.debug(f'Added missing nested attribute: {field.name}') - value[field.name] = None - data[name] = converter.to_preserialization_data( value, default_to_skip=Missing diff --git a/datafiles/tests/test_converters.py b/datafiles/tests/test_converters.py index 17b6d167..f9c68f9b 100644 --- a/datafiles/tests/test_converters.py +++ b/datafiles/tests/test_converters.py @@ -146,11 +146,6 @@ def when_immutable(expect, converter, data, value): (MyDataclassConverter, None, MyDataclass(foobar=0)), (MyDataclassConverterList, None, []), (MyDataclassConverterList, 42, [MyDataclass(foobar=0)]), - ( - MyNestedDataclassConverter, - None, - MyNestedDataclass(name='', dc=MyDataclass(foobar=0, flag=False)), - ), ], ) def when_mutable(expect, converter, data, value): diff --git a/poetry.lock b/poetry.lock index 4cf9cc8d..94189d89 100644 --- a/poetry.lock +++ b/poetry.lock @@ -43,6 +43,14 @@ wrapt = ">=1.11.0,<1.12.0" python = "<3.8" version = ">=1.4.0,<1.5" +[[package]] +category = "dev" +description = "Async generators and context managers for Python 3.5+" +name = "async-generator" +optional = false +python-versions = ">=3.5" +version = "1.10" + [[package]] category = "dev" description = "Atomic file writes." @@ -562,12 +570,14 @@ category = "dev" description = "A client library for executing notebooks. Formally nbconvert's ExecutePreprocessor." name = "nbclient" optional = false -python-versions = ">=3.5" -version = "0.1.0" +python-versions = ">=3.6" +version = "0.2.0" [package.dependencies] -jupyter-client = ">=5.3.4" +async-generator = "*" +jupyter-client = ">=6.1.0" nbformat = ">=5.0" +nest-asyncio = "*" traitlets = ">=4.2" [package.extras] @@ -608,7 +618,7 @@ description = "The Jupyter Notebook format" name = "nbformat" optional = false python-versions = ">=3.5" -version = "5.0.4" +version = "5.0.5" [package.dependencies] ipython-genutils = "*" @@ -630,6 +640,14 @@ version = "0.3.7" [package.dependencies] nbformat = "*" +[[package]] +category = "dev" +description = "Patch asyncio to allow nested event loops" +name = "nest-asyncio" +optional = false +python-versions = ">=3.5" +version = "1.3.2" + [[package]] category = "dev" description = "A web-based notebook environment for interactive computing" @@ -681,25 +699,22 @@ category = "dev" description = "Parametrize and run Jupyter and nteract Notebooks" name = "papermill" optional = false -python-versions = ">=3.5" -version = "2.0.0" +python-versions = ">=3.6" +version = "2.1.0" [package.dependencies] ansiwrap = "*" +black = "*" click = "*" entrypoints = "*" jupyter-client = "*" -nbclient = "*" +nbclient = ">=0.2.0" nbformat = "*" pyyaml = "*" requests = "*" tenacity = "*" tqdm = ">=4.32.2" -[package.dependencies.black] -python = ">=3.6" -version = "*" - [package.extras] all = ["boto3", "azure-datalake-store (>=0.0.30)", "azure-storage-blob (>=12.1.0)", "requests (>=2.21.0)", "gcsfs (>=0.2.0)", "pyarrow"] azure = ["azure-datalake-store (>=0.0.30)", "azure-storage-blob (>=12.1.0)", "requests (>=2.21.0)"] @@ -933,17 +948,17 @@ version = "2.1" [[package]] category = "dev" -description = "Thin-wrapper around the mock package for easier use with py.test" +description = "Thin-wrapper around the mock package for easier use with pytest" name = "pytest-mock" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "2.0.0" +python-versions = ">=3.5" +version = "3.0.0" [package.dependencies] pytest = ">=2.7" [package.extras] -dev = ["pre-commit", "tox"] +dev = ["pre-commit", "tox", "pytest-asyncio"] [[package]] category = "dev" @@ -1349,6 +1364,10 @@ astroid = [ {file = "astroid-2.3.3-py3-none-any.whl", hash = "sha256:840947ebfa8b58f318d42301cf8c0a20fd794a33b61cc4638e28e9e61ba32f42"}, {file = "astroid-2.3.3.tar.gz", hash = "sha256:71ea07f44df9568a75d0f354c49143a4575d90645e9fead6dfb52c26a85ed13a"}, ] +async-generator = [ + {file = "async_generator-1.10-py3-none-any.whl", hash = "sha256:01c7bf666359b4967d2cda0000cc2e4af16a0ae098cbffcb8472fb9e8ad6585b"}, + {file = "async_generator-1.10.tar.gz", hash = "sha256:6ebb3d106c12920aaae42ccb6f787ef5eefdcdd166ea3d628fa8476abe712144"}, +] atomicwrites = [ {file = "atomicwrites-1.3.0-py2.py3-none-any.whl", hash = "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4"}, {file = "atomicwrites-1.3.0.tar.gz", hash = "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6"}, @@ -1612,21 +1631,25 @@ mypy-extensions = [ {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, ] nbclient = [ - {file = "nbclient-0.1.0-py3-none-any.whl", hash = "sha256:02af062077ae0ec00201f52e207942a03a49e7dbb2dd7589c1713a3f40bea0df"}, - {file = "nbclient-0.1.0.tar.gz", hash = "sha256:39fa0b3b24cb597827f6b104718dc952c679a8f5ba2d6f1a4113a3bd7f1528c6"}, + {file = "nbclient-0.2.0-py3-none-any.whl", hash = "sha256:193731fd5039061dbd7d6d3f765a2fa59d23012594d68b1798deea9c3eae4a01"}, + {file = "nbclient-0.2.0.tar.gz", hash = "sha256:44dde0356def1d9345908c8f58dc604a434f2fe61c49ac13fac6e2da2ae429de"}, ] nbconvert = [ {file = "nbconvert-5.6.1-py2.py3-none-any.whl", hash = "sha256:f0d6ec03875f96df45aa13e21fd9b8450c42d7e1830418cccc008c0df725fcee"}, {file = "nbconvert-5.6.1.tar.gz", hash = "sha256:21fb48e700b43e82ba0e3142421a659d7739b65568cc832a13976a77be16b523"}, ] nbformat = [ - {file = "nbformat-5.0.4-py3-none-any.whl", hash = "sha256:f4bbbd8089bd346488f00af4ce2efb7f8310a74b2058040d075895429924678c"}, - {file = "nbformat-5.0.4.tar.gz", hash = "sha256:562de41fc7f4f481b79ab5d683279bf3a168858268d4387b489b7b02be0b324a"}, + {file = "nbformat-5.0.5-py3-none-any.whl", hash = "sha256:65a79936a128fd85aef392b7fea520166364037118b6fe3ed52de742d06c4558"}, + {file = "nbformat-5.0.5.tar.gz", hash = "sha256:f0c47cf93c505cb943e2f131ef32b8ae869292b5f9f279db2bafb35867923f69"}, ] nbstripout = [ {file = "nbstripout-0.3.7-py2.py3-none-any.whl", hash = "sha256:cf745ae8c49fccdb3068b73fc3b783898d5d62ee929429e9af37a6dfefba34b7"}, {file = "nbstripout-0.3.7.tar.gz", hash = "sha256:62f1b1fe9c7c298061089fd9bd5d297eb6209f7fbef0758631dbe58d38fc828f"}, ] +nest-asyncio = [ + {file = "nest_asyncio-1.3.2-py3-none-any.whl", hash = "sha256:b4cdd08655e2848098d204a26590cbfa39fcbc4ad1811c568678ffc8a0c8e279"}, + {file = "nest_asyncio-1.3.2.tar.gz", hash = "sha256:14e194b72144052a82173ca9109bd07c57813a320f42c7acfad1e4d329988350"}, +] notebook = [ {file = "notebook-6.0.3-py3-none-any.whl", hash = "sha256:3edc616c684214292994a3af05eaea4cc043f6b4247d830f3a2f209fa7639a80"}, {file = "notebook-6.0.3.tar.gz", hash = "sha256:47a9092975c9e7965ada00b9a20f0cf637d001db60d241d479f53c0be117ad48"}, @@ -1639,8 +1662,8 @@ pandocfilters = [ {file = "pandocfilters-1.4.2.tar.gz", hash = "sha256:b3dd70e169bb5449e6bc6ff96aea89c5eea8c5f6ab5e207fc2f521a2cf4a0da9"}, ] papermill = [ - {file = "papermill-2.0.0-py3-none-any.whl", hash = "sha256:7ad0dd5bce86df5e973a88e29457be99662099d365b99d640acb27119b1b7812"}, - {file = "papermill-2.0.0.tar.gz", hash = "sha256:83459eeb378b2a2f885fc2b36890306a99fda160d483d28847aa112454311bab"}, + {file = "papermill-2.1.0-py3-none-any.whl", hash = "sha256:10f86551cd28d09acea1f63b04a61d061ed4c3d3e82aa675b6a593c4451860c6"}, + {file = "papermill-2.1.0.tar.gz", hash = "sha256:1ff390a2bea5ea1538c1fcb5e2abb08c2a261d4427286905bc803416c72c26b0"}, ] parse = [ {file = "parse-1.15.0.tar.gz", hash = "sha256:a6d4e2c2f1fbde6717d28084a191a052950f758c0cbd83805357e6575c2b95c0"}, @@ -1709,8 +1732,8 @@ pytest-expecter = [ {file = "pytest_expecter-2.1-py3-none-any.whl", hash = "sha256:ab66120671a22be41f7df4c29d76c02d62e3420b8277b1455d2e683a042a0608"}, ] pytest-mock = [ - {file = "pytest-mock-2.0.0.tar.gz", hash = "sha256:b35eb281e93aafed138db25c8772b95d3756108b601947f89af503f8c629413f"}, - {file = "pytest_mock-2.0.0-py2.py3-none-any.whl", hash = "sha256:cb67402d87d5f53c579263d37971a164743dc33c159dfb4fb4a86f37c5552307"}, + {file = "pytest-mock-3.0.0.tar.gz", hash = "sha256:a4494016753a30231f8519bfd160242a0f3c8fb82ca36e7b6f82a7fb602ac6b8"}, + {file = "pytest_mock-3.0.0-py2.py3-none-any.whl", hash = "sha256:98e02534f170e4f37d7e1abdfc5973fd4207aa609582291717f643764e71c925"}, ] pytest-profiling = [ {file = "pytest-profiling-1.7.0.tar.gz", hash = "sha256:93938f147662225d2b8bd5af89587b979652426a8a6ffd7e73ec4a23e24b7f29"}, diff --git a/pyproject.toml b/pyproject.toml index bb9b96f5..85862d3d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] name = "datafiles" -version = "0.8.1" +version = "0.9b1" description = "File-based ORM for dataclasses." license = "MIT" diff --git a/tests/test_saving.py b/tests/test_saving.py index bb4585d4..02aded90 100644 --- a/tests/test_saving.py +++ b/tests/test_saving.py @@ -2,8 +2,11 @@ # pylint: disable=unused-variable,assigning-non-slot +from typing import Optional + import pytest +from datafiles import datafile from datafiles.utils import dedent, logbreak, read, write from .samples import ( @@ -224,6 +227,26 @@ def with_nones(expect): """ ) + def when_nested_dataclass_is_none(expect): + @datafile + class Name: + value: str + + @datafile("../tmp/samples/{self.key}.yml") + class Sample: + + key: int + name: Optional[Name] + value: float = 0.0 + + sample = Sample(42, None) + + expect(read('tmp/samples/42.yml')) == dedent( + """ + name: + """ + ) + def describe_defaults(): def with_custom_values(expect):