Merge pull request #162 from jacebrowning/release/v0.8

Release v0.8

.python-version

@@ -1 +1 @@
-3.7.2
+3.8.2

CHANGELOG.md

@@ -1,3 +1,8 @@
+# 0.8 (2020-03-28)
+
+- Updated the `@datafile(...)` decorator to be used as a drop-in replacement for `@dataclass(...)`.
+- Added support for loading unlimited levels of nested objects.
+
 # 0.7 (2020-02-20)
 
 - Added a `YAML_LIBRARY` setting to control the underlying YAML library.

Makefile

@@ -102,6 +102,7 @@ $(MKDOCS_INDEX): docs/requirements.txt mkdocs.yml docs/*.md
 
 docs/requirements.txt: poetry.lock
 	@ poetry run pip freeze -qqq | grep mkdocs > $@
+	@ poetry run pip freeze -qqq | grep Pygments >> $@
 
 .PHONY: mkdocs-serve
 mkdocs-serve: mkdocs

README.md

@@ -16,6 +16,8 @@ Popular use cases include:
 - Loading data fixtures for demonstration or testing purposes
 - Prototyping data models agnostic of persistence backends
 
+Watch [my lightning talk](https://www.youtube.com/watch?v=moYkuNrmc1I&feature=youtu.be&t=1225) for a demo of this in action!
+
 ## Overview
 
 Take an existing dataclass such as [this example](https://docs.python.org/3/library/dataclasses.html#module-dataclasses) from the documentation:
@@ -96,8 +98,6 @@ Objects can also be restored from the filesystem:
 100
 ```
 
-Demo: [Jupyter Notebook](https://github.com/jacebrowning/datafiles/blob/develop/notebooks/readme.ipynb)
-
 ## Installation
 
 Because datafiles relies on dataclasses and type annotations, Python 3.7+ is required. Install this library directly into an activated virtual environment:

datafiles/converters/__init__.py

@@ -3,6 +3,7 @@
 from typing import Any, Dict, Optional, Union
 
 import log
+from ruamel.yaml.scalarfloat import ScalarFloat
 
 from ..utils import cached
 from ._bases import Converter
@@ -23,6 +24,7 @@ def register(cls: Union[type, str], converter: type):
 
 register(Integer.TYPE, Integer)
 register(Float.TYPE, Float)
+register(ScalarFloat, Float)
 register(Boolean.TYPE, Boolean)
 register(String.TYPE, String)
 

datafiles/converters/containers.py

@@ -1,10 +1,10 @@
 from collections.abc import Iterable
 from contextlib import suppress
-from dataclasses import _MISSING_TYPE as Missing
 from typing import Callable, Dict
 
 import log
 
+from ..utils import Missing, get_default_field_value
 from ._bases import Converter
 
 
@@ -144,8 +144,18 @@ def to_python_value(cls, deserialized_data, *, target_object):
                 data.pop(name)
 
         for name, converter in cls.CONVERTERS.items():
-            if name not in data:
-                data[name] = converter.to_python_value(None, target_object=None)
+            log.debug(f"Converting '{name}' data with {converter}")
+            if name in data:
+                converted = converter.to_python_value(data[name], target_object=None)
+            else:
+                if target_object is None:
+                    converted = converter.to_python_value(None, target_object=None)
+                else:
+                    converted = get_default_field_value(target_object, name)
+                    if converted is Missing:
+                        converted = getattr(target_object, name)
+
+            data[name] = converted
 
         new_value = cls.DATACLASS(**data)  # pylint: disable=not-callable
 

datafiles/decorators.py

@@ -13,16 +13,23 @@
 
 
 def datafile(
-    pattern: Union[str, Callable],
+    pattern: Union[str, Callable, None] = None,
     attrs: Optional[Dict[str, Converter]] = None,
     manual: bool = Meta.datafile_manual,
     defaults: bool = Meta.datafile_defaults,
     auto_load: bool = Meta.datafile_auto_load,
     auto_save: bool = Meta.datafile_auto_save,
     auto_attr: bool = Meta.datafile_auto_attr,
+    **kwargs,
 ):
     """Synchronize a data class to the specified path."""
 
+    if pattern is None:
+        return dataclasses.dataclass(**kwargs)
+
+    if callable(pattern):
+        return dataclasses.dataclass(pattern)  # type: ignore
+
     def decorator(cls=None):
         if dataclasses.is_dataclass(cls):
             dataclass = cls
@@ -40,9 +47,6 @@ def decorator(cls=None):
             auto_attr=auto_attr,
         )
 
-    if callable(pattern):
-        return dataclasses.dataclass(pattern)  # type: ignore
-
     return decorator
 
 

datafiles/hooks.py

@@ -1,5 +1,5 @@
-import dataclasses
 from contextlib import contextmanager, suppress
+from dataclasses import is_dataclass
 from functools import wraps
 
 import log
@@ -52,22 +52,26 @@ def apply(instance, mapper):
             modified_method = save_after(cls, method)
             setattr(cls, method_name, modified_method)
 
-    if dataclasses.is_dataclass(instance):
+    if is_dataclass(instance):
         for attr_name in instance.datafile.attrs:
             attr = getattr(instance, attr_name)
-            if not dataclasses.is_dataclass(attr):
-                # pylint: disable=unidiomatic-typecheck
-                if type(attr) == list:
-                    attr = List(attr)
-                    setattr(instance, attr_name, attr)
-                elif type(attr) == dict:
-                    attr = Dict(attr)
-                    setattr(instance, attr_name, attr)
-                else:
-                    continue
+            if isinstance(attr, list):
+                attr = List(attr)
+                setattr(instance, attr_name, attr)
+            elif isinstance(attr, dict):
+                attr = Dict(attr)
+                setattr(instance, attr_name, attr)
+            elif not is_dataclass(attr):
+                continue
             attr.datafile = create_mapper(attr, root=mapper)
             apply(attr, mapper)
 
+    elif isinstance(instance, list):
+        for item in instance:
+            if is_dataclass(item):
+                item.datafile = create_mapper(item, root=mapper)
+                apply(item, mapper)
+
 
 def load_before(cls, method):
     """Decorate methods that should load before call."""

datafiles/manager.py

@@ -66,7 +66,12 @@ def all(self) -> Iterator[HasDatafile]:
         if path.is_absolute() or self.model.Meta.datafile_pattern.startswith('./'):
             pattern = str(path.resolve())
         else:
-            root = Path(inspect.getfile(self.model)).parent
+            try:
+                root = Path(inspect.getfile(self.model)).parent
+            except TypeError:
+                level = log.DEBUG if '__main__' in str(self.model) else log.WARNING
+                log.log(level, f'Unable to determine module for {self.model}')
+                root = Path.cwd()
             pattern = str(root / self.model.Meta.datafile_pattern)
 
         splatted = pattern.format(self=Splats())

datafiles/mapper.py

@@ -13,11 +13,14 @@
 
 from . import config, formats, hooks
 from .converters import Converter, List, map_type
-from .utils import display, recursive_update, write
-
-
-Trilean = Optional[bool]
-Missing = dataclasses._MISSING_TYPE
+from .utils import (
+    Missing,
+    Trilean,
+    display,
+    get_default_field_value,
+    recursive_update,
+    write,
+)
 
 
 class Mapper:
@@ -64,7 +67,7 @@ def path(self) -> Optional[Path]:
         cls = self._instance.__class__
         try:
             root = Path(inspect.getfile(cls)).parent
-        except TypeError:  # pragma: no cover
+        except TypeError:
             level = log.DEBUG if '__main__' in str(cls) else log.WARNING
             log.log(level, f'Unable to determine module for {cls}')
             root = Path.cwd()
@@ -147,11 +150,11 @@ def _get_data(self, include_default_values: Trilean = None) -> Dict:
                     value,
                     default_to_skip=Missing
                     if include_default_values
-                    else self._get_default_field_value(name),
+                    else get_default_field_value(self._instance, name),
                 )
 
             elif (
-                value == self._get_default_field_value(name)
+                value == get_default_field_value(self._instance, name)
                 and not include_default_values
             ):
                 log.debug(f"Skipped default value of {value!r} for {name!r} attribute")
@@ -178,9 +181,9 @@ def _get_text(self, **kwargs) -> str:
     def text(self, value: str):
         write(self.path, value.strip() + '\n')
 
-    def load(self, *, _log=True, _first=False) -> None:
+    def load(self, *, _log=True, _first_load=False) -> None:
         if self._root:
-            self._root.load(_log=_log, _first=_first)
+            self._root.load(_log=_log, _first_load=_first_load)
             return
 
         if self.path:
@@ -197,76 +200,48 @@ def load(self, *, _log=True, _first=False) -> None:
 
             for name, value in data.items():
                 if name not in self.attrs and self.auto_attr:
-                    cls: Any = type(value)
-                    if issubclass(cls, list):
-                        cls.__origin__ = list
-
-                        if value:
-                            item_cls = type(value[0])
-                            for item in value:
-                                if not isinstance(item, item_cls):
-                                    log.warn(f'{name!r} list type cannot be inferred')
-                                    item_cls = Converter
-                                    break
-                        else:
-                            log.warn(f'{name!r} list type cannot be inferred')
-                            item_cls = Converter
-
-                        log.debug(f'Inferring {name!r} type: {cls} of {item_cls}')
-                        self.attrs[name] = map_type(cls, name=name, item_cls=item_cls)
-                    elif issubclass(cls, dict):
-                        cls.__origin__ = dict
-
-                        log.debug(f'Inferring {name!r} type: {cls}')
-                        self.attrs[name] = map_type(cls, name=name, item_cls=Converter)
-                    else:
-                        log.debug(f'Inferring {name!r} type: {cls}')
-                        self.attrs[name] = map_type(cls, name=name)
+                    self.attrs[name] = self._infer_attr(name, value)
 
             for name, converter in self.attrs.items():
-                log.debug(f"Converting '{name}' data with {converter}")
-
-                if getattr(converter, 'DATACLASS', None):
-                    self._set_dataclass_value(data, name, converter)
-                else:
-                    self._set_attribute_value(data, name, converter, _first)
+                self._set_value(self._instance, name, converter, data, _first_load)
 
             hooks.apply(self._instance, self)
 
         self.modified = False
 
-    def _set_dataclass_value(self, data, name, converter):
-        # TODO: Support nesting unlimited levels
-        # https://github.com/jacebrowning/datafiles/issues/22
-        nested_data = data.get(name)
-        if nested_data is None:
-            return
+    @staticmethod
+    def _infer_attr(name, value):
+        cls: Any = type(value)
+        if issubclass(cls, list):
+            cls.__origin__ = list
+            if value:
+                item_cls = type(value[0])
+                for item in value:
+                    if not isinstance(item, item_cls):
+                        log.warn(f'{name!r} list type cannot be inferred')
+                        item_cls = Converter
+                        break
+            else:
+                log.warn(f'{name!r} list type cannot be inferred')
+                item_cls = Converter
+            log.debug(f'Inferring {name!r} type: {cls} of {item_cls}')
+            return map_type(cls, name=name, item_cls=item_cls)
 
-        log.debug(f'Converting nested data to Python: {nested_data}')
+        if issubclass(cls, dict):
+            cls.__origin__ = dict
+            log.debug(f'Inferring {name!r} type: {cls}')
+            return map_type(cls, name=name, item_cls=Converter)
 
-        dataclass = getattr(self._instance, name)
-        if dataclass is None:
-            for field in dataclasses.fields(converter.DATACLASS):
-                if field.name not in nested_data:
-                    nested_data[field.name] = None
-            dataclass = converter.to_python_value(nested_data, target_object=dataclass)
+        log.debug(f'Inferring {name!r} type: {cls}')
+        return map_type(cls, name=name)
 
-        mapper = create_mapper(dataclass)
-        for name2, converter2 in mapper.attrs.items():
-            _value = nested_data.get(name2, mapper._get_default_field_value(name2))
-            value = converter2.to_python_value(
-                _value, target_object=getattr(dataclass, name2)
-            )
-            log.debug(f"'{name2}' as Python: {value!r}")
-            setattr(dataclass, name2, value)
+    @staticmethod
+    def _set_value(instance, name, converter, data, first_load):
+        log.debug(f"Converting '{name}' data with {converter}")
 
-        log.debug(f"Setting '{name}' value: {dataclass!r}")
-        setattr(self._instance, name, dataclass)
-
-    def _set_attribute_value(self, data, name, converter, first_load):
         file_value = data.get(name, Missing)
-        init_value = getattr(self._instance, name, Missing)
-        default_value = self._get_default_field_value(name)
+        init_value = getattr(instance, name, Missing)
+        default_value = get_default_field_value(instance, name)
 
         if first_load:
             log.debug(
@@ -291,21 +266,7 @@ def _set_attribute_value(self, data, name, converter, first_load):
             value = converter.to_python_value(file_value, target_object=init_value)
 
         log.debug(f"Setting '{name}' value: {value!r}")
-        setattr(self._instance, name, value)
-
-    def _get_default_field_value(self, name):
-        for field in dataclasses.fields(self._instance):
-            if field.name == name:
-                if not isinstance(field.default, Missing):
-                    return field.default
-
-                if not isinstance(field.default_factory, Missing):  # type: ignore
-                    return field.default_factory()  # type: ignore
-
-                if not field.init and hasattr(self._instance, '__post_init__'):
-                    return getattr(self._instance, name)
-
-        return Missing
+        setattr(instance, name, value)
 
     def save(self, *, include_default_values: Trilean = None, _log=True) -> None:
         if self._root:

datafiles/model.py

@@ -29,7 +29,7 @@ def __post_init__(self):
                     log.debug(f'Datafile exists: {exists}')
 
                     if exists:
-                        self.datafile.load(_first=True)
+                        self.datafile.load(_first_load=True)
                     elif path and create:
                         self.datafile.save()