Improve Sync Data Structures

.github/dependabot.yml

@@ -0,0 +1,9 @@
+version: 2
+updates:
+- package-ecosystem: pip
+  directory: "/"
+  schedule:
+    interval: monthly
+  open-pull-requests-limit: 10
+  reviewers:
+    - glotzerlab/signac-committers

requirements-benchmark.txt

@@ -1,7 +1,7 @@
 click>=7.0
-numpy==1.18
-gitdb2==3.0.2
-GitPython==3.0.8
-pandas==0.25.3; implementation_name!='cpython' --no-binary pandas
-pandas==0.25.3; implementation_name=='cpython' --no-binary :none:
-psutil==5.6.7
+numpy==1.19.0
+gitdb2==4.0.2
+GitPython==3.1.3
+pandas==1.0.5; implementation_name!='cpython' --no-binary pandas
+pandas==1.0.5; implementation_name=='cpython' --no-binary :none:
+psutil==5.7.0

requirements-dev.txt

@@ -1,13 +1,13 @@
-coverage==5.0.3
-numpy==1.18
-pandas==0.25.3; implementation_name!='cpython' --no-binary pandas
-pandas==0.25.3; implementation_name=='cpython' --no-binary :none:
+coverage==5.2
+numpy==1.19.0
+pandas==1.0.5; implementation_name!='cpython' --no-binary pandas
+pandas==1.0.5; implementation_name=='cpython' --no-binary :none:
 h5py==2.10; implementation_name=='cpython'
 tables==3.6.1; implementation_name=='cpython'
 click>=7.0
 ruamel.yaml>=0.15.89
 pytest==5.3.4
-pytest-subtests==0.3.0
+pytest-subtests==0.3.1
 pydocstyle==5.0.2
-pytest-cov==2.8.1
+pytest-cov==2.10.0
 pymongo==3.10.1

setup.cfg

@@ -15,7 +15,7 @@ max-line-length = 100
 exclude = configobj,passlib,cite.py,conf.py
 
 [pydocstyle]
-match = jsondict.py
+match = jsondict.py | synced_collection.py | jsoncollection.py | syncedattrdict.py | synced_list.py 
 ignore = D105, D107, D203, D213
 
 [mypy]

signac/core/jsoncollection.py

@@ -0,0 +1,151 @@
+# Copyright (c) 2020 The Regents of the University of Michigan
+# All rights reserved.
+# This software is licensed under the BSD 3-Clause License.
+"""Implements JSON-backend.
+
+This implements the JSON-backend for SyncedCollection API by
+implementing sync and load methods.
+"""
+
+import os
+import json
+import errno
+import uuid
+
+from .synced_collection import SyncedCollection
+from .syncedattrdict import SyncedAttrDict
+from .synced_list import SyncedList
+
+
+class JSONCollection(SyncedCollection):
+    """Implement sync and load using a JSON back end."""
+
+    backend = __name__  # type: ignore
+
+    def __init__(self, filename=None, write_concern=False, **kwargs):
+        self._filename = os.path.realpath(filename) if filename is not None else None
+        self._write_concern = write_concern
+        super().__init__(**kwargs)
+        if (filename is None) == (self._parent is None):
+            raise ValueError(
+                "Illegal argument combination, one of the two arguments, "
+                "parent or filename must be None, but not both.")
+
+    def _load(self):
+        """Load the data from a JSON-file."""
+        try:
+            with open(self._filename, 'rb') as file:
+                blob = file.read()
+                return json.loads(blob)
+        except IOError as error:
+            if error.errno == errno.ENOENT:
+                return None
+
+    def _sync(self):
+        """Write the data to json file."""
+        data = self.to_base()
+        # Serialize data:
+        blob = json.dumps(data).encode()
+        # When write_concern flag is set, we write the data into dummy file and then
+        # replace that file with original file.
+        if self._write_concern:
+            dirname, filename = os.path.split(self._filename)
+            fn_tmp = os.path.join(dirname, '._{uid}_{fn}'.format(
+                uid=uuid.uuid4(), fn=filename))
+            with open(fn_tmp, 'wb') as tmpfile:
+                tmpfile.write(blob)
+            os.replace(fn_tmp, self._filename)
+        else:
+            with open(self._filename, 'wb') as file:
+                file.write(blob)
+
+
+class JSONDict(JSONCollection, SyncedAttrDict):
+    """A dict-like mapping interface to a persistent JSON file.
+
+    The JSONDict inherits from :class:`~core.collection_api.SyncedCollection`
+    and :class:`~core.syncedattrdict.SyncedAttrDict`.
+
+    .. code-block:: python
+
+        doc = JSONDict('data.json', write_concern=True)
+        doc['foo'] = "bar"
+        assert doc.foo == doc['foo'] == "bar"
+        assert 'foo' in doc
+        del doc['foo']
+
+    .. code-block:: python
+
+        >>> doc['foo'] = dict(bar=True)
+        >>> doc
+        {'foo': {'bar': True}}
+        >>> doc.foo.bar = False
+        {'foo': {'bar': False}}
+
+    .. warning::
+
+        While the JSONDict object behaves like a dictionary, there are
+        important distinctions to remember. In particular, because operations
+        are reflected as changes to an underlying file, copying (even deep
+        copying) a JSONDict instance may exhibit unexpected behavior. If a
+        true copy is required, you should use the `to_base()` method to get a
+        dictionary representation, and if necessary construct a new JSONDict
+        instance: `new_dict = JSONDict(old_dict.to_base())`.
+
+    Parameters
+    ----------
+    filename: str, optional
+        The filename of the associated JSON file on disk (Default value = None).
+    data: mapping, optional
+        The intial data pass to JSONDict. Defaults to `list()`
+    parent: object, optional
+        A parent instance of JSONDict or None (Default value = None).
+    write_concern: bool, optional
+        Ensure file consistency by writing changes back to a temporary file
+        first, before replacing the original file (Default value = None).
+    """
+
+    pass
+
+
+class JSONList(JSONCollection, SyncedList):
+    """A non-string sequence interface to a persistent JSON file.
+
+    The JSONDict inherits from :class:`~core.collection_api.SyncedCollection`
+    and :class:`~core.syncedlist.SyncedList`.
+
+    .. code-block:: python
+
+        synced_list = JSONList('data.json', write_concern=True)
+        synced_list.append("bar")
+        assert synced_list[0] == "bar"
+        assert len(synced_list) == 1
+        del synced_list[0]
+
+    .. warning::
+
+        While the JSONList object behaves like a list, there are
+        important distinctions to remember. In particular, because operations
+        are reflected as changes to an underlying file, copying (even deep
+        copying) a JSONList instance may exhibit unexpected behavior. If a
+        true copy is required, you should use the `to_base()` method to get a
+        dictionary representation, and if necessary construct a new JSONList
+        instance: `new_list = JSONList(old_list.to_base())`.
+
+    Parameters
+    ----------
+    filename: str
+        The filename of the associated JSON file on disk (Default value = None).
+    data: non-str Sequence
+        The intial data pass to JSONDict
+    parent: object
+        A parent instance of JSONDict or None (Default value = None).
+    write_concern: bool
+        Ensure file consistency by writing changes back to a temporary file
+        first, before replacing the original file (Default value = None).
+    """
+
+    pass
+
+
+SyncedCollection.register(JSONDict, JSONList)

signac/core/synced_collection.py

@@ -0,0 +1,164 @@
+# Copyright (c) 2020 The Regents of the University of Michigan
+# All rights reserved.
+# This software is licensed under the BSD 3-Clause License.
+"""Implement the SyncedCollection class.
+
+SyncedCollection encapsulates the synchronization of different data-structures.
+These features are implemented in different subclasses which enable us to use a
+backend with different data-structures or vice-versa. It declares as abstract
+methods the methods that must be implemented by any subclass to match the API.
+"""
+
+from contextlib import contextmanager
+from abc import abstractmethod
+from collections import defaultdict
+from collections.abc import Collection
+
+try:
+    import numpy
+    NUMPY = True
+except ImportError:
+    NUMPY = False
+
+
+class SyncedCollection(Collection):
+    """The base synced collection represents a collection that is synced with a backend.
+
+    The class is intended for use as an ABC. The SyncedCollection is a
+    :class:`~collections.abc.Collection` where all data is stored persistently
+    in the underlying backend. The backend name wil be same as the module name.
+    """
+
+    backend = None
+
+    def __init__(self, parent=None):
+        self._data = None
+        self._parent = parent
+        self._suspend_sync_ = 0
+
+    @classmethod
+    def register(cls, *args):
+        """Register the synced data structures.
+
+        Registry is used when recursively converting synced data structures to determine
+        what to convert their children into.
+
+        Parameters
+        ----------
+        *args
+            Classes to register
+        """
+        if not hasattr(cls, 'registry'):
+            cls.registry = defaultdict(list)
+        for _cls in args:
+            cls.registry[_cls.backend].append(_cls)
+
+    @classmethod
+    def from_base(cls, data, backend=None, **kwargs):
+        """Dynamically resolve the type of object to the corresponding synced collection.
+
+        Parameters
+        ----------
+        data : any
+            Data to be converted from base class.
+        backend: str
+            Name of backend for synchronization. Default to backend of class.
+        **kwargs:
+            Kwargs passed to instance of synced collection.
+
+        Returns
+        -------
+        data : object
+            Synced object of corresponding base type.
+        """
+        backend = cls.backend if backend is None else backend
+        if backend is None:
+            raise ValueError("No backend found.")
+        for _cls in cls.registry[backend]:
+            if _cls.is_base_type(data):
+                return _cls(data=data, **kwargs)
+        if NUMPY:
+            if isinstance(data, numpy.number):
+                return data.item()
+        return data
+
+    @abstractmethod
+    def to_base(self):
+        """Dynamically resolve the synced collection to the corresponding base type."""
+        pass
+
+    @contextmanager
+    def _suspend_sync(self):
+        """Prepare context where load and sync are suspended."""
+        self._suspend_sync_ += 1
+        yield
+        self._suspend_sync_ -= 1
+
+    @classmethod
+    @abstractmethod
+    def is_base_type(cls, data):
+        """Check whether data is of the same base type (such as list or dict) as this class."""
+        pass
+
+    @abstractmethod
+    def _load(self):
+        """Load data from underlying backend."""
+        pass
+
+    @abstractmethod
+    def _sync(self):
+        """Write data to underlying backend."""
+        pass
+
+    def sync(self):
+        """Synchronize the data with the underlying backend."""
+        if self._suspend_sync_ <= 0:
+            if self._parent is None:
+                self._sync()
+            else:
+                self._parent.sync()
+
+    def load(self):
+        """Load the data from the underlying backend."""
+        if self._suspend_sync_ <= 0:
+            if self._parent is None:
+                data = self._load()
+                with self._suspend_sync():
+                    self._update(data)
+            else:
+                self._parent.load()
+
+    # methods having same implementaion for all data-structures
+    def __getitem__(self, key):
+        self.load()
+        return self._data[key]
+
+    def __delitem__(self, item):
+        del self._data[item]
+        self.sync()
+
+    def __iter__(self):
+        self.load()
+        return iter(self._data)
+
+    def __len__(self):
+        self.load()
+        return len(self._data)
+
+    def __call__(self):
+        self.load()
+        return self.to_base()
+
+    def __eq__(self, other):
+        if isinstance(other, type(self)):
+            return self() == other()
+        else:
+            return self() == other
+
+    def __repr__(self):
+        self.load()
+        return repr(self._data)
+
+    def __str__(self):
+        self.load()
+        return str(self._data)