extension: adds new system fields feature

* System fields provides managed access to the record's dictionary.
  By default this behavior is not enabled on the records.

* Adds extension capabilities of the record API, so that it's possible
  to make e.g. a system fields extension that can hook into the record
  API. In many ways, this is similar to the signals, but provides a
  more strict and extensible API.

invenio_records/api.py

@@ -33,13 +33,21 @@ class RecordBase(dict):
     format_checker = None
     """Class-level attribute to specify a default JSONSchema format checker."""
 
+    _extensions = []
+    """Record extensions registry.
+
+    Allows extensions (like system fields) to be registered on the record.
+    """
+
     def __init__(self, data, model=None):
         """Initialize instance with dictionary data and SQLAlchemy model.
 
         :param data: Dict with record metadata.
         :param model: :class:`~invenio_records.models.RecordMetadata` instance.
         """
         self.model = model
+        for e in self._extensions:
+            e.init(self, data, model=model)
         super(RecordBase, self).__init__(data or {})
 
     @property

invenio_records/systemfields/__init__.py

@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2020 CERN.
+#
+# Invenio is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+
+
+"""System fields provides managed access to the record's dictionary.
+
+### A simple example
+
+Take the following record subclass:
+
+.. code-block:: python
+
+    class MyRecord(Record, SystemFieldsMixin):
+        test = ConstantField('mykey', 'myval')
+
+The class defines a system field named ``test`` of the type ``ConstantField``.
+The constant field adds a key (``mykey``) to the record with the value
+``myval`` when a record is created:
+
+.. code-block:: python
+
+    record = MyRecord({})
+
+The key ``mykey`` is part of the record's dictionary (i.e. you can do
+``record['mykey']`` to acecss the value)::
+
+    record['mykey'] == 'myval'
+
+The key can however also be accessed through the field (i.e. ``record.test``)::
+
+    record.test == 'myval'
+
+System fields is thus a way to manage a subpart of record an allows you the
+field to hook into the record API. This is a powerful API that can be used
+to create fields which provides integration with related objects.
+
+### A more advanced example
+
+Imagine the following record subclass using an imaginary ``PIDField``:
+
+.. code-block:: python
+
+    class MyRecord(Record, SystemFieldsMixin):
+        pid = PIDField(pid_type='recid', object_type='rec')
+
+You could use this field to set a PID on the record::
+
+    record.pid = PersistentIdentifier(...)
+
+Or, you could access the PID on a record you get from the database:
+
+.. code-block:: python
+
+    record = MyRecord.get_record()
+    record.pid  # would return a PersistentIdentifier object.
+
+The simple example only worked with the record itself. The more advanced
+example here, the record is integrated with related objects.
+
+## Data access layer
+
+System fields can do a lot, however you should seen them as part of the data
+access layer. This means that they primarily simplifies data access between
+records and related objects.
+"""
+
+from .base import SystemField, SystemFieldsMeta, SystemFieldsMixin
+from .constant import ConstantField
+
+__all__ = (
+    'ConstantField,'
+    'SystemField,'
+    'SystemFieldsMeta,'
+    'SystemFieldsMixin,'
+)

invenio_records/systemfields/base.py

@@ -0,0 +1,194 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2020 CERN.
+#
+# Invenio is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+
+"""The core of the system fields implementation."""
+
+import inspect
+
+
+def _get_fields(attrs, field_class):
+    """Get system fields from a class' attributes dict.
+
+    :param attrs: Dict of class' attributes.
+    :param field_class: Base class for all system fields.
+    """
+    fields = {}
+    for name, val in attrs.items():
+        if isinstance(val, field_class):
+            fields[name] = val
+    return fields
+
+
+def _get_inherited_fields(class_, field_class):
+    """Get system fields from all base classes respecting the MRO.
+
+    :param class_: The class that has just been constructed.
+    :param field_class: Base class for all system fields.
+    """
+    # The variable mro holds a list of all super classes of "class_", in
+    # correct MRO (mehtod resolution order). This ensures that we respect
+    # Python inheritance rules when the same field is defined in both
+    # the class and one or more super classes.
+    mro = inspect.getmro(class_)
+
+    fields = {}
+
+    # Reverse order through MRO so that a field is taken from the most
+    # specific class.
+    for base in reversed(mro):
+        fields.update(_get_fields(base.__dict__, field_class))
+    return fields
+
+
+class SystemField:
+    """Base class for all system fields.
+
+    A system field is a Python data descriptor set on a record class that can
+    also hook into a record via the extensions API (e.g on record creation,
+    dumping etc).
+
+    A subclass should as minimum overwrite the __get__() method.
+    """
+
+    #
+    # Extension API
+    #
+    def init(self, data, model=None):
+        """Initialise the data.
+
+        Called when a new record is instanciated (i.e. during all
+        ``Record({...})```). This means it's also called when e.g. a record
+        is created via ``Record.create()``.
+
+        .. note::
+            Subclasses should not return anything when overwriting this method.
+            Instead modifications directly on the passed ``data`` and ``model``
+            arguments.
+
+        :param data: The dict passed to the record's constructor.
+        :param model: The model class used for initialization.
+        """
+        pass
+
+    def dump(self, data):
+        """Initialise the data.
+
+        Called when a new record is instanciated with the data.
+        """
+        return data
+
+    #
+    # Data descriptor definition
+    #
+    def __get__(self, instance, class_):
+        """Accessing the object attribute.
+
+        A subclass that overwrites this method, should handle two cases:
+
+        1. Class access - If ``instance`` is None, the field is accessed
+           through the class (e.g. Record.myfield). In this case the field
+           itself should be returned.
+        2. Instance access -  If ``instance`` is not None, the field is
+           accessed through an instance of the class (e.g. record``.myfield``).
+
+        A simple example is provided below:
+
+        .. code-block:: python
+
+            def __get__(self, instance, class_):
+                if instance is None:
+                    return self  # returns the field itself.
+                if 'mykey' in instance:
+                    return instance['mykey']
+                return None
+
+        :param instance: The instance through which the field is being accessed
+                         or ``None`` if the field is accessed through the
+                         class.
+        :param class_: The class_ which owns the field. In most cases you
+                       should use this variable.
+        """
+        # Class access
+        # - by default a system field accessed through a class (e.g.
+        #   Record.myattr will return the field itself).
+        if instance is None:
+            return self
+        # Instance access
+        # - by default a system field accessed through an object instance (e.g.
+        #   record.myattr will raise an Attribute error)
+        raise AttributeError
+
+    def __set__(self, instance, value):
+        """Setting the attribute (instance access only).
+
+        This method only handles set operations from an instance (e.g.
+        ``record.myfield = val``). This is opposite to ``__get__()`` which
+        needs to handle both class and instance access.
+        """
+        raise AttributeError()
+
+
+class SystemFieldsExt:
+    """Record extension for system fields.
+
+    This extension is responsible for iterating over all declared system fields
+    on a class for each extension point.
+    """
+
+    def __init__(self, declared_fields):
+        """Save the declared fields on the extension."""
+        self.declared_fields = declared_fields
+
+    def init(self, record, data, model=None):
+        """Called when a new record instance is initialized."""
+        for field in self.declared_fields.values():
+            field.init(data, model=model)
+
+
+class SystemFieldsMeta(type):
+    """Metaclass for a record class.
+
+    The metaclass
+    """
+
+    def __new__(mcs, name, bases, attrs):
+        """Create a new record class."""
+        # Initialise an "_extensions" attribute on each class, to ensure each
+        # class have a separate extensions registry.
+        if '_extensions' not in attrs:
+            attrs['_extensions'] = []
+
+        # Construct the class.
+        class_ = super().__new__(mcs, name, bases, attrs)
+
+        # Get system fields and ensure inheritance is respected.
+        declared_fields = _get_inherited_fields(class_, SystemField)
+        declared_fields.update(_get_fields(attrs, SystemField))
+
+        # Register the system fields extension on the record class.
+        class_._extensions.append(SystemFieldsExt(declared_fields))
+
+        return class_
+
+
+class SystemFieldsMixin(metaclass=SystemFieldsMeta):
+    """Mixin class for records that add system fields capabilities.
+
+    This class is primarily syntax sugar for being able to do::
+
+      class MyRecord(Record, SystemsFieldsMixin):
+          pass
+
+    instead of::
+
+      class MyRecord(Record, metaclass=SystemFieldsMeta):
+          pass
+
+    There are subtle differences though between the two above methods. Mainly
+    which classes will execute the ``__new__()`` method on the metaclass.
+    """

invenio_records/systemfields/constant.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2020 CERN.
+#
+# Invenio is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+
+"""Constant system field."""
+
+from .base import SystemField
+
+
+class ConstantField(SystemField):
+    """Constant fields add a constant value to key in the record."""
+
+    def __init__(self, key, value):
+        """Initialize the field."""
+        self.key = key
+        self.value = value
+
+    def init(self, data, model=None):
+        """Initialise data."""
+        if self.key not in data:
+            data[self.key] = self.value
+
+    def __get__(self, instance, class_):
+        """Accessing the attribute."""
+        # Class access
+        if instance is None:
+            return self
+        # Instance access
+        if self.key in instance:
+            return instance[self.key]
+        return None