Merge 0420767 into 9dc6ec9

CHANGES.rst

@@ -23,6 +23,8 @@
 - ``update_from_external_object`` caches certain information about the
   types of the updater objects, making it 8-25% faster.
 
+- ``update_from_external_object`` mutates sequences contained in a
+  dict in-place instead of overwriting with a new list.
 
 1.0.0a2 (2018-07-05)
 ====================

docs/conf.py

@@ -22,6 +22,10 @@
 import os
 import sys
 import pkg_resources
+# Use the python versions instead of the cython compiled versions
+# for better documentation extraction and ease of tweaking docs.
+os.environ['PURE_PYTHON'] = '1'
+
 sys.path.append(os.path.abspath('../src'))
 rqmt = pkg_resources.require('nti.externalization')[0]
 
@@ -79,8 +83,18 @@
 # This patterns also effect to html_static_path and html_extra_path
 exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
 
+# The reST default role (used for this markup: `text`) to use for all documents.
+default_role = 'obj'
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+add_module_names = False
+
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = 'perldoc'
 
 # If true, `todo` and `todoList` produce output, else they produce nothing.
 todo_include_todos = False

src/nti/externalization/_compat.py

@@ -21,7 +21,7 @@
 OSX = sys.platform == 'darwin'
 
 
-PURE_PYTHON = PYPY or os.getenv('PURE_PYTHON')
+PURE_PYTHON = PYPY or os.getenv('PURE_PYTHON') or os.getenv("NTI_EXT_PURE_PYTHON")
 
 
 def to_unicode(s, encoding='utf-8', err='strict'):

src/nti/externalization/_datastructures.pxd

@@ -13,10 +13,14 @@ from nti.externalization.__base_interfaces cimport get_standard_internal_fields
 from nti.externalization.__base_interfaces cimport StandardInternalFields as SIF
 
 from nti.externalization.internalization._fields cimport validate_named_field_value
+from nti.externalization.internalization._factories cimport find_factory_for
 
 from nti.externalization.__interface_cache cimport cache_for
 
-cdef IInternalObjectIO
+
+
+cdef IInternalObjectIOFinder
+cdef IAnonymousObjectFactory
 cdef SEF StandardExternalFields
 cdef SIF StandardInternalFields
 cdef validate_named_field_value
@@ -46,6 +50,9 @@ cdef class AbstractDynamicObjectIO(ExternalizableDictionaryMixin):
     cpdef _ext_accept_update_key(self, k, ext_self, ext_keys)
     cpdef _ext_accept_external_id(self, ext_self, parsed)
 
+    cpdef find_factory_for_named_value(self, key, value, registry)
+    cdef _updateFromExternalObject(self, parsed)
+
 
 cdef class ExternalizableInstanceDict(AbstractDynamicObjectIO):
     pass

src/nti/externalization/datastructures.py

@@ -25,7 +25,8 @@
 
 from nti.schema.interfaces import find_most_derived_interface
 
-from .interfaces import IInternalObjectIO
+from .interfaces import IInternalObjectIOFinder
+from .interfaces import IAnonymousObjectFactory
 from .interfaces import StandardInternalFields
 
 # Things imported from cython with matching cimport
@@ -36,6 +37,7 @@
 from .externalization.externalizer import to_external_object as _toExternalObject
 
 from .internalization import validate_named_field_value
+from .internalization.factories import find_factory_for
 from .representation import make_repr
 
 from ._base_interfaces import get_standard_external_fields
@@ -47,6 +49,14 @@
 StandardExternalFields = get_standard_external_fields()
 StandardInternalFields = get_standard_internal_fields()
 
+__all__ = [
+    'ExternalizableDictionaryMixin',
+    'AbstractDynamicObjectIO',
+    'ExternalizableInstanceDict',
+    'InterfaceObjectIO',
+    'ModuleScopedInterfaceObjectIO',
+]
+
 class ExternalizableDictionaryMixin(object):
     """
     Implements a toExternalDictionary method as a base for subclasses.
@@ -56,9 +66,6 @@ class ExternalizableDictionaryMixin(object):
     #: produce the *minimal* dictionary. See :func:`~to_minimal_standard_external_dictionary`
     __external_use_minimal_base__ = False
 
-    def __init__(self, *args):
-        super(ExternalizableDictionaryMixin, self).__init__(*args)
-
     def _ext_replacement(self):
         return self
 
@@ -87,6 +94,8 @@ class AbstractDynamicObjectIO(ExternalizableDictionaryMixin):
     Abstractions are in place to allow subclasses to map external and internal names
     independently (this type never uses getattr/setattr/hasattr, except for some
     standard fields).
+
+    See `InterfaceObjectIO` for a complete implementation.
     """
 
     # TODO: there should be some better way to customize this if desired (an explicit list)
@@ -123,6 +132,15 @@ class AbstractDynamicObjectIO(ExternalizableDictionaryMixin):
     _ext_primitive_out_ivars_ = frozenset()
     _prefer_oid_ = False
 
+    def find_factory_for_named_value(self, key, value, registry):
+        """
+        Uses `.find_factory_for` to locate a factory.
+
+        This does not take into account the current object (context)
+        or the *key*. It only handles finding factories based on the
+        class or MIME type found within *value*.
+        """
+        return find_factory_for(value, registry)
 
     def _ext_all_possible_keys(self):
         """
@@ -239,6 +257,9 @@ def _ext_accept_external_id(self, ext_self, parsed):
         return False  # false by default
 
     def updateFromExternalObject(self, parsed, *unused_args, **unused_kwargs):
+        return self._updateFromExternalObject(parsed)
+
+    def _updateFromExternalObject(self, parsed):
         updated = False
 
         ext_self = self._ext_replacement()
@@ -270,7 +291,7 @@ def updateFromExternalObject(self, parsed, *unused_args, **unused_kwargs):
 
         return updated
 
-interface.classImplements(AbstractDynamicObjectIO, IInternalObjectIO)
+interface.classImplements(AbstractDynamicObjectIO, IInternalObjectIOFinder)
 
 
 class ExternalizableInstanceDict(AbstractDynamicObjectIO):
@@ -314,12 +335,18 @@ class InterfaceObjectIO(AbstractDynamicObjectIO):
     """
     Externalizes to a dictionary based on getting the attributes of an
     object defined by an interface. If any attribute has a true value
-    for the tagged value ``_ext_excluded_out``, it will not be considered
-    for reading or writing.
-
-    Meant to be used as an adapter, so accepts the object to
-    externalize in the constructor, as well as the interface to use to
-    guide the process. The object is externalized using the
+    for the tagged value ``_ext_excluded_out``, it will not be
+    considered for reading or writing.
+
+    This is an implementation of
+    `~nti.externalization.interfaces.IInternalObjectIOFinder`, meaning
+    it can both internalize (update existing objects) and externalize
+    (producing dictionaries), and that it gets to choose the factories
+    used for sub-objects when internalizing.
+
+    This class is meant to be used as an adapter, so it accepts the
+    object to externalize in the constructor, as well as the interface
+    to use to guide the process. The object is externalized using the
     most-derived version of the interface given to the constructor
     that it implements.
 
@@ -328,8 +355,10 @@ class InterfaceObjectIO(AbstractDynamicObjectIO):
     the ``Class`` key, or a callable
     ``__external_class_name__(interface, object ) -> name.``
 
-    (TODO: In the future extend this to multiple, non-overlapping interfaces, and better
-    interface detection (see :class:`ModuleScopedInterfaceObjectIO` for a limited version of this.)
+    (TODO: In the future extend this to multiple, non-overlapping
+    interfaces, and better interface detection (see
+    :class:`ModuleScopedInterfaceObjectIO` for a limited version of
+    this.)
     """
 
     _ext_iface_upper_bound = None
@@ -346,7 +375,7 @@ def __init__(self, ext_self, iface_upper_bound=None, validate_after_update=True)
             schema will be validated after an object has been updated with
             :meth:`update_from_external_object`, not just the keys that were assigned.
         """
-        super(InterfaceObjectIO, self).__init__()
+        AbstractDynamicObjectIO.__init__(self)
         self._ext_self = ext_self
         # Cache all of this data that we use. It's required often and, if not quite a bottleneck,
         # does show up in the profiling data
@@ -360,7 +389,7 @@ def __init__(self, ext_self, iface_upper_bound=None, validate_after_update=True)
 
         if not cache.ext_primitive_out_ivars:
             keys = self._ext_find_primitive_keys()
-            cache.ext_primitive_out_ivars = self._ext_primitive_out_ivars_.union(keys)
+            cache.ext_primitive_out_ivars = self._ext_primitive_out_ivars_ | keys
         self._ext_primitive_out_ivars_ = cache.ext_primitive_out_ivars
 
         self.validate_after_update = validate_after_update
@@ -432,8 +461,59 @@ def _ext_accept_external_id(self, ext_self, parsed):
                 cache.ext_accept_external_id = False
         return cache.ext_accept_external_id
 
+    def find_factory_for_named_value(self, key, value, registry):
+        """
+        If `AbstractDynamicObjectIO.find_factory_for_named_value`
+        cannot find a factory based on examining *value*, then we use
+        the context objects's schema to find a factory.
+
+        If the schema contains an attribute named *key*, it will be
+        queried for the tagged value ``__external_factory__``. If
+        present, this tagged value should be the name of a factory
+        object implementing `.IAnonymousObjectFactory` registered in
+        *registry* (typically registered in the global site).
+
+        The ZCML directive `.IAnonymousObjectFactoryDirective` sets up both the
+        registration and the tagged value.
+
+        This is useful for internalizing data from external sources
+        that does not provide a class or MIME field within the data.
+
+        The most obvious limitation of this is that if the *value* is part
+        of a sequence, it must be a homogeneous sequence. The factory is
+        called with no arguments, so the only way to deal with heterogeneous
+        sequences is to subclass this object and override this method to
+        examine the value itself.
+
+        A second limitation is that the external data key must match
+        the internal schema field name. Again, the only way to
+        remove this limitation is to subclass this object.
+        """
+        factory = AbstractDynamicObjectIO.find_factory_for_named_value(self, key, value, registry)
+        if factory is None:
+            # Is there a factory on the field?
+            try:
+                field = self._iface[key]
+                # See zcml.py:anonymousObjectFactoryDirective.
+                # This *should* be a string giving the dottedname of a factory utility.
+                # For test purposes we also allow it to be an actual object.
+
+                # TODO: If this becomes a bottleneck, the ZCML could
+                # have an argument global=False to allow setting the type
+                # directly instead of a string; the user would have to
+                # *know* that no sites would ever need a different value.
+            except KeyError:
+                pass
+            else:
+                factory = field.queryTaggedValue('__external_factory__')
+                # When it is a string, we require the factory to exist.
+                # Anything else is a programming error.
+                if isinstance(factory, str):
+                    factory = registry.getUtility(IAnonymousObjectFactory, factory)
+        return factory
+
     def updateFromExternalObject(self, parsed, *unused_args, **unused_kwargs):
-        result = AbstractDynamicObjectIO.updateFromExternalObject(self, parsed)
+        result = AbstractDynamicObjectIO._updateFromExternalObject(self, parsed)
         # If we make it this far, then validate the object.
 
         # TODO: Should probably just make sure that there are no /new/

src/nti/externalization/externalization/externalizer.py

@@ -17,10 +17,11 @@
 import warnings
 try:
     from collections.abc import Set
-    from collections.abc import Mapping
 except ImportError:
     from collections import Set
     from collections import Mapping
+else: # pragma: no cover
+    from collections.abc import Mapping
 from collections import defaultdict
 from weakref import WeakKeyDictionary
 

src/nti/externalization/factory.py

@@ -14,11 +14,19 @@
 
 from nti.externalization.interfaces import IClassObjectFactory
 from nti.externalization.interfaces import IMimeObjectFactory
+from nti.externalization.interfaces import IAnonymousObjectFactory
 
 _builtin_callable = callable
 
 # pylint: disable=redefined-builtin, protected-access
 
+__all__ = [
+    'ObjectFactory',
+    'MimeObjectFactory',
+    'ClassObjectFactory',
+    'AnonymousObjectFactory',
+]
+
 class ObjectFactory(Factory):
     """
     A convenient :class:`zope.component.interfaces.IFactory` meant to be
@@ -91,3 +99,13 @@ class ClassObjectFactory(ObjectFactory):
     Default implementation of
     :class:`nti.externalization.interfaces.IClassObjectFactory`.
     """
+
+
+@interface.implementer(IAnonymousObjectFactory)
+class AnonymousObjectFactory(ObjectFactory):
+    """
+    Default implementation of
+    :class:`nti.externalization.interfaces.IAnonymousObjectFactory`.
+
+    .. versionadded:: 1.0a3
+    """