Better externalization of numbers.

.coveragerc

@@ -9,4 +9,5 @@ exclude_lines =
     pragma: no cover
     raise NotImplementedError
     raise AssertionError
+    Python 2
     if __name__ == .__main__.:

CHANGES.rst

@@ -3,10 +3,15 @@
 =========
 
 
-1.0.1 (unreleased)
+1.1.0 (unreleased)
 ==================
 
-- Nothing changed yet.
+- Make instances of ``fractions.Fraction`` externalize as a string
+  such as ``"1/3"``. When received by a schema field that can parse
+  this format, such as ``zope.schema.Rational`` (or higher on the
+  numeric tower), this means fractions can be round-tripped.
+- Support externalizing ``decimal.Decimal`` objects in the YAML
+  representation.
 
 
 1.0.0 (2020-03-19)

src/nti/externalization/_base_interfaces.py

@@ -13,9 +13,10 @@
 from __future__ import division
 from __future__ import print_function
 
-import numbers
+import decimal
 import six
 
+
 __all__ = [
     'NotGiven',
     'LocatedExternalDict',
@@ -52,7 +53,7 @@ class LocatedExternalDict(dict):
 
     # interfaces are applied in interfaces.py
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args, **kwargs): # pylint:disable=super-init-not-called
         dict_init(self, *args, **kwargs)
         self.__name__ = u''
         self.__parent__ = None
@@ -220,14 +221,16 @@ def __init__(self):
         #: to a text string to fill in `.StandardExternalFields.CREATOR`.
         self.CREATOR = 'creator'
         #: 'createdTime': The Unix timestamp of the creation of this object.
-        #: If no value can be found, we will attempt to adapt to `zope.dublincore.interfaces.IDCTimes`
+        #: If no value can be found, we will attempt to adapt to
+        #: `zope.dublincore.interfaces.IDCTimes`
         #: and use its 'created' attribute. Fills `StandardExternalFields.CREATED_TIME`
         self.CREATED_TIME = 'createdTime'
         #: 'containerId': The ID of the container of this object.
         #: Fills `StandardExternalFields.CONTAINER_ID`.
         self.CONTAINER_ID = 'containerId'
         #: 'lastModified': The Unix timestamp of the last modification of this object.
-        #: If no value can be found, we will attempt to adapt to `zope.dublincore.interfaces.IDCTimes`
+        #: If no value can be found, we will attempt to adapt to
+        #: zope.dublincore.interfaces.IDCTimes`
         #: and use its 'modified' attribute. Fills `.StandardExternalFields.LAST_MODIFIED`
         self.LAST_MODIFIED = 'lastModified'
 
@@ -239,8 +242,32 @@ def __init__(self):
 def get_standard_internal_fields():
     return _standard_internal_fields
 
+# Note that we DO NOT include ``numbers.Number``
+# as a primitive type. That's because ``numbers.Number``
+# is an ABC and arbitrary types can register as it; but
+# arbitrary types are not necessarily understood as proper
+# external objects by all representers. In particular,
+# ``fractions.Fraction`` cannot be handled by default and
+# needs to go through the adaptation process, as does ``complex``.
+# simplejson can handle ``decimal.Decimal``, but YAML cannot.
+_PRIMITIVE_NUMBER_TYPES = (
+    int, # bool is a subclass of int.
+    float,
+    decimal.Decimal,
+)
+try:
+    long
+except NameError:
+    pass
+else: # Python 2
+    _PRIMITIVE_NUMBER_TYPES += (
+        long,
+    )
+
 
-PRIMITIVES = six.string_types + (numbers.Number, bool, type(None))
+PRIMITIVES = six.string_types + (
+    type(None),
+) + _PRIMITIVE_NUMBER_TYPES
 
 
 

src/nti/externalization/configure.zcml

@@ -40,6 +40,9 @@
     <utility factory=".representation.JsonRepresenter" />
     <utility factory=".representation.YamlRepresenter" />
 
+    <!-- -->
+    <!-- Dates and datetimes -->
+    <!-- -->
     <!-- Offer to adapt strings to dates -->
     <!-- TODO: This is a bit ad-hoc. Surely there's a more formal set
     of transforms somewhere? -->
@@ -49,11 +52,20 @@
     <adapter factory=".datetime.datetime_from_timestamp" for="int" />
     <adapter factory=".datetime.datetime_from_timestamp" for="float" />
 
-    <!-- Do the reverse as well.-->
+    <!-- Do the reverse as well. -->
     <adapter factory=".datetime.date_to_string" />
     <adapter factory=".datetime.datetime_to_string" />
     <adapter factory=".datetime.duration_to_string" />
 
+    <!-- -->
+    <!-- Numbers (fractions) -->
+    <!-- -->
+    <!-- Anything not represented directly goes out as a string. -->
+    <adapter factory=".numbers.second_chance_number_externalizer" />
+
+    <!-- -->
+    <!-- Object factories -->
+    <!-- -->
     <adapter for="*"
              factory=".internalization.default_externalized_object_factory_finder_factory"
              provides=".interfaces.IExternalizedObjectFactoryFinder" />

src/nti/externalization/externalization/externalizer.py

@@ -17,7 +17,7 @@
 import warnings
 try:
     from collections.abc import Set
-except ImportError:
+except ImportError: # Python 2
     from collections import Set
     from collections import Mapping
 else: # pragma: no cover
@@ -360,10 +360,11 @@ def to_external_object(
     :param decorate_callback: Callable to be invoked in case there is
         no decaration
     """
-
     # Catch the primitives up here, quickly. This catches
-    # numbers, strings, and None
-    if isinstance(obj, PRIMITIVES):
+    # numbers, strings, and None. Only do this if we're not on the
+    # second-pass fallback; that means that some representer couldn't handle
+    # a primitive natively (usually a decimal)
+    if name != 'second-pass' and isinstance(obj, PRIMITIVES):
         return obj
 
     manager_top = _manager_get() # (name, memos)

src/nti/externalization/internalization/externals.py

@@ -16,7 +16,7 @@
 # stdlib imports
 try:
     from collections.abc import MutableSequence
-except ImportError:
+except ImportError: # Python 2
     from collections import MutableSequence
 
 

src/nti/externalization/internalization/fields.py

@@ -348,7 +348,7 @@ def validate_named_field_value(self, iface, field_name, value):
 def _as_native_str(s):
     if isinstance(s, str):
         return s
-    return s.encode('ascii')
+    return s.encode('ascii') # Python 2
 
 
 from nti.externalization._compat import import_c_accel # pylint:disable=wrong-import-position,wrong-import-order

src/nti/externalization/internalization/updater.py

@@ -13,7 +13,7 @@
 # stdlib imports
 try:
     from collections.abc import MutableSequence
-except ImportError:
+except ImportError: # Python 2
     from collections import MutableSequence
     from collections import MutableMapping
 else: # pragma: no cover
@@ -90,11 +90,11 @@ def _get_update_signature(updater):
     if spec is None:
         try:
             func = updater.updateFromExternalObject
-            if hasattr(inspect, 'getfullargspec'): # pragma: no cover
+            if hasattr(inspect, 'getfullargspec'):
                 # Python 3. getargspec() is deprecated.
                 argspec = inspect.getfullargspec(func) # pylint:disable=no-member
                 keywords = argspec.varkw
-            else:
+            else: # Python 2
                 argspec = inspect.getargspec(func)
                 keywords = argspec.keywords
             args = argspec.args

src/nti/externalization/numbers.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+"""
+Support for externalizing arbitrary numbers.
+
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import fractions
+import decimal
+
+from zope import interface
+from zope import component
+from zope.interface.common.numbers import INumber
+from zope.interface.common.numbers import IRational
+
+from nti.externalization.interfaces import IInternalObjectExternalizer
+
+
+@interface.implementer(IInternalObjectExternalizer)
+@component.adapter(INumber)
+class second_chance_number_externalizer(object):
+    def __init__(self, context):
+        self.context = context
+
+    def toExternalObject(self, **unused_kwargs):
+        return str(self.context)
+
+# Depending on the order of imports, these may or may not have
+# been declared already.
+if not IRational.providedBy(fractions.Fraction('1/3')): # pragma: no cover
+    interface.classImplements(fractions.Fraction, IRational)
+
+if not INumber.providedBy(decimal.Decimal('1')): # pragma: no cover
+    # NOT an IReal; see notes in stdlib numbers.py for why.
+    interface.classImplements(decimal.Decimal, INumber)
+
+assert IRational.providedBy(fractions.Fraction('1/3'))
+assert INumber.providedBy(decimal.Decimal('1'))

src/nti/externalization/persistence.py

@@ -19,7 +19,7 @@
 # stdlib imports
 try:
     from collections.abc import Sequence
-except ImportError:
+except ImportError: # Python 2
     from collections import Sequence
 
 try:

src/nti/externalization/representation.py

@@ -13,6 +13,7 @@
 from __future__ import division
 from __future__ import print_function
 
+import decimal
 import warnings
 
 from persistent import Persistent
@@ -80,7 +81,7 @@ def to_json_representation(obj):
 def _second_pass_to_external_object(obj):
     result = toExternalObject(obj, name='second-pass')
     if result is obj:
-        raise TypeError(repr(obj) + " is not JSON serializable")
+        raise TypeError(repr(obj) + " is not serializable")
     return result
 
 
@@ -141,14 +142,44 @@ class _ExtDumper(yaml.SafeDumper):
 
     Therefore we must register their base types as multi-representers.
     """
+
+# The difference between 'add_representer' and 'add_multi_representer'
+# is that the multi version accepts subclasses, but the plain version
+# requires an exact type match.
 _ExtDumper.add_multi_representer(list, _ExtDumper.represent_list)
 _ExtDumper.add_multi_representer(dict, _ExtDumper.represent_dict)
-if str is bytes:
+if str is bytes: # Python 2
+     # pylint:disable=undefined-variable,no-member
     _ExtDumper.add_multi_representer(unicode, _ExtDumper.represent_unicode)
-else: # pragma: no cover Python 3
+else: # Python 3
     _ExtDumper.add_multi_representer(str, _ExtDumper.represent_str)
 
-class _UnicodeLoader(yaml.SafeLoader):  # pylint:disable=R0904
+def _yaml_represent_decimal(dumper, data):
+    s = str(data)
+    if '.' not in s:
+        try:
+            int(s)
+        except ValueError:
+            pass
+        else:
+            return dumper.represent_int(data)
+    if data.is_nan():
+        return dumper.represent_float(float('nan'))
+    if data.is_infinite():
+        return dumper.represent_float(float('-inf') if data.is_signed() else float('+inf'))
+    return dumper.represent_scalar('tag:yaml.org,2002:float', str(data).lower())
+_ExtDumper.add_representer(decimal.Decimal, _yaml_represent_decimal)
+
+# PyYAML uses the multi dumper on ``None`` as the fallback when
+# nothing else can be found.
+def _yaml_represent_unknown(dumper, data):
+    ext_obj = _second_pass_to_external_object(data)
+    return dumper.represent_data(ext_obj)
+_ExtDumper.add_multi_representer(None, _yaml_represent_unknown)
+
+
+
+class _UnicodeLoader(yaml.SafeLoader):
 
     def construct_yaml_str(self, node):
         # yaml defines strings to be unicode, but
@@ -207,6 +238,7 @@ class _PReprException(Exception):
     # Raised for the sole purpose of carrying a smuggled
     # repr.
     def __init__(self, value):
+        Exception.__init__(self)
         self.value = value
 
     def __repr__(self):

src/nti/externalization/tests/test_externalization.py

@@ -67,8 +67,8 @@
 
 try:
     from collections import UserDict
-except ImportError:
-    from UserDict import UserDict # Python 2
+except ImportError: # Python 2
+    from UserDict import UserDict
 
 
 # disable: accessing protected members, too many methods

src/nti/externalization/tests/test_persistence.py

@@ -247,7 +247,7 @@ def _persist_cpickle(self, obj):
         except ImportError: # pragma: no cover
             # Python 3
             raise TypeError("Not allowed to pickle")
-        else:
+        else: # pragma: no cover
             cPickle.dumps(obj)
 
     def _all_persists_fail(self, factory):