diff --git a/setup.py b/setup.py index 8055573..d722f1e 100755 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ def _read(fname): 'PyYAML', 'pytz', 'simplejson', - 'six', + 'six >= 1.11.0', # for the reference cycle fix in reraise() 'ZODB', 'zope.annotation', 'zope.cachedescriptors', diff --git a/src/nti/externalization/datastructures.py b/src/nti/externalization/datastructures.py index f867281..e2c5d2b 100644 --- a/src/nti/externalization/datastructures.py +++ b/src/nti/externalization/datastructures.py @@ -40,16 +40,16 @@ def _syntheticKeys(): def _isMagicKey(key): - """ + """ For our mixin objects that have special keys, defines - those keys that are special and not settable by the user. + those keys that are special and not settable by the user. """ return key in _syntheticKeys() isSyntheticKey = _isMagicKey class ExternalizableDictionaryMixin(object): - """ + """ Implements a toExternalDictionary method as a base for subclasses. """ @@ -78,7 +78,7 @@ def toExternalDictionary(self, mergeFrom=None, *unused_args, **kwargs): **kwargs) def stripSyntheticKeysFromExternalDictionary(self, external): - """ + """ Given a mutable dictionary, removes all the external keys that might have been added by toExternalDictionary and echoed back. """ @@ -235,18 +235,18 @@ def updateFromExternalObject(self, parsed, *unused_args, **unused_kwargs): if StandardExternalFields.CONTAINER_ID in parsed \ and getattr(ext_self, StandardInternalFields.CONTAINER_ID, parsed) is None: - setattr(ext_self, + setattr(ext_self, StandardInternalFields.CONTAINER_ID, parsed[StandardExternalFields.CONTAINER_ID]) if StandardExternalFields.CREATOR in parsed \ and getattr(ext_self, StandardExternalFields.CREATOR, parsed) is None: - setattr(ext_self, + setattr(ext_self, StandardExternalFields.CREATOR, parsed[StandardExternalFields.CREATOR]) if ( StandardExternalFields.ID in parsed and getattr(ext_self, StandardInternalFields.ID, parsed) is None and self._ext_accept_external_id(ext_self, parsed)): - setattr(ext_self, + setattr(ext_self, StandardInternalFields.ID, parsed[StandardExternalFields.ID]) @@ -368,7 +368,7 @@ def __init__(self, ext_self, iface_upper_bound=None, validate_after_update=True) # does show up in the profiling data cache = _InterfaceCache.cache_for(self, ext_self) if not cache.iface: - cache.iface = self._ext_find_schema(ext_self, + cache.iface = self._ext_find_schema(ext_self, iface_upper_bound or self._ext_iface_upper_bound) self._iface = cache.iface @@ -382,7 +382,7 @@ def __init__(self, ext_self, iface_upper_bound=None, validate_after_update=True) @property def schema(self): - """ + """ The schema we will use to guide the process """ return self._iface @@ -460,10 +460,9 @@ def _validate_after_update(self, iface, ext_self): try: raise errors[0][1] except SchemaNotProvided as e: - exc_info = sys.exc_info() if not e.args: # zope.schema doesn't fill in the details, which sucks e.args = (errors[0][0],) - raise exc_info[0], exc_info[1], exc_info[2] + raise def toExternalObject(self, mergeFrom=None, **kwargs): ext_class_name = None @@ -472,7 +471,7 @@ def toExternalObject(self, mergeFrom=None, **kwargs): if callable(ext_class_name): # Even though the tagged value may have come from a superclass, # give the actual class (interface) we're using - ext_class_name = ext_class_name(self._iface, + ext_class_name = ext_class_name(self._iface, self._ext_replacement()) if ext_class_name: break @@ -526,7 +525,7 @@ def _ext_find_schema(self, ext_self, iface_upper_bound): "Searching module %s and considered %s on object %s of class %s and type %s" % (most_derived, iface, self._ext_search_module, list(self._ext_schemas_to_consider(ext_self)), - ext_self, ext_self.__class__, + ext_self, ext_self.__class__, type(ext_self))) return most_derived diff --git a/src/nti/externalization/datetime.py b/src/nti/externalization/datetime.py index 289ba38..47483f1 100644 --- a/src/nti/externalization/datetime.py +++ b/src/nti/externalization/datetime.py @@ -12,15 +12,13 @@ from __future__ import print_function, absolute_import, division __docformat__ = "restructuredtext en" -logger = __import__('logging').getLogger(__name__) - import sys import time from datetime import datetime import isodate - import pytz +import six from zope import component from zope import interface @@ -37,13 +35,14 @@ def _parse_with(func, string): try: return func(string) - except isodate.ISO8601Error: - _, v, tb = sys.exc_info() - e = InvalidValue(*v.args, value=string) - raise e, None, tb - - -@component.adapter(basestring) + except isodate.ISO8601Error as e: + e = InvalidValue(*e.args, value=string) + six.reraise(InvalidValue, e, sys.exc_info()[2]) + +_input_type = (str if sys.version_info[0] >= 3 else basestring) +# XXX: This should really be either unicode or str on Python 2. We need to *know* +# what our input type is. All the tests pass on Python 3 with this registered to 'str'. +@component.adapter(_input_type) @interface.implementer(IDate) def _date_from_string(string): """ @@ -89,7 +88,7 @@ def _local_tzinfo(local_tzname=None): and all((bool(x) for x in local_tzname))): offset_hours = time.timezone // 3600 local_tzname = '%s%d%s' % (local_tzname[0], - offset_hours, + offset_hours, local_tzname[1]) tzinfo = _pytz_timezone(local_tzname) @@ -120,7 +119,7 @@ def _as_utc_naive(dt, assume_local=True, local_tzname=None): return dt -@component.adapter(basestring) +@component.adapter(_input_type) @interface.implementer(IDateTime) def datetime_from_string(string, assume_local=False, local_tzname=None): """ diff --git a/src/nti/externalization/externalization.py b/src/nti/externalization/externalization.py index 60c3cb1..15b428d 100644 --- a/src/nti/externalization/externalization.py +++ b/src/nti/externalization/externalization.py @@ -8,13 +8,13 @@ from __future__ import print_function, absolute_import, division __docformat__ = "restructuredtext en" -logger = __import__('logging').getLogger(__name__) - -import six import numbers import collections from collections import defaultdict +import six +from six import iteritems + from zope import component from zope import interface from zope import deprecation @@ -48,6 +48,8 @@ from nti.externalization.oids import to_external_oid +logger = __import__('logging').getLogger(__name__) + # Local for speed StandardExternalFields_ID = StandardExternalFields.ID StandardExternalFields_OID = StandardExternalFields.OID @@ -151,7 +153,7 @@ class _ExternalizationState(object): registry = None def __init__(self, **kwargs): - for k, v in kwargs.iteritems(): + for k, v in iteritems(kwargs): setattr(self, k, v) @CachedProperty @@ -316,7 +318,7 @@ def toExternalObject(obj, decorate_callback=None, default_non_externalizable_replacer=DefaultNonExternalizableReplacer, **kwargs): - """ + """ Translates the object into a form suitable for external distribution, through some data formatting process. See :const:`SEQUENCE_TYPES` and :const:`MAPPING_TYPES` for details on what we can handle by default. @@ -393,7 +395,7 @@ def get_externals(): def get_external_param(name, default=None): """ - Return the currently value for an externalization param or default + Return the currently value for an externalization param or default """ try: return get_externals()[name] @@ -403,9 +405,9 @@ def get_external_param(name, default=None): def stripSyntheticKeysFromExternalDictionary(external): - """ + """ Given a mutable dictionary, removes all the external keys - that might have been added by :func:`to_standard_external_dictionary` and echoed back. + that might have been added by :func:`to_standard_external_dictionary` and echoed back. """ for key in _syntheticKeys(): external.pop(key, None) @@ -417,7 +419,7 @@ def _syntheticKeys(): def _isMagicKey(key): - """ + """ For our mixin objects that have special keys, defines those keys that are special and not settable by the user. """ diff --git a/src/nti/externalization/internalization.py b/src/nti/externalization/internalization.py index 077168d..feebcae 100644 --- a/src/nti/externalization/internalization.py +++ b/src/nti/externalization/internalization.py @@ -10,14 +10,14 @@ from __future__ import print_function, absolute_import, division __docformat__ = "restructuredtext en" -logger = __import__('logging').getLogger(__name__) - -import six import sys import inspect import numbers import collections +import six +from six import reraise + from zope import component from zope import interface @@ -45,6 +45,8 @@ from nti.externalization.interfaces import ObjectModifiedFromExternalEvent from nti.externalization.interfaces import IExternalizedObjectFactoryFinder +logger = __import__('logging').getLogger(__name__) + LEGACY_FACTORY_SEARCH_MODULES = set() StandardExternalFields_CLASS = StandardExternalFields.CLASS @@ -97,7 +99,7 @@ def _search_for_external_factory(typeName, search_set=None): module = resolve(module_name) except (AttributeError, ImportError): # This is a programming error, so that's why we log it - logger.exception("Failed to resolve legacy factory search module %s", + logger.exception("Failed to resolve legacy factory search module %s", module_name) value = getattr(module, '__dict__', _EMPTY_DICT) if mod_dict is None else mod_dict @@ -122,12 +124,12 @@ def default_externalized_object_factory_finder(externalized_object): name=mime_type) if not factory: # What about a named utility? - factory = component.queryUtility(IMimeObjectFactory, + factory = component.queryUtility(IMimeObjectFactory, name=mime_type) if not factory: # Is there a default? - factory = component.queryAdapter(externalized_object, + factory = component.queryAdapter(externalized_object, IMimeObjectFactory) if not factory and StandardExternalFields_CLASS in externalized_object: @@ -164,7 +166,7 @@ def find_factory_for(externalized_object, registry=component): Given a :class:`IExternalizedObject`, locate and return a factory to produce a Python object to hold its contents. """ - factory_finder = registry.getAdapter(externalized_object, + factory_finder = registry.getAdapter(externalized_object, IExternalizedObjectFactoryFinder) return factory_finder.find_factory(externalized_object) @@ -202,7 +204,7 @@ def _resolve_externals(object_io, updating_object, externalObject, elif len(inspect.getargspec(resolver_func)[0]) == 4: # instance method _resolver_func = resolver_func - def resolver_func(x, y, z): + def resolver_func(x, y, z): return _resolver_func(object_io, x, y, z) externalObject[ext_key] = resolver_func(context, externalObject, @@ -381,10 +383,10 @@ def update_from_external_object(containedObject, externalObject, # The signature may vary. argspec = inspect.getargspec(updater.updateFromExternalObject) if 'context' in argspec.args or (argspec.keywords and 'dataserver' not in argspec.args): - updated = updater.updateFromExternalObject(externalObject, + updated = updater.updateFromExternalObject(externalObject, context=context) elif argspec.keywords or 'dataserver' in argspec.args: - updated = updater.updateFromExternalObject(externalObject, + updated = updater.updateFromExternalObject(externalObject, dataserver=context) else: updated = updater.updateFromExternalObject(externalObject) @@ -436,7 +438,9 @@ def validate_field_value(self, field_name, field, value): # Nope. TypeError (or AttrError - Variant) means we couldn't adapt, # and a validation error means we could adapt, but it still wasn't # right. Raise the original SchemaValidationError. - raise exc_info[0], exc_info[1], exc_info[2] + reraise(*exc_info) + finally: + del exc_info except WrongType as e: # Like SchemaNotProvided, but for a primitive type, # most commonly a date @@ -454,7 +458,7 @@ def validate_field_value(self, field_name, field, value): value = component.getAdapter(value, schema) except (LookupError, TypeError): # No registered adapter, darn - raise exc_info[0], exc_info[1], exc_info[2] + raise reraise(*exc_info) except ValidationError as e: # Found an adapter, but it does its own validation, # and that validation failed (eg, IDate below) @@ -462,6 +466,8 @@ def validate_field_value(self, field_name, field, value): # so go with it after ensuring it has a field e.field = field raise + finally: + del exc_info # Lets try again with the adapted value return validate_field_value(self, field_name, field, value) @@ -499,7 +505,11 @@ def converter(x): return x # to raise the original error. If we could adapt, # but the converter does its own validation (e.g., fromObject) # then we want to let that validation error rise - raise exc_info[0], exc_info[1], exc_info[2] + try: + raise reraise(*exc_info) + finally: + del exc_info + # Now try to set the converted value try: @@ -508,7 +518,9 @@ def converter(x): return x # Nope. TypeError means we couldn't adapt, and a # validation error means we could adapt, but it still wasn't # right. Raise the original SchemaValidationError. - raise exc_info[0], exc_info[1], exc_info[2] + raise reraise(*exc_info) + finally: + del exc_info if ( field.readonly and field.get(self) is None diff --git a/src/nti/externalization/representation.py b/src/nti/externalization/representation.py index 194b448..7646e91 100644 --- a/src/nti/externalization/representation.py +++ b/src/nti/externalization/representation.py @@ -9,10 +9,11 @@ from __future__ import print_function, absolute_import, division __docformat__ = "restructuredtext en" -logger = __import__('logging').getLogger(__name__) +import collections +import plistlib import six -import collections +from six import iteritems from zope import component from zope import interface @@ -59,7 +60,10 @@ def to_json_representation(obj): # Plist -import plistlib + +# The API changed in Python 3.4 +_plist_dump = getattr(plistlib, 'dump', None) or plistlib.writePlist +_plist_dumps = getattr(plistlib, 'dumps', None) or plistlib.writePlistToString @interface.named(EXT_REPR_PLIST) @@ -67,23 +71,23 @@ def to_json_representation(obj): class PlistRepresenter(object): def stripNoneFromExternal(self, obj): - """ + """ Given an already externalized object, strips ``None`` values. """ - if isinstance(obj, list) or isinstance(obj, tuple): + if isinstance(obj, (list, tuple)): obj = [self.stripNoneFromExternal(x) for x in obj if x is not None] elif isinstance(obj, collections.Mapping): obj = {k: self.stripNoneFromExternal(v) - for k, v in obj.iteritems() + for k, v in iteritems(obj) if v is not None and k is not None} return obj def dump(self, obj, fp=None): ext = self.stripNoneFromExternal(obj) if fp is not None: - plistlib.writePlist(ext, fp) + _plist_dump(ext, fp) else: - return plistlib.writePlistToString(ext) + return _plist_dumps(ext) # JSON @@ -122,9 +126,9 @@ def dump(self, obj, fp=None): return simplejson.dumps(obj, **self._DUMP_ARGS) def load(self, stream): - # We need all string values to be unicode objects. simplejson is different from - # the built-in json and returns strings that can be represented as ascii as str - # objects if the input was a bytestring. + # We need all string values to be unicode objects. simplejson is different from + # the built-in json and returns strings that can be represented as ascii as str + # objects if the input was a bytestring. # The only way to get it to return unicode is if the input is unicode, or # to use a hook to do so incrementally. The hook saves allocating the entire request body # as a unicode string in memory and is marginally faster in some cases. However, @@ -161,9 +165,10 @@ class _ExtDumper(yaml.SafeDumper): # pylint:disable=R0904 """ _ExtDumper.add_multi_representer(list, _ExtDumper.represent_list) _ExtDumper.add_multi_representer(dict, _ExtDumper.represent_dict) -if six.PY2: - _ExtDumper.add_multi_representer(six.text_type, _ExtDumper.represent_unicode) - +if str is bytes: + _ExtDumper.add_multi_representer(unicode, _ExtDumper.represent_unicode) +else: + _ExtDumper.add_multi_representer(str, _ExtDumper.represent_str) class _UnicodeLoader(yaml.SafeLoader): # pylint:disable=R0904 @@ -196,7 +201,7 @@ def load(self, stream): def make_repr(default=None): if default is None: - def default(self): + def default(self): return "%s().__dict__.update( %s )" % (self.__class__.__name__, self.__dict__) def __repr__(self): diff --git a/src/nti/externalization/singleton.py b/src/nti/externalization/singleton.py index fa7efc0..bbe1402 100644 --- a/src/nti/externalization/singleton.py +++ b/src/nti/externalization/singleton.py @@ -27,6 +27,9 @@ class SingletonDecorator(type): A singleton class has only one instance which is returned every time the class is instantiated. + .. note:: We cannot be used with :func:`six.with_metaclass` because it introduces + temporary classes. You'll need to use the metaclass constructor directly. + ** Developer notes ** The class is instanciated immediately at the point where it is defined by calling cls.__new__(cls). This instance is cached and cls.__new__ is diff --git a/src/nti/externalization/tests/test_externalization.py b/src/nti/externalization/tests/test_externalization.py index febfd3d..e1143ab 100644 --- a/src/nti/externalization/tests/test_externalization.py +++ b/src/nti/externalization/tests/test_externalization.py @@ -97,7 +97,7 @@ class T(object): T._p_state = 42 assert_that(getPersistentState(T()), is_(42)) - def f(unused_s): + def f(unused_s): return 99 T.getPersistentState = f del T._p_state @@ -110,27 +110,27 @@ class T(object): t = T() t._p_oid = b'\x00\x01' - assert_that(toExternalOID(t), is_('0x01')) + assert_that(toExternalOID(t), is_(b'0x01')) t._p_jar = t db = T() db.database_name = 'foo' t.db = lambda: db del t._v_to_external_oid - assert_that(toExternalOID(t), is_('0x01:666f6f')) + assert_that(toExternalOID(t), is_(b'0x01:666f6f')) - assert_that(fromExternalOID('0x01:666f6f')[0], + assert_that(fromExternalOID('0x01:666f6f')[0], is_(b'\x00\x00\x00\x00\x00\x00\x00\x01')) assert_that(fromExternalOID('0x01:666f6f')[0], is_(bytes)) - assert_that(fromExternalOID('0x01:666f6f')[1], is_('foo')) + assert_that(fromExternalOID('0x01:666f6f')[1], is_(b'foo')) # Given a plain OID, we return just the plain OID oid = b'\x00\x00\x00\x00\x00\x00\x00\x01' - assert_that(fromExternalOID(oid), + assert_that(fromExternalOID(oid), contains(same_instance(oid), '', None)) - + def test_hookable(self): - assert_that(set_external_identifiers, + assert_that(set_external_identifiers, has_attr('implementation', is_not(none()))) def test_to_external_representation_none_handling(self): @@ -139,16 +139,18 @@ def test_to_external_representation_none_handling(self): assert_that(json.loads(to_external_representation(d, EXT_FORMAT_JSON)), is_(d)) # PList strips it - assert_that(plistlib.readPlistFromString(to_external_representation(d, EXT_FORMAT_PLIST)), + # The api changed in Python 3.4 + read_plist = getattr(plistlib, 'loads', getattr(plistlib, 'readPlistFromString', None)) + assert_that(read_plist(to_external_representation(d, EXT_FORMAT_PLIST)), is_({'a': 1})) def test_to_external_representation_yaml(self): l = LocatedExternalList() l.append(LocatedExternalDict(k='v')) - class SubUnicode(unicode): + class SubUnicode(str if bytes is not str else unicode): pass - l.append(LocatedExternalDict(k2=SubUnicode('foo'))) + l.append(LocatedExternalDict(k2=SubUnicode(u'foo'))) assert_that(to_external_representation(l, EXT_REPR_YAML), is_('- {k: v}\n- {k2: foo}\n')) @@ -163,9 +165,9 @@ class C(UserDict, ExternalizableDictionaryMixin): def test_broken(self): # Without the devmode hooks gsm = component.getGlobalSiteManager() - gsm.unregisterAdapter(factory=_DevmodeNonExternalizableObjectReplacer, + gsm.unregisterAdapter(factory=_DevmodeNonExternalizableObjectReplacer, required=()) - gsm.unregisterAdapter(factory=_DevmodeNonExternalizableObjectReplacer, + gsm.unregisterAdapter(factory=_DevmodeNonExternalizableObjectReplacer, required=(interface.Interface,)) assert_that(toExternalObject(Broken(), registry=gsm), @@ -179,8 +181,8 @@ class Raises(object): def toExternalObject(self, **unused_kwargs): assert False - assert_that(toExternalObject([Raises()], - catch_components=(AssertionError,), + assert_that(toExternalObject([Raises()], + catch_components=(AssertionError,), catch_component_action=catch_replace_action), is_([catch_replace_action(None, None)])) @@ -198,7 +200,7 @@ class X(object): x.FooBar = Y # Something with a __dict__ already - assert_that(_search_for_external_factory('FooBar', search_set=[x]), + assert_that(_search_for_external_factory('FooBar', search_set=[x]), same_instance(Y)) # Something in sysmodules @@ -206,12 +208,12 @@ class X(object): assert n not in sys.modules sys.modules[n] = x - assert_that(_search_for_external_factory('FooBar', search_set=[n]), + assert_that(_search_for_external_factory('FooBar', search_set=[n]), same_instance(Y)) del sys.modules[n] # something unresolvable - assert_that(_search_for_external_factory('FooBar', search_set=[n]), + assert_that(_search_for_external_factory('FooBar', search_set=[n]), is_(none())) def test_removed_unserializable(self): @@ -262,7 +264,7 @@ class C(persistent.Persistent): class TestExternalizableInstanceDict(ExternalizationLayerTest): class C(ExternalizableInstanceDict): - + def __init__(self): super(TestExternalizableInstanceDict.C, self).__init__() self.A1 = None @@ -410,9 +412,9 @@ class X(object): assert_that(X(), verifiably_provides(dub_interfaces.IDCTimes)) ex_dic = to_standard_external_dictionary(X()) - assert_that(ex_dic, + assert_that(ex_dic, has_entry(StandardExternalFields.LAST_MODIFIED, is_(Number))) - assert_that(ex_dic, + assert_that(ex_dic, has_entry(StandardExternalFields.CREATED_TIME, is_(Number))) diff --git a/src/nti/externalization/tests/test_singleton.py b/src/nti/externalization/tests/test_singleton.py index fb92c07..16ed31a 100644 --- a/src/nti/externalization/tests/test_singleton.py +++ b/src/nti/externalization/tests/test_singleton.py @@ -7,6 +7,8 @@ # disable: accessing protected members, too many methods # pylint: disable=W0212,R0904 +from six import with_metaclass + from hamcrest import is_ from hamcrest import assert_that from hamcrest import same_instance @@ -20,8 +22,9 @@ class TestSingleton(ExternalizationLayerTest): def test_singleton_decorator(self): - class X(object): - __metaclass__ = SingletonDecorator + # Torturous way of getting a metaclass in a Py2/Py3 compatible + # way. + X = SingletonDecorator('X', (object,), {}) # No context assert_that(X(), is_(same_instance(X())))