Basic cythonization of datastructures, externalization, internalizati…

…on and singleton. No optimizations yet. All tests pass.

.gitignore

@@ -1,4 +1,9 @@
 *.py[cod]
+*.so
+*.c
+
+__pycache__/*
+benchmark_results/
 
 /.project
 /.pydevproject

.travis.yml

@@ -5,6 +5,12 @@ python:
   - 3.6
   - pypy
   - pypy3
+matrix:
+  include:
+    - python: 2.7
+      env: PURE_PYTHON=1
+    - python: 3.6
+      env: PURE_PYTHON=1
 script:
   - coverage run -m zope.testrunner --test-path=src
 after_success:

CHANGES.rst

@@ -10,6 +10,12 @@
   ``update_from_external_object``. See
   https://github.com/NextThought/nti.externalization/issues/29.
 
+- A number of deprecated aliases for moved functions have been
+  removed.
+
+- On CPython, some of the modules are compiled as extension modules
+  using Cython for a 10-30% increase in speed. Set the ``PURE_PYTHON``
+  environment variable to disable this at runtime.
 
 1.0.0a1 (2017-09-29)
 ====================

MANIFEST.in

@@ -17,3 +17,4 @@ recursive-include docs *.rst
 recursive-include docs Makefile
 
 recursive-include src *.zcml
+recursive-include src *.c

setup.cfg

@@ -5,4 +5,4 @@ cover-package=nti.externalization
 dev = develop easy_install nti.externalization[test]
 
 [bdist_wheel]
-universal = 1
+universal = 0

setup.py

@@ -1,5 +1,12 @@
+import os
+import sys
 import codecs
-from setuptools import setup, find_packages
+
+from setuptools import setup
+from setuptools import find_packages
+from setuptools import Extension
+
+PYPY = hasattr(sys, 'pypy_version_info')
 
 entry_points = {
     'console_scripts': [
@@ -17,6 +24,54 @@ def _read(fname):
     with codecs.open(fname, encoding='utf-8') as f:
         return f.read()
 
+# Cython
+
+try:
+    from Cython.Build import cythonize
+except ImportError:
+    # The .c files had better already exist, as they should in
+    # an sdist. Based on code from
+    # http://cython.readthedocs.io/en/latest/src/reference/compilation.html#distributing-cython-modules
+    def cythonize(extensions, **_kwargs):
+        for extension in extensions:
+            sources = []
+            for sfile in extension.sources:
+                path, ext = os.path.splitext(sfile)
+                if ext in ('.pyx', '.py'):
+                    ext = '.c'
+                    sfile = path + ext
+                sources.append(sfile)
+            extension.sources[:] = sources
+        return extensions
+
+def cythonize1(ext):
+    new_ext = cythonize([ext])[0]
+    return new_ext
+
+ext_modules = []
+
+# Modules we want to compile with Cython. These *should* have a parallel
+# .pxd file (with a leading _) defining cython attributes.
+# They should also have a cython comment at the top giving options,
+# and mention that they are compiled with cython on CPython.
+# The bottom of the file must call import_c_accel.
+# We use the support from Cython 28 to be able to parallel compile
+# and cythonize modules to a different name with a leading _.
+# This list is derived from the profile of bm_simple_iface
+# https://github.com/NextThought/nti.externalization/commit/0bc4733aa8158acd0d23c14de2f9347fb698c040
+if not PYPY:
+    for mod_name in (
+            'datastructures',
+            'externalization',
+            'internalization',
+            'singleton',
+    ):
+        ext_modules.append(
+            cythonize1(
+                Extension(
+                    'nti.externalization._' + mod_name,
+                    sources=["src/nti/externalization/" + mod_name + '.py'],
+                    depends=["src/nti/externalization/_" + mod_name + '.pxd'])))
 
 setup(
     name='nti.externalization',
@@ -45,6 +100,7 @@ def _read(fname):
     package_dir={'': 'src'},
     include_package_data=True,
     namespace_packages=['nti'],
+    ext_modules=ext_modules,
     tests_require=TESTS_REQUIRE,
     install_requires=[
         'setuptools',
@@ -60,8 +116,6 @@ def _read(fname):
         'zope.component',
         'zope.configuration',
         'zope.container',
-        'zope.deferredimport',
-        'zope.deprecation >= 4.3.0',
         'zope.dottedname',
         'zope.dublincore',
         'zope.event',

src/nti/externalization/_compat.py

@@ -1,15 +1,29 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 """
-
+System spanning utilities.
 """
 
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import os
+import sys
+
 from six import text_type
 
+PY2 = sys.version_info[0] == 2
+PY3 = sys.version_info[0] >= 3
+PYPY = hasattr(sys, 'pypy_version_info')
+WIN = sys.platform.startswith("win")
+LINUX = sys.platform.startswith('linux')
+OSX = sys.platform == 'darwin'
+
+
+PURE_PYTHON = PYPY or os.getenv('PURE_PYTHON')
+
+
 def to_unicode(s, encoding='utf-8', err='strict'):
     """
     Decode a byte sequence and unicode result
@@ -34,3 +48,44 @@ def bytes_(s, encoding='utf-8', errors='strict'):
     from cytoolz import identity
 except ImportError: # PyPy pragma: no cover
     from toolz import identity
+
+identity = identity
+
+def import_c_accel(globs, cname):
+    """
+    Import the C-accelerator for the __name__
+    and copy its globals.
+    """
+
+    name = globs.get('__name__')
+
+    if not name or name == cname:
+        # Do nothing if we're being exec'd as a file (no name)
+        # or we're running from the C extension
+        return
+
+
+    if PURE_PYTHON:
+        return
+
+    import importlib
+    import warnings
+    with warnings.catch_warnings():
+        # Python 3.7 likes to produce
+        # "ImportWarning: can't resolve
+        #   package from __spec__ or __package__, falling back on
+        #   __name__ and __path__"
+        # when we load cython compiled files. This is probably a bug in
+        # Cython, but it doesn't seem to have any consequences, it's
+        # just annoying to see and can mess up our unittests.
+        warnings.simplefilter('ignore', ImportWarning)
+        mod = importlib.import_module(cname)
+
+    # By adopting the entire __dict__, we get a more accurate
+    # __file__ and module repr, plus we don't leak any imported
+    # things we no longer need.
+    globs.clear()
+    globs.update(mod.__dict__)
+
+    if 'import_c_accel' in globs:
+        del globs['import_c_accel']

src/nti/externalization/autopackage.py

@@ -11,13 +11,12 @@
 
 from ZODB.loglevels import TRACE
 from zope import interface
-from zope.deprecation import Suppressor
+
 from zope.dottedname import resolve as dottedname
 from zope.mimetype.interfaces import IContentTypeAware
 
 from nti.externalization.datastructures import ModuleScopedInterfaceObjectIO
-with Suppressor(): # XXX: Temporary
-    from nti.externalization.internalization import register_legacy_search_module
+
 
 logger = __import__('logging').getLogger(__name__)
 

src/nti/externalization/datastructures.py

@@ -16,7 +16,6 @@
 from six import iteritems
 from zope import interface
 from zope import schema
-import zope.deferredimport
 from zope.schema.interfaces import SchemaNotProvided
 
 from nti.schema.interfaces import find_most_derived_interface
@@ -541,13 +540,5 @@ def _ext_schemas_to_consider(self, ext_self):
                 if x.__module__ == self._ext_search_module.__name__
                 and not x.queryTaggedValue('_ext_is_marker_interface'))
 
-# Things that have moved
-zope.deferredimport.initialize()
-zope.deferredimport.deprecatedFrom(
-    "Moved to nti.externalization.interfaces",
-    "nti.externalization.interfaces",
-    "IExternalObject",
-    "LocatedExternalDict",
-    "LocatedExternalList",
-    "ILocatedExternalMapping",
-    "ILocatedExternalSequence")
+from nti.externalization._compat import import_c_accel
+import_c_accel(globals(), 'nti.externalization._datastructures')

src/nti/externalization/externalization.py

@@ -14,6 +14,7 @@
 import collections
 from collections import defaultdict
 import numbers
+import warnings
 
 import BTrees.OOBTree
 from ZODB.POSException import POSKeyError
@@ -23,10 +24,8 @@
 
 
 from zope import component
-from zope import deprecation
 from zope import interface
 
-import zope.deferredimport
 from zope.dublincore.interfaces import IDCTimes
 from zope.interface.common.sequence import IFiniteSequence
 from zope.security.interfaces import IPrincipal
@@ -567,9 +566,11 @@ def decorate_external_mapping(self, result, registry=component, request=_NotGive
 
     return result
 
-toExternalDictionary = to_standard_external_dictionary
-deprecation.deprecated('toExternalDictionary',
-                       'Prefer to_standard_external_dictionary')
+#: This is a deprecated alias
+def toExternalDictionary(*args, **kwargs): # pragma: no cover
+    warnings.warn("Use to_standard_external_dictionary", FutureWarning)
+    return to_standard_external_dictionary(*args, **kwargs)
+
 
 
 def to_minimal_standard_external_dictionary(self, mergeFrom=None, **unused_kwargs):
@@ -626,29 +627,10 @@ def _clean(m):
     _clean(ext)
     return ext
 
-# Things that have moved
-
-zope.deferredimport.initialize()
-zope.deferredimport.deprecatedFrom(
-    "Import from .persistence",
-    "nti.externalization.persistence",
-    "NoPickle")
 
 #: Constant requesting JSON format data
 EXT_FORMAT_JSON = 'json'
 
 
-zope.deferredimport.deprecatedFrom(
-    "Import from .representation",
-    "nti.externalization.representation",
-    "to_external_representation",
-    "to_json_representation",
-    "to_json_representation_externalized",
-    "make_repr",
-    "WithRepr")
-
-zope.deferredimport.initialize()
-zope.deferredimport.deprecatedFrom(
-    "Import from .nti.ntiids.oids",
-    "nti.ntiids.oids",
-    "to_external_ntiid_oid")
+from nti.externalization._compat import import_c_accel
+import_c_accel(globals(), 'nti.externalization._externalization')

src/nti/externalization/interfaces.py

@@ -397,3 +397,11 @@ class IList(IIterable):
     Marker interface for lists
     """
 interface.classImplements(list, IList)
+
+
+class _ILegacySearchModuleFactory(interface.Interface):
+
+    def __call__(*args, **kwargs): # pylint:disable=no-method-argument,arguments-differ
+        """
+        Create and return the object.
+        """