Add the ability to force use of C extensions.

With PURE_PYTHON=0, like in zope.interface.

Also always require all three extensions. This solves mysterious issues you can get if you wind up mixing and matching (#124).

Fixes #131

Add travis and tox tests for this.

.travis.yml

@@ -32,6 +32,10 @@ jobs:
       python: 3.7
       env: PURE_PYTHON=1
 
+    - name: "Python: 3.8, (forced C extensions)"
+      python: 3.7
+      env: PURE_PYTHON=0
+
     - name: "Documentation"
       python: 3.6
       install:

CHANGES.rst

@@ -1,9 +1,19 @@
 ``persistent`` Changelog
 ========================
 
-4.5.2 (unreleased)
+4.6.0 (unreleased)
 ------------------
 
+- Update the handling of the ``PURE_PYTHON`` environment variable.
+  Now, a value of 0 requires the C extensions to be used; any other
+  value prevents the extensions from being used. Also, all C
+  extensions are required together or none of them will be used. This
+  prevents strange errors that arise from a mismatch of Python and C
+  implementations. See `issue 131 <https://github.com/zopefoundation/persistent/issues/131>`_.
+
+  Note that some private implementation details such as the names of
+  the pure-Python implementations have changed.
+
 - Fix ``PersistentList`` to mark itself as changed after calling
   ``clear``. See `PR 115 <https://github.com/zopefoundation/persistent/pull/115/>`_.
 

persistent/__init__.py

@@ -28,38 +28,21 @@
     'PickleCache',
     'TimeStamp',
 ]
-from persistent._compat import PURE_PYTHON
+
 from persistent.interfaces import IPersistent
-from persistent.interfaces import IPickleCache
 
 import persistent.timestamp as TimeStamp
 
-from persistent import persistence as pyPersistence
-from persistent import picklecache as pyPickleCache
-
-try:
-    # Be careful not to shadow the modules
-    from persistent import cPersistence as _cPersistence
-    from persistent import cPickleCache as _cPickleCache
-except ImportError: # pragma: no cover
-    _cPersistence = None
-    _cPickleCache = None
-else:
-    # Make an interface declaration for Persistent
-    # Note that the Python version already does this.
-    from zope.interface import classImplements
-    classImplements(_cPersistence.Persistent, IPersistent)
-    classImplements(_cPickleCache.PickleCache, IPickleCache)
-
-
-_persistence = pyPersistence if PURE_PYTHON or _cPersistence is None else _cPersistence
-_picklecache = pyPickleCache if PURE_PYTHON or _cPickleCache is None else _cPickleCache
+from persistent import persistence as _persistence
+from persistent import picklecache as _picklecache
 
 Persistent = _persistence.Persistent
+PersistentPy = _persistence.PersistentPy
 GHOST = _persistence.GHOST
 UPTODATE = _persistence.UPTODATE
 CHANGED = _persistence.CHANGED
 STICKY = _persistence.STICKY
 PickleCache = _picklecache.PickleCache
+PickleCachePy = _picklecache.PickleCachePy
 
 sys.modules['persistent.TimeStamp'] = sys.modules['persistent.timestamp']

persistent/_compat.py

@@ -15,7 +15,8 @@
 import sys
 import os
 
-PURE_PYTHON = os.environ.get('PURE_PYTHON')
+from zope.interface import implementedBy
+from zope.interface import classImplements
 
 if sys.version_info[0] > 2:
     import copyreg as copy_reg
@@ -35,3 +36,124 @@
     PYTHON2 = True
 
     intern = intern
+
+PYPY = hasattr(sys, 'pypy_version_info')
+
+
+def _c_optimizations_required():
+    """
+    Return a true value if the C optimizations are required.
+
+    This uses the ``PURE_PYTHON`` variable as documented in `_use_c_impl`.
+    """
+    pure_env = os.environ.get('PURE_PYTHON')
+    require_c = pure_env == "0"
+    return require_c
+
+
+def _c_optimizations_available():
+    """
+    Return the C optimization modules, if available, otherwise
+    a false value.
+
+    If the optimizations are required but not available, this
+    raises the ImportError. Either all optimization modules are
+    available or none are.
+
+    This does not say whether they should be used or not.
+    """
+    catch = () if _c_optimizations_required() else (ImportError,)
+    try:
+        from persistent import cPersistence
+        from persistent import cPickleCache
+        from persistent import _timestamp
+        return {
+            'persistent.persistence': cPersistence,
+            'persistent.picklecache': cPickleCache,
+            'persistent.timestamp': _timestamp,
+        }
+    except catch: # pragma: no cover (only Jython doesn't build extensions)
+        return {}
+
+
+def _c_optimizations_ignored():
+    """
+    The opposite of `_c_optimizations_required`.
+    """
+    pure_env = os.environ.get('PURE_PYTHON')
+    # The extensions can be compiled with PyPy 7.3, but they don't work.
+    return PYPY or (pure_env is not None and pure_env != "0")
+
+
+def _should_attempt_c_optimizations():
+    """
+    Return a true value if we should attempt to use the C optimizations.
+
+    This takes into account whether we're on PyPy and the value of the
+    ``PURE_PYTHON`` environment variable, as defined in `_use_c_impl`.
+    """
+    if _c_optimizations_required():
+        return True
+    if PYPY: # pragma: no cover
+        return False
+    return not _c_optimizations_ignored()
+
+
+def use_c_impl(py_impl, name=None, globs=None):
+    """
+    Decorator. Given an object implemented in Python, with a name like
+    ``Foo``, import the corresponding C implementation from
+    ``persistent.c<NAME>`` with the name
+    ``Foo`` and use it instead (where ``NAME`` is the module name).
+
+    If the ``PURE_PYTHON`` environment variable is set to any value
+    other than ``"0"``, or we're on PyPy, ignore the C implementation
+    and return the Python version. If the C implementation cannot be
+    imported, return the Python version. If ``PURE_PYTHON`` is set to
+    0, *require* the C implementation (let the ImportError propagate);
+    note that PyPy can import the C implementation in this case (and all
+    tests pass).
+
+    In all cases, the Python version is kept available in the module
+    globals with the name ``FooPy``.
+
+    If the Python version is a class that implements interfaces,
+    then the C version will be declared to also implement those interfaces.
+
+    This can also be used for constants and other things that do not have
+    a name by passing the name as the second argument.
+
+    Example::
+
+        @use_c_impl
+        class Foo(object):
+            ...
+
+        GHOST = use_c_impl(12, 'GHOST')
+    """
+    name = name or py_impl.__name__
+    globs = globs or sys._getframe(1).f_globals
+    mod_name = globs['__name__']
+
+    def find_impl():
+        if not _should_attempt_c_optimizations():
+            return py_impl
+
+        c_opts = _c_optimizations_available()
+        if not c_opts: # pragma: no cover (only Jython doesn't build extensions)
+            return py_impl
+
+        __traceback_info__ = c_opts
+        c_opt = c_opts[mod_name]
+        return getattr(c_opt, name)
+
+    c_impl = find_impl()
+    # Always make available by the FooPy name
+    globs[name + 'Py'] = py_impl
+
+    if c_impl is not py_impl and isinstance(py_impl, type):
+        # copy the interface declarations.
+        implements = list(implementedBy(py_impl))
+        if implements:
+            classImplements(c_impl, *implements)
+    return c_impl

persistent/cPersistence.c

@@ -37,8 +37,8 @@ typedef unsigned long long uint64_t;
 #  include <inttypes.h>
 #endif
 
-/* These two objects are initialized when the module is loaded */
-static PyObject *TimeStamp, *py_simple_new;
+/* These objects are initialized when the module is loaded */
+static PyObject *py_simple_new;
 
 /* Strings initialized by init_strings() below. */
 static PyObject *py_keys, *py_setstate, *py___dict__, *py_timeTime;
@@ -1257,6 +1257,7 @@ Per_set_serial(cPersistentObject *self, PyObject *v)
 static PyObject *
 Per_get_mtime(cPersistentObject *self)
 {
+    static PyObject* TimeStamp;
     PyObject *t, *v;
 
     if (unghostify(self) < 0)
@@ -1270,6 +1271,18 @@ Per_get_mtime(cPersistentObject *self)
         return Py_None;
     }
 
+    if (!TimeStamp)
+    {
+        PyObject* ts_module;
+        ts_module = PyImport_ImportModule("persistent._timestamp");
+        if (!ts_module)
+            return NULL;
+        TimeStamp = PyObject_GetAttrString(ts_module, "TimeStamp");
+        if (!TimeStamp)
+          return NULL;
+        Py_DECREF(ts_module);
+    }
+
 #ifdef PY3K
     t = PyObject_CallFunction(TimeStamp, "y#", self->serial, (Py_ssize_t)8);
 #else
@@ -1701,7 +1714,7 @@ static struct PyModuleDef moduledef =
 static PyObject*
 module_init(void)
 {
-    PyObject *module, *ts_module, *capi;
+    PyObject *module, *capi;
     PyObject *copy_reg;
 
     if (init_strings() < 0)
@@ -1777,15 +1790,6 @@ module_init(void)
       return NULL;
     }
 
-    if (!TimeStamp)
-    {
-      ts_module = PyImport_ImportModule("persistent.timestamp");
-      if (!ts_module)
-        return NULL;
-      TimeStamp = PyObject_GetAttrString(ts_module, "TimeStamp");
-      Py_DECREF(ts_module);
-      /* fall through to immediate return on error */
-    }
     return module;
 }