Merge 328253d into 29d207e

.travis.yml

@@ -5,18 +5,27 @@ env:
     - TWINE_USERNAME: zope.wheelbuilder
     # this sets $PYPIPASSWORD
     - secure: "NTWzDr5p8KRPNt+sniTot7csbzC87rzir/XfLtENE0GpQ49FlKw3lBhsDqAPoD8Ea5lwiHXmC/C/ci1UZhFvVEkAoQ2qJlMRnhqUdRJSrqcitmRt0fT6mLaTd+Lr+DxKlBxpssobrEm2G42V/G1s0Ggym04OqF8T+s6MF5ywgJM="
+    # We want to require the C extensions to build and function
+    # everywhere (except where we specifically opt-out, currently just
+    # PyPy, where they build but don't quite work).
+    - PURE_PYTHON: 0
+
 
 python:
   - 2.7
   - 3.5
   - 3.6
   - 3.7
   - 3.8
-  - pypy
-  - pypy3
 
 jobs:
   include:
+    # Don't test C extensions on PyPy.
+    - python: pypy
+      env: PURE_PYTHON=1
+
+    - python: pypy3
+      env: PURE_PYTHON=1
 
     # Special Linux builds
     - name: "Python: 2.7, pure (no C extensions)"

CHANGES.rst

@@ -1,7 +1,7 @@
 ``persistent`` Changelog
 ========================
 
-4.5.2 (unreleased)
+4.6.0 (unreleased)
 ------------------
 
 - Fix slicing of ``PersistentList`` to always return instances of the
@@ -11,6 +11,16 @@
   ``copy.copy`` to also copy the underlying data object. This was
   broken prior to Python 3.7.4.
 
+- Update the handling of the ``PURE_PYTHON`` environment variable.
+  Now, a value of 0 requires the C extensions to be used; any other
+  value prevents the extensions from being used. Also, all C
+  extensions are required together or none of them will be used. This
+  prevents strange errors that arise from a mismatch of Python and C
+  implementations. See `issue 131 <https://github.com/zopefoundation/persistent/issues/131>`_.
+
+  Note that some private implementation details such as the names of
+  the pure-Python implementations have changed.
+
 - Fix ``PersistentList`` to mark itself as changed after calling
   ``clear``. See `PR 115 <https://github.com/zopefoundation/persistent/pull/115/>`_.
 

appveyor.yml

@@ -3,6 +3,7 @@ environment:
     TWINE_USERNAME: zope.wheelbuilder
     TWINE_PASSWORD:
       secure: UcdTh6W78cRLVGfKRFoa5A==
+    PURE_PYTHON: 0
 
   matrix:
     - python: 27

persistent/__init__.py

@@ -28,38 +28,21 @@
     'PickleCache',
     'TimeStamp',
 ]
-from persistent._compat import PURE_PYTHON
+
 from persistent.interfaces import IPersistent
-from persistent.interfaces import IPickleCache
 
 import persistent.timestamp as TimeStamp
 
-from persistent import persistence as pyPersistence
-from persistent import picklecache as pyPickleCache
-
-try:
-    # Be careful not to shadow the modules
-    from persistent import cPersistence as _cPersistence
-    from persistent import cPickleCache as _cPickleCache
-except ImportError: # pragma: no cover
-    _cPersistence = None
-    _cPickleCache = None
-else:
-    # Make an interface declaration for Persistent
-    # Note that the Python version already does this.
-    from zope.interface import classImplements
-    classImplements(_cPersistence.Persistent, IPersistent)
-    classImplements(_cPickleCache.PickleCache, IPickleCache)
-
-
-_persistence = pyPersistence if PURE_PYTHON or _cPersistence is None else _cPersistence
-_picklecache = pyPickleCache if PURE_PYTHON or _cPickleCache is None else _cPickleCache
+from persistent import persistence as _persistence
+from persistent import picklecache as _picklecache
 
 Persistent = _persistence.Persistent
+PersistentPy = _persistence.PersistentPy
 GHOST = _persistence.GHOST
 UPTODATE = _persistence.UPTODATE
 CHANGED = _persistence.CHANGED
 STICKY = _persistence.STICKY
 PickleCache = _picklecache.PickleCache
+PickleCachePy = _picklecache.PickleCachePy
 
 sys.modules['persistent.TimeStamp'] = sys.modules['persistent.timestamp']

persistent/_compat.py

@@ -14,9 +14,23 @@
 
 import sys
 import os
+import types
 
-PURE_PYTHON = os.environ.get('PURE_PYTHON')
+from zope.interface import implementedBy
+from zope.interface import classImplements
 
+__all__ = [
+    'use_c_impl',
+    'copy_reg',
+    'IterableUserDict',
+    'UserList',
+    'intern',
+    'PYPY',
+    'PYTHON3',
+    'PYTHON2',
+]
+
+# pylint:disable=import-error,self-assigning-variable
 if sys.version_info[0] > 2:
     import copyreg as copy_reg
     from collections import UserDict as IterableUserDict
@@ -25,7 +39,6 @@
 
     PYTHON3 = True
     PYTHON2 = False
-
 else: # pragma: no cover
     import copy_reg
     from UserDict import IterableUserDict
@@ -35,3 +48,179 @@
     PYTHON2 = True
 
     intern = intern
+
+PYPY = hasattr(sys, 'pypy_version_info')
+
+
+def _c_optimizations_required():
+    """
+    Return a true value if the C optimizations are required.
+
+    This uses the ``PURE_PYTHON`` variable as documented in `_use_c_impl`.
+    """
+    pure_env = os.environ.get('PURE_PYTHON')
+    require_c = pure_env == "0"
+    return require_c
+
+
+def _c_optimizations_available():
+    """
+    Return the C optimization modules, if available, otherwise
+    a false value.
+
+    If the optimizations are required but not available, this
+    raises the ImportError. Either all optimization modules are
+    available or none are.
+
+    This does not say whether they should be used or not.
+    """
+    catch = () if _c_optimizations_required() else (ImportError,)
+    try:
+        from persistent import cPersistence
+        from persistent import cPickleCache
+        from persistent import _timestamp
+        return {
+            'persistent.persistence': cPersistence,
+            'persistent.picklecache': cPickleCache,
+            'persistent.timestamp': _timestamp,
+        }
+    except catch: # pragma: no cover (only Jython doesn't build extensions)
+        return {}
+
+
+def _c_optimizations_ignored():
+    """
+    The opposite of `_c_optimizations_required`.
+    """
+    pure_env = os.environ.get('PURE_PYTHON')
+    # The extensions can be compiled with PyPy 7.3, but they don't work.
+    return PYPY or (pure_env is not None and pure_env != "0")
+
+
+def _should_attempt_c_optimizations():
+    """
+    Return a true value if we should attempt to use the C optimizations.
+
+    This takes into account whether we're on PyPy and the value of the
+    ``PURE_PYTHON`` environment variable, as defined in `_use_c_impl`.
+    """
+    if _c_optimizations_required():
+        return True
+    if PYPY: # pragma: no cover
+        return False
+    return not _c_optimizations_ignored()
+
+
+def use_c_impl(py_impl, name=None, globs=None):
+    """
+    Decorator. Given an object implemented in Python, with a name like
+    ``Foo``, import the corresponding C implementation from
+    ``persistent.c<NAME>`` with the name ``Foo`` and use it instead
+    (where ``NAME`` is the module name).
+
+    This can also be used for constants and other things that do not
+    have a name by passing the name as the second argument.
+
+    Example::
+
+        @use_c_impl
+        class Foo(object):
+            ...
+
+        GHOST = use_c_impl(12, 'GHOST')
+
+    If the ``PURE_PYTHON`` environment variable is set to any value
+    other than ``"0"``, or we're on PyPy, ignore the C implementation
+    and return the Python version. If the C implementation cannot be
+    imported, return the Python version. If ``PURE_PYTHON`` is set to
+    0, *require* the C implementation (let the ImportError propagate);
+    note that PyPy can import the C implementation in this case (and
+    all tests pass).
+
+    In all cases, the Python version is kept available in the module
+    globals with the name ``FooPy``.
+
+    If the Python version is a class that implements interfaces, then
+    the C version will be declared to also implement those interfaces.
+
+    If the Python version is a class, then each function defined
+    directly in that class will be replaced with a new version using
+    globals that still use the original name of the class for the
+    Python implementation. This lets the function bodies refer to the
+    class using the name the class is defined with, as it would
+    expect. (Only regular functions and static methods are handled.)
+    However, it also means that mutating the module globals later on
+    will not be visible to the methods of the class. In this example,
+    ``Foo().method()`` will always return 1::
+
+        GLOBAL_OBJECT = 1
+        @use_c_impl
+        class Foo(object):
+            def method(self):
+                super(Foo, self).method()
+                return GLOBAL_OBJECT
+        GLOBAL_OBJECT = 2
+    """
+    name = name or py_impl.__name__
+    globs = globs or sys._getframe(1).f_globals
+    mod_name = globs['__name__']
+
+    def find_impl():
+        if not _should_attempt_c_optimizations():
+            return py_impl
+
+        c_opts = _c_optimizations_available()
+        if not c_opts: # pragma: no cover (only Jython doesn't build extensions)
+            return py_impl
+
+        __traceback_info__ = c_opts
+        c_opt = c_opts[mod_name]
+        return getattr(c_opt, name)
+
+    c_impl = find_impl()
+    # Always make available by the FooPy name
+    globs[name + 'Py'] = py_impl
+
+    if c_impl is not py_impl and isinstance(py_impl, type):
+        # Rebind the globals of all the functions to still see the
+        # object under its original class name, so that references
+        # in function bodies work as expected.
+        py_attrs = vars(py_impl)
+        new_globals = None
+        for k, v in list(py_attrs.items()):
+            static = isinstance(v, staticmethod)
+            if static:
+                # Often this is __new__
+                v = v.__func__
+
+            if not isinstance(v, types.FunctionType):
+                continue
+
+            # Somewhat surprisingly, on Python 2, while
+            # ``Class.function`` results in a
+            # ``types.UnboundMethodType`` (``instancemethed``) object,
+            # ``Class.__dict__["function"]`` returns a
+            # ``types.FunctionType``, just like ``Class.function``
+            # (and the dictionary access, of course) does on Python 3.
+            # The upshot is, we don't need different version-dependent
+            # code. Hooray!
+            if new_globals is None:
+                new_globals = v.__globals__.copy()
+                new_globals[py_impl.__name__] = py_impl
+            # On Python 2, all arguments are optional, but an Python 3, all
+            # are required.
+            v = types.FunctionType(
+                v.__code__,
+                new_globals,
+                k, # name
+                v.__defaults__,
+                v.__closure__,
+            )
+            if static:
+                v = staticmethod(v)
+            setattr(py_impl, k, v)
+        # copy the interface declarations.
+        implements = list(implementedBy(py_impl))
+        if implements:
+            classImplements(c_impl, *implements)
+    return c_impl

persistent/cPersistence.c

@@ -37,8 +37,8 @@ typedef unsigned long long uint64_t;
 #  include <inttypes.h>
 #endif
 
-/* These two objects are initialized when the module is loaded */
-static PyObject *TimeStamp, *py_simple_new;
+/* These objects are initialized when the module is loaded */
+static PyObject *py_simple_new;
 
 /* Strings initialized by init_strings() below. */
 static PyObject *py_keys, *py_setstate, *py___dict__, *py_timeTime;
@@ -1257,6 +1257,7 @@ Per_set_serial(cPersistentObject *self, PyObject *v)
 static PyObject *
 Per_get_mtime(cPersistentObject *self)
 {
+    static PyObject* TimeStamp;
     PyObject *t, *v;
 
     if (unghostify(self) < 0)
@@ -1270,6 +1271,18 @@ Per_get_mtime(cPersistentObject *self)
         return Py_None;
     }
 
+    if (!TimeStamp)
+    {
+        PyObject* ts_module;
+        ts_module = PyImport_ImportModule("persistent._timestamp");
+        if (!ts_module)
+            return NULL;
+        TimeStamp = PyObject_GetAttrString(ts_module, "TimeStamp");
+        if (!TimeStamp)
+          return NULL;
+        Py_DECREF(ts_module);
+    }
+
 #ifdef PY3K
     t = PyObject_CallFunction(TimeStamp, "y#", self->serial, (Py_ssize_t)8);
 #else
@@ -1701,7 +1714,7 @@ static struct PyModuleDef moduledef =
 static PyObject*
 module_init(void)
 {
-    PyObject *module, *ts_module, *capi;
+    PyObject *module, *capi;
     PyObject *copy_reg;
 
     if (init_strings() < 0)
@@ -1777,15 +1790,6 @@ module_init(void)
       return NULL;
     }
 
-    if (!TimeStamp)
-    {
-      ts_module = PyImport_ImportModule("persistent.timestamp");
-      if (!ts_module)
-        return NULL;
-      TimeStamp = PyObject_GetAttrString(ts_module, "TimeStamp");
-      Py_DECREF(ts_module);
-      /* fall through to immediate return on error */
-    }
     return module;
 }