Merge pull request #34 from NextThought/pypy

Support PyPy.

.gitignore

@@ -17,3 +17,4 @@ coverage.xml
 dist
 testing.log
 .eggs/
+.dir-locals.el

.travis.yml

@@ -1,13 +1,15 @@
 language: python
 sudo: false
 python:
+    - pypy
+    - pypy3
     - 2.6
     - 2.7
     - 3.2
     - 3.3
     - 3.4
 install:
-    - travis_retry pip install BTrees ZConfig manuel persistent six transaction zc.lockfile zdaemon zope.interface zope.testing zope.testrunner==4.4.4
+    - travis_retry pip install BTrees ZConfig manuel persistent six transaction zc.lockfile zdaemon zope.interface zope.testing zope.testrunner
     - travis_retry pip install -e .
 script:
     - zope-testrunner -u --test-path=src --auto-color --auto-progress

CHANGES.rst

@@ -2,12 +2,19 @@
  Change History
 ================
 
-4.1.1 (unreleased)
+4.2.0 (unreleased)
 ==================
 
 - Fix command-line parsing of --verbose and --verify arguments.
   (The short versions -v and -V were parsed correctly.)
 
+- Add support for PyPy.
+
+- Fix the methods in ``ZODB.serialize`` that find object references
+  under Python 2.7 (used in scripts like ``referrers``, ``netspace``,
+  and ``fsrecover`` among others). This requires the addition of the
+  ``zodbpickle`` dependency.
+
 4.1.0 (2015-01-11)
 ==================
 

setup.py

@@ -20,9 +20,10 @@
 interface, rich transaction support, and undo.
 """
 
-VERSION = "4.1.0"
+VERSION = "4.2.0.dev0"
 
 import os
+import platform
 import sys
 from setuptools import setup, find_packages
 
@@ -35,6 +36,10 @@
     sys.exit(0)
 
 PY3 = sys.version_info >= (3,)
+PY27 = sys.version_info >= (2,7)
+py_impl = getattr(platform, 'python_implementation', lambda: None)
+PYPY = py_impl() == 'PyPy'
+
 
 # The (non-obvious!) choices for the Trove Development Status line:
 # Development Status :: 5 - Production/Stable
@@ -54,6 +59,7 @@
 Programming Language :: Python :: 3.3
 Programming Language :: Python :: 3.4
 Programming Language :: Python :: Implementation :: CPython
+Programming Language :: Python :: Implementation :: PyPy
 Topic :: Database
 Topic :: Software Development :: Libraries :: Python Modules
 Operating System :: Microsoft :: Windows
@@ -153,15 +159,15 @@ def read_file(*path):
       tests_require = tests_require,
       extras_require = dict(test=tests_require),
       install_requires = [
-        'persistent',
-        'BTrees',
+        'persistent >= 4.1.0',
+        'BTrees >= 4.1.3',
         'ZConfig',
-        'transaction >= 1.4.1' if PY3 else 'transaction',
+        'transaction >= 1.4.4',
         'six',
         'zc.lockfile',
         'zdaemon >= 4.0.0a1',
         'zope.interface',
-        ] + (['zodbpickle >= 0.2'] if PY3 else []),
+        ] + (['zodbpickle >= 0.6.0'] if (PY3 or PY27 or PYPY) else []),
       zip_safe = False,
       entry_points = """
       [console_scripts]

src/ZODB/ConflictResolution.py

@@ -13,13 +13,12 @@
 ##############################################################################
 
 import logging
-import sys
 
 import six
 import zope.interface
 from ZODB.POSException import ConflictError
 from ZODB.loglevels import BLATHER
-from ZODB._compat import BytesIO, Unpickler, Pickler, _protocol
+from ZODB._compat import BytesIO, PersistentUnpickler, PersistentPickler, _protocol
 
 # Subtle: Python 2.x has pickle.PicklingError and cPickle.PicklingError,
 # and these are unrelated classes!  So we shouldn't use pickle.PicklingError,
@@ -74,9 +73,7 @@ def state(self, oid, serial, prfactory, p=''):
     p = p or self.loadSerial(oid, serial)
     p = self._crs_untransform_record_data(p)
     file = BytesIO(p)
-    unpickler = Unpickler(file)
-    unpickler.find_global = find_global
-    unpickler.persistent_load = prfactory.persistent_load
+    unpickler = PersistentUnpickler(find_global, prfactory.persistent_load, file)
     unpickler.load() # skip the class tuple
     return unpickler.load()
 
@@ -243,9 +240,7 @@ def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
         prfactory = PersistentReferenceFactory()
         newpickle = self._crs_untransform_record_data(newpickle)
         file = BytesIO(newpickle)
-        unpickler = Unpickler(file)
-        unpickler.find_global = find_global
-        unpickler.persistent_load = prfactory.persistent_load
+        unpickler = PersistentUnpickler(find_global, prfactory.persistent_load, file)
         meta = unpickler.load()
         if isinstance(meta, tuple):
             klass = meta[0]
@@ -286,11 +281,7 @@ def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
         resolved = resolve(old, committed, newstate)
 
         file = BytesIO()
-        pickler = Pickler(file, _protocol)
-        if sys.version_info[0] < 3:
-            pickler.inst_persistent_id = persistent_id
-        else:
-            pickler.persistent_id = persistent_id
+        pickler = PersistentPickler(persistent_id, file, _protocol)
         pickler.dump(meta)
         pickler.dump(resolved)
         return self._crs_transform_record_data(file.getvalue())

src/ZODB/Connection.py

@@ -328,10 +328,10 @@ def close(self, primary=True):
                 # get back here.
         else:
             self.opened = None
-            
+
         am = self._db._activity_monitor
         if am is not None:
-            am.closedConnection(self)        
+            am.closedConnection(self)
 
     def db(self):
         """Returns a handle to the database this connection belongs to."""
@@ -439,7 +439,6 @@ def abort(self, transaction):
         # the savepoint, then they won't have _p_oid or _p_jar after
         # they've been unadded. This will make the code in _abort
         # confused.
-
         self._abort()
 
         if self._savepoint_storage is not None:
@@ -463,7 +462,6 @@ def _abort(self):
                 if obj._p_changed:
                     obj._p_changed = False
             else:
-
                 # Note: If we invalidate a non-ghostifiable object
                 # (i.e. a persistent class), the object will
                 # immediately reread its state.  That means that the
@@ -868,16 +866,19 @@ def setstate(self, obj):
                 raise
 
         try:
-            self._setstate(obj)
+            self._setstate(obj, oid)
         except ConflictError:
             raise
         except:
             self._log.exception("Couldn't load state for %s %s",
                                 className(obj), oid_repr(oid))
             raise
 
-    def _setstate(self, obj):
+    def _setstate(self, obj, oid):
         # Helper for setstate(), which provides logging of failures.
+        # We accept the oid param, which must be the same as obj._p_oid,
+        # as a performance optimization for the pure-Python persistent implementation
+        # where accessing an attribute involves __getattribute__ calls
 
         # The control flow is complicated here to avoid loading an
         # object revision that we are sure we aren't going to use.  As
@@ -892,7 +893,7 @@ def _setstate(self, obj):
         if self.before is not None:
             # Load data that was current before the time we have.
             before = self.before
-            t = self._storage.loadBefore(obj._p_oid, before)
+            t = self._storage.loadBefore(oid, before)
             if t is None:
                 raise POSKeyError() # historical connection!
             p, serial, end = t
@@ -905,16 +906,16 @@ def _setstate(self, obj):
             if self._invalidatedCache:
                 raise ReadConflictError()
 
-            if (obj._p_oid in self._invalidated):
+            if (oid in self._invalidated):
                 self._load_before_or_conflict(obj)
                 return
 
-            p, serial = self._storage.load(obj._p_oid, '')
+            p, serial = self._storage.load(oid, '')
             self._load_count += 1
 
             self._inv_lock.acquire()
             try:
-                invalid = obj._p_oid in self._invalidated
+                invalid = oid in self._invalidated
             finally:
                 self._inv_lock.release()
 
@@ -924,13 +925,13 @@ def _setstate(self, obj):
 
         self._reader.setGhostState(obj, p)
         obj._p_serial = serial
-        self._cache.update_object_size_estimation(obj._p_oid, len(p))
+        self._cache.update_object_size_estimation(oid, len(p))
         obj._p_estimated_size = len(p)
 
         # Blob support
         if isinstance(obj, Blob):
             obj._p_blob_uncommitted = None
-            obj._p_blob_committed = self._storage.loadBlob(obj._p_oid, serial)
+            obj._p_blob_committed = self._storage.loadBlob(oid, serial)
 
     def _load_before_or_conflict(self, obj):
         """Load non-current state for obj or raise ReadConflictError."""

src/ZODB/DB.py

@@ -530,7 +530,11 @@ def f(con, detail=detail):
     def cacheExtremeDetail(self):
         detail = []
         conn_no = [0]  # A mutable reference to a counter
-        def f(con, detail=detail, rc=sys.getrefcount, conn_no=conn_no):
+        # sys.getrefcount is a CPython implementation detail
+        # not required to exist on, e.g., PyPy.
+        rc = getattr(sys, 'getrefcount', None)
+
+        def f(con, detail=detail, rc=rc, conn_no=conn_no):
             conn_no[0] += 1
             cn = conn_no[0]
             for oid, ob in con._cache_items():
@@ -555,12 +559,15 @@ def f(con, detail=detail, rc=sys.getrefcount, conn_no=conn_no):
                 # sys.getrefcount(ob) returns.  But, in addition to that,
                 # the cache holds an extra reference on non-ghost objects,
                 # and we also want to pretend that doesn't exist.
+                # If we have no way to get a refcount, we return False to symbolize
+                # that. As opposed to None, this has the advantage of being usable
+                # as a number (0) in case clients depended on that.
                 detail.append({
                     'conn_no': cn,
                     'oid': oid,
                     'id': id,
                     'klass': "%s%s" % (module, ob.__class__.__name__),
-                    'rc': rc(ob) - 3 - (ob._p_changed is not None),
+                    'rc': rc(ob) - 3 - (ob._p_changed is not None) if rc else False,
                     'state': ob._p_changed,
                     #'references': con.references(oid),
                     })

src/ZODB/DemoStorage.test

@@ -14,7 +14,10 @@ existing, base, storage without updating the storage.
     ...     return now
     >>> import time
     >>> real_time_time = time.time
-    >>> time.time = faux_time_time
+    >>> if isinstance(time,type):
+    ...    time.time = staticmethod(faux_time_time) # Jython
+    ... else:
+    ...     time.time = faux_time_time
 
 To see how this works, we'll start by creating a base storage and
 puting an object (in addition to the root object) in it:
@@ -45,6 +48,13 @@ and combine the 2 in a demofilestorage:
     >>> from ZODB.DemoStorage import DemoStorage
     >>> storage = DemoStorage(base=base, changes=changes)
 
+The storage will assign OIDs in a pseudo-random fashion, but for test
+purposes we need to control where they start (since the random seeds
+can be different on different platforms):
+
+    >>> storage._next_oid = 3553260803050964942
+
+
 If there are no transactions, the storage reports the lastTransaction
 of the base database:
 
@@ -375,20 +385,20 @@ Now, we create a demostorage.
 
 If we ask for an oid, we'll get 1042.
 
-    >>> u64(storage.new_oid())
+    >>> print(u64(storage.new_oid()))
     1042
 
 oids are allocated seuentially:
 
-    >>> u64(storage.new_oid())
+    >>> print(u64(storage.new_oid()))
     1043
 
 Now, we'll save 1044 in changes so that it has to pick a new one randomly.
 
     >>> t = transaction.get()
     >>> ZODB.tests.util.store(storage.changes, 1044)
 
-    >>> u64(storage.new_oid())
+    >>> print(u64(storage.new_oid()))
     called randint
     2042
 
@@ -400,7 +410,7 @@ to force another attempt:
     >>> oid = storage.new_oid()
     called randint
     called randint
-    >>> u64(oid)
+    >>> print(u64(oid))
     3042
 
 DemoStorage keeps up with the issued OIDs to know when not to reissue them...
@@ -426,4 +436,3 @@ DemoStorage keeps up with the issued OIDs to know when not to reissue them...
 .. restore time
 
     >>> time.time = real_time_time
-

src/ZODB/ExportImport.py

@@ -15,7 +15,6 @@
 
 import logging
 import os
-import sys
 from tempfile import TemporaryFile
 
 import six
@@ -25,7 +24,7 @@
 from ZODB.POSException import ExportError
 from ZODB.serialize import referencesf
 from ZODB.utils import p64, u64, cp, mktemp
-from ZODB._compat import Pickler, Unpickler, BytesIO, _protocol
+from ZODB._compat import PersistentPickler, Unpickler, BytesIO, _protocol
 
 
 logger = logging.getLogger('ZODB.ExportImport')
@@ -178,11 +177,7 @@ def persistent_load(ooid):
             unpickler.persistent_load = persistent_load
 
             newp = BytesIO()
-            pickler = Pickler(newp, _protocol)
-            if sys.version_info[0] < 3:
-                pickler.inst_persistent_id = persistent_id
-            else:
-                pickler.persistent_id = persistent_id
+            pickler = PersistentPickler(persistent_id, newp, _protocol)
 
             pickler.dump(unpickler.load())
             pickler.dump(unpickler.load())

src/ZODB/FileStorage/iterator.test

@@ -13,7 +13,10 @@ We'll make some assertions about time, so we'll take it over:
     ...     return now
     >>> import time
     >>> time_time = time.time
-    >>> time.time = faux_time
+    >>> if isinstance(time,type):
+    ...    time.time = staticmethod(faux_time) # Jython
+    ... else:
+    ...     time.time = faux_time
 
 Commit a bunch of transactions:
 

src/ZODB/POSException.py

@@ -61,6 +61,16 @@ def __reduce__(self):
 
             return (_recon, (self.__class__, state))
 
+        def __setstate__(self, state):
+            # PyPy doesn't store the 'args' attribute in an instance's
+            # __dict__; instead, it uses what amounts to a slot. Because
+            # we customize the pickled representation to just be a dictionary,
+            # the args would then get lost, leading to unprintable exceptions
+            # and worse. Manually assign to args from the state to be sure
+            # this doesn't happen.
+            super(POSError,self).__setstate__(state)
+            self.args = state['args']
+
 class POSKeyError(POSError, KeyError):
     """Key not found in database."""