Checkpoint on ZODB under PyPy: 25 failures, 25 errors (needs my persi…

…stent fork). Many of them are printing differences, some have to do with the Python persistent implementation, especially PickleCache

.gitignore

@@ -17,3 +17,4 @@ coverage.xml
 dist
 testing.log
 .eggs/
+.dir-locals.el

src/ZODB/Connection.py

@@ -328,10 +328,10 @@ def close(self, primary=True):
                 # get back here.
         else:
             self.opened = None
-            
+
         am = self._db._activity_monitor
         if am is not None:
-            am.closedConnection(self)        
+            am.closedConnection(self)
 
     def db(self):
         """Returns a handle to the database this connection belongs to."""
@@ -1020,9 +1020,10 @@ def _cache_items(self):
         items = self._cache.lru_items()
         # fine everything. some on the lru list, some not
         everything = self._cache.cache_data
-        # remove those items that are on the lru list
+        # remove those items that are on the lru list (which may not actually
+        # be in the full cache, under the Python implementation)
         for k,v in items:
-            del everything[k]
+            everything.pop(k, None)
         # return a list of [ghosts....not recently used.....recently used]
         return list(everything.items()) + items
 

src/ZODB/DB.py

@@ -530,7 +530,13 @@ def f(con, detail=detail):
     def cacheExtremeDetail(self):
         detail = []
         conn_no = [0]  # A mutable reference to a counter
-        def f(con, detail=detail, rc=sys.getrefcount, conn_no=conn_no):
+        try:
+            rc = sys.getrefcount
+        except AttributeError:
+            # sys.getrefcount is a CPython implementation detail
+            # not required to exist on, e.g., PyPy. Here we fake it.
+            rc = lambda o: 4
+        def f(con, detail=detail, rc=rc, conn_no=conn_no):
             conn_no[0] += 1
             cn = conn_no[0]
             for oid, ob in con._cache_items():

src/ZODB/broken.py

@@ -23,6 +23,14 @@
 from ZODB._compat import IMPORT_MAPPING
 from ZODB._compat import NAME_MAPPING
 
+# Add our magic names to the Python implementation of persistence
+# so that it won't try to re-activate the broken object.
+import persistent.persistence
+persistent.persistence.SPECIAL_NAMES += ('__Broken_newargs__',
+                                         '__Broken_initargs__',
+                                         #'__Broken_state__',
+                                         '__reduce__',
+                                         '__getstate__')
 
 broken_cache = {}
 
@@ -308,7 +316,7 @@ class PersistentBroken(Broken, persistent.Persistent):
           >>> a.__reduce__()    # doctest: +NORMALIZE_WHITESPACE
           Traceback (most recent call last):
           ...
-          BrokenModified: 
+          BrokenModified:
           <persistent broken not.there.Atall instance '\x00\x00\x00\x00****'>
 
         but you can get their state:

src/ZODB/serialize.py

@@ -483,6 +483,10 @@ def find_global(modulename, name):
             return factory(conn, modulename, name)
 
         unpickler.find_global = find_global
+        try:
+            unpickler.find_class = find_global # PyPy, zodbpickle, the non-c-accelerated version
+        except AttributeError:
+            pass
 
         return unpickler
 

src/ZODB/tests/testBroken.py

@@ -67,7 +67,7 @@ def test_integration():
     >>> conn3 = db.open()
     >>> a3 = conn3.root()['a']
     >>> a3  # doctest: +NORMALIZE_WHITESPACE
-    <persistent broken ZODB.not.there.Atall instance 
+    <persistent broken ZODB.not.there.Atall instance
         '\x00\x00\x00\x00\x00\x00\x00\x01'>
 
     >>> a3.__Broken_state__

src/ZODB/tests/testCache.py

@@ -32,6 +32,7 @@
 import ZODB
 import ZODB.MappingStorage
 import ZODB.tests.util
+from ZODB.tests.util import PYPY
 
 PY2 = sys.version_info[0] == 2
 
@@ -190,6 +191,7 @@ def testClassItems(self):
 
 class LRUCacheTests(CacheTestBase):
 
+    @unittest.skipIf(PYPY, "Implementation details of the PickleCache")
     def testLRU(self):
         # verify the LRU behavior of the cache
         dataset_size = 5
@@ -339,7 +341,10 @@ def testBogusObject(self):
         def add(key, obj):
             self.cache[key] = obj
 
-        nones = sys.getrefcount(None)
+        # getrefcount is an implementation detail of CPython,
+        # not present under PyPy/Jython
+        rc = getattr(sys, 'getrefcount', lambda x: 1)
+        nones = rc(None)
 
         key = p64(2)
         # value isn't persistent
@@ -369,7 +374,7 @@ def add(key, obj):
             # structure that adds a new reference to None for each executed
             # line of code, which interferes with this test.  So check it
             # only if we're running without coverage tracing.
-            self.assertEqual(sys.getrefcount(None), nones)
+            self.assertEqual(rc(None), nones)
 
     def testTwoCaches(self):
         jar2 = StubDataManager()

src/ZODB/tests/util.py

@@ -18,6 +18,7 @@
 import atexit
 import os
 import persistent
+import platform
 import re
 import sys
 import tempfile
@@ -184,3 +185,17 @@ def faux_time():
 
     time.time = faux_time
 
+
+py_impl = getattr(platform, 'python_implementation', lambda: None)
+PYPY = py_impl() == 'PyPy'
+
+if not hasattr(unittest, 'skipIf'):
+    def skipIf(condition, reason):
+        if condition:
+            def skip(f):
+                return lambda self: None
+            return skip
+
+        def run(f):
+            return f
+        return run