Depend on zodbpickle on Python 3

Since now we need to pass an extra argument to loads() and Unpickler(),
but only on Python 3, the old way of ``from ZODB._compat import pickle``
no longer pays out, and I had to import the names directly.

setup.py

@@ -131,7 +131,7 @@ def read_file(*path):
         'zc.lockfile',
         'zdaemon >= 4.0.0a1',
         'zope.interface',
-        ],
+        ] + ['zodbpickle'] if PY3 else [],
       zip_safe = False,
       entry_points = """
       [console_scripts]

src/ZODB/BaseStorage.py

@@ -31,7 +31,7 @@
 from ZODB import POSException
 from ZODB.utils import z64, oid_repr, byte_ord, byte_chr
 from ZODB.UndoLogCompatible import UndoLogCompatible
-from ZODB._compat import pickle, py2_hasattr
+from ZODB._compat import dumps, py2_hasattr
 
 log = logging.getLogger("ZODB.BaseStorage")
 
@@ -235,7 +235,7 @@ def tpc_begin(self, transaction, tid=None, status=' '):
             desc = transaction.description
             ext = transaction._extension
             if ext:
-                ext = pickle.dumps(ext, 1)
+                ext = dumps(ext, 1)
             else:
                 ext = ""
 

src/ZODB/ConflictResolution.py

@@ -19,8 +19,8 @@
 import zope.interface
 from ZODB.POSException import ConflictError
 from ZODB.loglevels import BLATHER
-from ZODB.serialize import _protocol, _Unpickler
-from ZODB._compat import BytesIO, pickle
+from ZODB.serialize import _protocol
+from ZODB._compat import BytesIO, Unpickler, Pickler
 
 # Subtle: Python 2.x has pickle.PicklingError and cPickle.PicklingError,
 # and these are unrelated classes!  So we shouldn't use pickle.PicklingError,
@@ -75,7 +75,7 @@ def state(self, oid, serial, prfactory, p=''):
     p = p or self.loadSerial(oid, serial)
     p = self._crs_untransform_record_data(p)
     file = BytesIO(p)
-    unpickler = _Unpickler(file)
+    unpickler = Unpickler(file)
     unpickler.find_global = find_global
     unpickler.persistent_load = prfactory.persistent_load
     unpickler.load() # skip the class tuple
@@ -240,7 +240,7 @@ def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
         prfactory = PersistentReferenceFactory()
         newpickle = self._crs_untransform_record_data(newpickle)
         file = BytesIO(newpickle)
-        unpickler = _Unpickler(file)
+        unpickler = Unpickler(file)
         unpickler.find_global = find_global
         unpickler.persistent_load = prfactory.persistent_load
         meta = unpickler.load()
@@ -283,7 +283,7 @@ def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
         resolved = resolve(old, committed, newstate)
 
         file = BytesIO()
-        pickler = pickle.Pickler(file, _protocol)
+        pickler = Pickler(file, _protocol)
         if sys.version_info[0] < 3:
             pickler.inst_persistent_id = persistent_id
         else:

src/ZODB/DB.py

@@ -23,7 +23,7 @@
 from ZODB.broken import find_global
 from ZODB.utils import z64
 from ZODB.Connection import Connection
-from ZODB._compat import pickle, BytesIO
+from ZODB._compat import Pickler, BytesIO
 import ZODB.serialize
 
 import transaction.weakset
@@ -448,7 +448,7 @@ def __init__(self, storage,
                 # Manually create a pickle for the root to put in the storage.
                 # The pickle must be in the special ZODB format.
                 file = BytesIO()
-                p = pickle.Pickler(file, ZODB.serialize._protocol)
+                p = Pickler(file, ZODB.serialize._protocol)
                 p.dump((root.__class__, None))
                 p.dump(root.__getstate__())
                 t = transaction.Transaction()

src/ZODB/ExportImport.py

@@ -25,7 +25,7 @@
 from ZODB.POSException import ExportError
 from ZODB.serialize import referencesf, _protocol
 from ZODB.utils import p64, u64, cp, mktemp
-from ZODB._compat import pickle, BytesIO
+from ZODB._compat import Pickler, Unpickler, BytesIO
 
 
 logger = logging.getLogger('ZODB.ExportImport')
@@ -167,11 +167,11 @@ def persistent_load(ooid):
                 blob_filename = None
 
             pfile = BytesIO(data)
-            unpickler = pickle.Unpickler(pfile)
+            unpickler = Unpickler(pfile)
             unpickler.persistent_load = persistent_load
 
             newp = BytesIO()
-            pickler = pickle.Pickler(newp, _protocol)
+            pickler = Pickler(newp, _protocol)
             if sys.version_info[0] < 3:
                 pickler.inst_persistent_id = persistent_id
             else:

src/ZODB/FileStorage/FileStorage.py

@@ -41,7 +41,7 @@
 from ZODB import BaseStorage, ConflictResolution, POSException
 from ZODB.POSException import UndoError, POSKeyError, MultipleUndoErrors
 from ZODB.utils import p64, u64, z64, as_bytes, as_text
-from ZODB._compat import pickle, decodebytes, encodebytes
+from ZODB._compat import Pickler, loads, decodebytes, encodebytes
 
 
 # Not all platforms have fsync
@@ -369,7 +369,7 @@ def _restore_index(self):
             if not self._is_read_only:
                 # Save the converted index.
                 f = open(index_name, 'wb')
-                p = pickle.Pickler(f, 1)
+                p = Pickler(f, 1)
                 info['index'] = index
                 p.dump(info)
                 f.close()
@@ -1001,7 +1001,7 @@ def history(self, oid, size=1, filter=None):
 
                 th = self._read_txn_header(h.tloc)
                 if th.ext:
-                    d = pickle.loads(th.ext)
+                    d = loads(th.ext)
                 else:
                     d = {}
 
@@ -1842,7 +1842,7 @@ def __next__(self):
                 e = {}
                 if h.elen:
                     try:
-                        e = pickle.loads(h.ext)
+                        e = loads(h.ext)
                     except:
                         pass
 
@@ -1984,7 +1984,7 @@ def _readnext(self):
         e = {}
         if el:
             try:
-                e = pickle.loads(self.file.read(el))
+                e = loads(self.file.read(el))
             except:
                 pass
         d = {'id': encodebytes(tid).rstrip(),

src/ZODB/_compat.py

@@ -14,16 +14,33 @@
 
 try:
     # Python 2.x
-    import cPickle as pickle
+    from cPickle import Pickler, Unpickler, dump, dumps, loads
     IMPORT_MAPPING = {}
     NAME_MAPPING = {}
 except ImportError:
     # Python 3.x: can't use stdlib's pickle because
     # http://bugs.python.org/issue6784
-    ## import zodbpickle as pickle
-    import pickle
+    from zodbpickle.pickle import Pickler, dump, dumps
+    from zodbpickle.pickle import Unpickler as _Unpickler, loads as _loads
     from _compat_pickle import IMPORT_MAPPING, NAME_MAPPING
 
+    class Unpickler(_Unpickler):
+        def __init__(self, f):
+            super(Unpickler, self).__init__(f, encoding='ASCII', errors='bytes')
+
+        # Py3: Python 3 doesn't allow assignments to find_global,
+        # instead, find_class can be overridden
+
+        find_global = None
+
+        def find_class(self, modulename, name):
+            if self.find_global is None:
+                return super(Unpickler, self).find_class(modulename, name)
+            return self.find_global(modulename, name)
+
+    def loads(s):
+        return _loads(s, encoding='ASCII', errors='bytes')
+
 
 # XXX: overridable Unpickler.find_global as used in serialize.py?
 # XXX: consistent spelling of inst_persistent_id/persistent_id?

src/ZODB/blob.py

@@ -30,9 +30,9 @@
 
 import ZODB.interfaces
 from ZODB.interfaces import BlobError
-from ZODB import utils, serialize
+from ZODB import utils
 from ZODB.POSException import POSKeyError
-from ZODB._compat import BytesIO
+from ZODB._compat import BytesIO, Unpickler
 
 
 if sys.version_info[0] >= 3:
@@ -934,7 +934,7 @@ def is_blob_record(record):
 
     """
     if record and (b'ZODB.blob' in record):
-        unpickler = serialize._Unpickler(BytesIO(record))
+        unpickler = Unpickler(BytesIO(record))
         unpickler.find_global = find_global_Blob
 
         try:

src/ZODB/broken.py

@@ -20,7 +20,7 @@
 import zope.interface
 
 import ZODB.interfaces
-from ZODB._compat import pickle, IMPORT_MAPPING, NAME_MAPPING
+from ZODB._compat import loads, dumps, IMPORT_MAPPING, NAME_MAPPING
 
 
 broken_cache = {}
@@ -82,7 +82,7 @@ class Broken(object):
          >>> r[2]
          {'x': 1}
 
-         >>> a2 = pickle.loads(pickle.dumps(a, 1))
+         >>> a2 = loads(dumps(a, 1))
          >>> a2
          <broken not.there.Atall instance>
          >>> a2.__Broken_newargs__

src/ZODB/fsIndex.py

@@ -44,7 +44,7 @@
 from BTrees.OOBTree import OOBTree
 import six
 
-from ZODB._compat import pickle
+from ZODB._compat import Pickler, Unpickler
 
 
 # convert between numbers and six-byte strings
@@ -97,7 +97,7 @@ def __getitem__(self, key):
 
     def save(self, pos, fname):
         with open(fname, 'wb') as f:
-            pickler = pickle.Pickler(f, 1)
+            pickler = Pickler(f, 1)
             pickler.fast = True
             pickler.dump(pos)
             for k, v in six.iteritems(self._data):
@@ -107,7 +107,7 @@ def save(self, pos, fname):
     @classmethod
     def load(class_, fname):
         with open(fname, 'rb') as f:
-            unpickler = pickle.Unpickler(f)
+            unpickler = Unpickler(f)
             pos = unpickler.load()
             if not isinstance(pos, int):
                 return pos                  # Old format

src/ZODB/fsrecover.py

@@ -83,7 +83,7 @@
 import ZODB.FileStorage
 from ZODB.utils import u64, as_text
 from ZODB.FileStorage import TransactionRecord
-from ZODB._compat import pickle
+from ZODB._compat import loads
 
 from persistent.TimeStamp import TimeStamp
 
@@ -144,7 +144,7 @@ def read_txn_header(f, pos, file_size, outp, ltid):
     user = f.read(ul)
     description = f.read(dl)
     if el:
-        try: e = pickle.loads(f.read(el))
+        try: e = loads(f.read(el))
         except: e = {}
     else: e = {}
 

src/ZODB/fstools.py

@@ -23,7 +23,7 @@
 from ZODB.FileStorage.format import TRANS_HDR, DATA_HDR, TRANS_HDR_LEN
 from ZODB.FileStorage.format import DATA_HDR_LEN
 from ZODB.utils import u64
-from ZODB._compat import pickle
+from ZODB._compat import loads
 from persistent.TimeStamp import TimeStamp
 
 
@@ -65,7 +65,7 @@ def read_meta(self):
             self.descr = self._file.read(self.descr_len)
         if self.ext_len:
             self._ext = self._file.read(self.ext_len)
-            self.ext = pickle.loads(self._ext)
+            self.ext = loads(self._ext)
 
     def get_data_offset(self):
         return (self._pos + TRANS_HDR_LEN + self.user_len + self.descr_len

src/ZODB/scripts/analyze.py

@@ -6,11 +6,8 @@
 import sys
 
 from ZODB.FileStorage import FileStorage
-from ZODB._compat import BytesIO
+from ZODB._compat import Unpickler, BytesIO
 
-# We must not use cPickle on Python 2: cPickle.Unpickler cannot be
-# subclassed.
-import pickle
 
 
 class FakeError(Exception):
@@ -19,9 +16,16 @@ def __init__(self, module, name):
         self.module = module
         self.name = name
 
-class FakeUnpickler(pickle.Unpickler):
-    def find_class(self, module, name):
-        raise FakeError(module, name)
+
+def fake_find_class(module, name):
+    raise FakeError(module, name)
+
+
+def FakeUnpickler(f):
+    unpickler = Unpickler(f)
+    unpickler.find_global = fake_find_class
+    return unpickler
+
 
 class Report:
     def __init__(self):
@@ -105,8 +109,6 @@ def get_type(record):
         unpickled = FakeUnpickler(BytesIO(record.data)).load()
     except FakeError as err:
         return "%s.%s" % (err.module, err.name)
-    except:
-        raise
     classinfo = unpickled[0]
     if isinstance(classinfo, tuple):
         mod, klass = classinfo

src/ZODB/serialize.py

@@ -140,19 +140,8 @@ class of an object, a new record with new class metadata would be
 from persistent.wref import WeakRefMarker, WeakRef
 from ZODB import broken
 from ZODB.POSException import InvalidObjectReference
-from ZODB._compat import pickle, BytesIO
-
-if sys.version_info[0] < 3:
-    _Unpickler = pickle.Unpickler
-else:
-    # Py3: Python 3 doesn't allow assignments to find_global,
-    # instead, find_class can be overridden
-    class _Unpickler(pickle.Unpickler):
-        find_global = None
-        def find_class(self, modulename, name):
-            if self.find_global is None:
-                return super(_Unpickler, self).find_class(modulename, name)
-            return self.find_global(modulename, name)
+from ZODB._compat import Pickler, Unpickler, BytesIO
+
 
 _oidtypes = bytes, type(None)
 
@@ -186,7 +175,7 @@ class ObjectWriter:
 
     def __init__(self, obj=None):
         self._file = BytesIO()
-        self._p = pickle.Pickler(self._file, _protocol)
+        self._p = Pickler(self._file, _protocol)
         if sys.version_info[0] < 3:
             self._p.inst_persistent_id = self.persistent_id
         else:
@@ -483,7 +472,7 @@ def _get_class(self, module, name):
 
     def _get_unpickler(self, pickle):
         file = BytesIO(pickle)
-        unpickler = _Unpickler(file)
+        unpickler = Unpickler(file)
         unpickler.persistent_load = self._persistent_load
         factory = self._factory
         conn = self._conn
@@ -642,7 +631,7 @@ def referencesf(p, oids=None):
     """
 
     refs = []
-    u = pickle.Unpickler(BytesIO(p))
+    u = Unpickler(BytesIO(p))
     if sys.version_info[0] < 3:
         u.persistent_load = refs
         u.noload()
@@ -685,7 +674,7 @@ def get_refs(a_pickle):
     """
 
     refs = []
-    u = pickle.Unpickler(BytesIO(a_pickle))
+    u = Unpickler(BytesIO(a_pickle))
     if sys.version_info[0] < 3:
         u.persistent_load = refs
         u.noload()