Extract most compat code into ZODB._compat

src/ZODB/BaseStorage.py

@@ -25,27 +25,13 @@
 from struct import pack as _structpack, unpack as _structunpack
 
 import zope.interface
-
 from persistent.TimeStamp import TimeStamp
 
 import ZODB.interfaces
 from ZODB import POSException
 from ZODB.utils import z64, oid_repr, byte_ord, byte_chr
 from ZODB.UndoLogCompatible import UndoLogCompatible
-
-try:
-    import cPickle as pickle
-except ImportError:
-    # Py3
-    import pickle
-
-# Py3: Python 3's `hasattr()` only swallows AttributeError.
-def py2_hasattr(obj, name):
-    try:
-        getattr(obj, name)
-    except:
-        return False
-    return True
+from ZODB._compat import pickle, py2_hasattr
 
 log = logging.getLogger("ZODB.BaseStorage")
 

src/ZODB/ConflictResolution.py

@@ -14,25 +14,18 @@
 
 import logging
 import sys
-from pickle import PicklingError
 
 import six
 import zope.interface
 from ZODB.POSException import ConflictError
 from ZODB.loglevels import BLATHER
 from ZODB.serialize import _protocol, _Unpickler
+from ZODB._compat import BytesIO, pickle
 
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    # Py3
-    from io import BytesIO
-
-try:
-    from cPickle import Pickler
-except ImportError:
-    # Py3
-    from pickle import Pickler
+# Subtle: Python 2.x has pickle.PicklingError and cPickle.PicklingError,
+# and these are unrelated classes!  So we shouldn't use pickle.PicklingError,
+# since on Python 2, ZODB._compat.pickle is cPickle.
+from pickle import PicklingError
 
 
 logger = logging.getLogger('ZODB.ConflictResolution')
@@ -290,7 +283,7 @@ def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
         resolved = resolve(old, committed, newstate)
 
         file = BytesIO()
-        pickler = Pickler(file, _protocol)
+        pickler = pickle.Pickler(file, _protocol)
         if sys.version_info[0] < 3:
             pickler.inst_persistent_id = persistent_id
         else:

src/ZODB/DB.py

@@ -23,6 +23,7 @@
 from ZODB.broken import find_global
 from ZODB.utils import z64
 from ZODB.Connection import Connection
+from ZODB._compat import pickle, BytesIO
 import ZODB.serialize
 
 import transaction.weakset
@@ -36,17 +37,6 @@
 from persistent.TimeStamp import TimeStamp
 import six
 
-try:
-    import cPickle as pickle
-except ImportError:
-    # Py3
-    import pickle
-
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    # Py3
-    from io import BytesIO
 
 logger = logging.getLogger('ZODB.DB')
 

src/ZODB/ExportImport.py

@@ -12,30 +12,21 @@
 #
 ##############################################################################
 """Support for database export and import."""
-import os
 
-from tempfile import TemporaryFile
 import logging
-import six
+import os
 import sys
+from tempfile import TemporaryFile
+
+import six
 
 from ZODB.blob import Blob
 from ZODB.interfaces import IBlobStorage
 from ZODB.POSException import ExportError
 from ZODB.serialize import referencesf, _protocol
 from ZODB.utils import p64, u64, cp, mktemp
+from ZODB._compat import pickle, BytesIO
 
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    # Py3
-    from io import BytesIO
-
-try:
-    from cPickle import Unpickler, Pickler
-except ImportError:
-    # Py3
-    from pickle import Unpickler, Pickler
 
 logger = logging.getLogger('ZODB.ExportImport')
 
@@ -176,11 +167,11 @@ def persistent_load(ooid):
                 blob_filename = None
 
             pfile = BytesIO(data)
-            unpickler = Unpickler(pfile)
+            unpickler = pickle.Unpickler(pfile)
             unpickler.persistent_load = persistent_load
 
             newp = BytesIO()
-            pickler = Pickler(newp, _protocol)
+            pickler = pickle.Pickler(newp, _protocol)
             if sys.version_info[0] < 3:
                 pickler.inst_persistent_id = persistent_id
             else:

src/ZODB/FileStorage/FileStorage.py

@@ -15,9 +15,23 @@
 """
 from __future__ import print_function
 
-from persistent.TimeStamp import TimeStamp
+import binascii
+import contextlib
+import errno
+import logging
+import os
+import threading
+import time
 from struct import pack, unpack
+
+import six
+import zope.interface
+from persistent.TimeStamp import TimeStamp
 from zc.lockfile import LockFile
+
+import ZODB.blob
+import ZODB.interfaces
+import ZODB.utils
 from ZODB.FileStorage.format import CorruptedError, CorruptedDataError
 from ZODB.FileStorage.format import FileStorageFormatter, DataHeader
 from ZODB.FileStorage.format import TRANS_HDR, TRANS_HDR_LEN
@@ -27,31 +41,7 @@
 from ZODB import BaseStorage, ConflictResolution, POSException
 from ZODB.POSException import UndoError, POSKeyError, MultipleUndoErrors
 from ZODB.utils import p64, u64, z64, as_bytes, as_text
-
-import binascii
-import contextlib
-import errno
-import logging
-import os
-import six
-import threading
-import time
-import ZODB.blob
-import ZODB.interfaces
-import zope.interface
-import ZODB.utils
-
-try:
-    from cPickle import Pickler, loads
-except ImportError:
-    # Py3
-    from pickle import Pickler, loads
-
-try:
-    # Py3
-    from base64 import decodebytes, encodebytes
-except ImportError:
-    from base64 import decodestring as decodebytes, encodestring as encodebytes
+from ZODB._compat import pickle, decodebytes, encodebytes
 
 
 # Not all platforms have fsync
@@ -379,7 +369,7 @@ def _restore_index(self):
             if not self._is_read_only:
                 # Save the converted index.
                 f = open(index_name, 'wb')
-                p = Pickler(f, 1)
+                p = pickle.Pickler(f, 1)
                 info['index'] = index
                 p.dump(info)
                 f.close()
@@ -1011,7 +1001,7 @@ def history(self, oid, size=1, filter=None):
 
                 th = self._read_txn_header(h.tloc)
                 if th.ext:
-                    d = loads(th.ext)
+                    d = pickle.loads(th.ext)
                 else:
                     d = {}
 
@@ -1852,7 +1842,7 @@ def __next__(self):
                 e = {}
                 if h.elen:
                     try:
-                        e = loads(h.ext)
+                        e = pickle.loads(h.ext)
                     except:
                         pass
 
@@ -1994,7 +1984,7 @@ def _readnext(self):
         e = {}
         if el:
             try:
-                e = loads(self.file.read(el))
+                e = pickle.loads(self.file.read(el))
             except:
                 pass
         d = {'id': encodebytes(tid).rstrip(),

src/ZODB/_compat.py

@@ -0,0 +1,65 @@
+##############################################################################
+#
+# Copyright (c) 2013 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+
+try:
+    # Python 2.x
+    import cPickle as pickle
+    IMPORT_MAPPING = {}
+    NAME_MAPPING = {}
+except ImportError:
+    # Python 3.x: can't use stdlib's pickle because
+    # http://bugs.python.org/issue6784
+    ## import zodbpickle as pickle
+    import pickle
+    from _compat_pickle import IMPORT_MAPPING, NAME_MAPPING
+
+
+# XXX: overridable Unpickler.find_global as used in serialize.py?
+# XXX: consistent spelling of inst_persistent_id/persistent_id?
+#      e.g. StorageTestBase and probably elsewhere
+
+
+try:
+    # Python 2.x
+    # XXX: why not just import BytesIO from io?
+    from cStringIO import StringIO as BytesIO
+except ImportError:
+    # Python 3.x
+    from io import BytesIO
+
+
+try:
+    # Python 3.x
+    from base64 import decodebytes, encodebytes
+except ImportError:
+    # Python 2.x
+    from base64 import decodestring as decodebytes, encodestring as encodebytes
+
+
+# Python 3.x: ``hasattr()`` swallows only AttributeError.
+def py2_hasattr(obj, name):
+    try:
+        getattr(obj, name)
+    except:
+        return False
+    return True
+
+
+try:
+    # Py2: simply reexport the builtin
+    long = long
+except NameError:
+    # Py3
+    long = int
+

src/ZODB/blob.py

@@ -26,28 +26,19 @@
 import weakref
 
 import zope.interface
+import persistent
 
 import ZODB.interfaces
 from ZODB.interfaces import BlobError
 from ZODB import utils, serialize
 from ZODB.POSException import POSKeyError
-import persistent
+from ZODB._compat import BytesIO
 
-try:
-    import cPickle
-except ImportError:
-    # Py3
-    import pickle as cPickle
-
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    # Py3
-    from io import BytesIO
 
 if sys.version_info[0] >= 3:
     from io import FileIO as file
 
+
 logger = logging.getLogger('ZODB.blob')
 
 BLOB_SUFFIX = ".blob"

src/ZODB/broken.py

@@ -15,18 +15,13 @@
 """
 
 import sys
-import persistent
 
+import persistent
 import zope.interface
 
 import ZODB.interfaces
+from ZODB._compat import pickle, IMPORT_MAPPING, NAME_MAPPING
 
-try:
-    # Python 3
-    import _compat_pickle
-except ImportError:
-    # Python 2
-    _compat_pickle = None
 
 broken_cache = {}
 
@@ -87,12 +82,7 @@ class Broken(object):
          >>> r[2]
          {'x': 1}
 
-         >>> try:
-         ...     import cPickle
-         ... except ImportError:
-         ...     # Py3
-         ...     import pickle as cPickle
-         >>> a2 = cPickle.loads(cPickle.dumps(a, 1))
+         >>> a2 = pickle.loads(pickle.dumps(a, 1))
          >>> a2
          <broken not.there.Atall instance>
          >>> a2.__Broken_newargs__
@@ -196,12 +186,10 @@ def find_global(modulename, globalname,
          >>> broken_cache.clear()
        """
 
-    if _compat_pickle is not None:
-        if (modulename, globalname) in _compat_pickle.NAME_MAPPING:
-            modulename, globalname = _compat_pickle.NAME_MAPPING[
-                                            (modulename, globalname)]
-        if modulename in _compat_pickle.IMPORT_MAPPING:
-            modulename = _compat_pickle.IMPORT_MAPPING[modulename]
+    if (modulename, globalname) in NAME_MAPPING:
+        modulename, globalname = NAME_MAPPING[(modulename, globalname)]
+    if modulename in IMPORT_MAPPING:
+        modulename = IMPORT_MAPPING[modulename]
 
     # short circuit common case:
     try: