diff --git a/src/ZODB/BaseStorage.py b/src/ZODB/BaseStorage.py index bce71292f..042e10b0a 100644 --- a/src/ZODB/BaseStorage.py +++ b/src/ZODB/BaseStorage.py @@ -25,27 +25,13 @@ from struct import pack as _structpack, unpack as _structunpack import zope.interface - from persistent.TimeStamp import TimeStamp import ZODB.interfaces from ZODB import POSException from ZODB.utils import z64, oid_repr, byte_ord, byte_chr from ZODB.UndoLogCompatible import UndoLogCompatible - -try: - import cPickle as pickle -except ImportError: - # Py3 - import pickle - -# Py3: Python 3's `hasattr()` only swallows AttributeError. -def py2_hasattr(obj, name): - try: - getattr(obj, name) - except: - return False - return True +from ZODB._compat import pickle, py2_hasattr log = logging.getLogger("ZODB.BaseStorage") diff --git a/src/ZODB/ConflictResolution.py b/src/ZODB/ConflictResolution.py index 2964350ce..ccd2db07b 100644 --- a/src/ZODB/ConflictResolution.py +++ b/src/ZODB/ConflictResolution.py @@ -14,25 +14,18 @@ import logging import sys -from pickle import PicklingError import six import zope.interface from ZODB.POSException import ConflictError from ZODB.loglevels import BLATHER from ZODB.serialize import _protocol, _Unpickler +from ZODB._compat import BytesIO, pickle -try: - from cStringIO import StringIO as BytesIO -except ImportError: - # Py3 - from io import BytesIO - -try: - from cPickle import Pickler -except ImportError: - # Py3 - from pickle import Pickler +# Subtle: Python 2.x has pickle.PicklingError and cPickle.PicklingError, +# and these are unrelated classes! So we shouldn't use pickle.PicklingError, +# since on Python 2, ZODB._compat.pickle is cPickle. +from pickle import PicklingError logger = logging.getLogger('ZODB.ConflictResolution') @@ -290,7 +283,7 @@ def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle, resolved = resolve(old, committed, newstate) file = BytesIO() - pickler = Pickler(file, _protocol) + pickler = pickle.Pickler(file, _protocol) if sys.version_info[0] < 3: pickler.inst_persistent_id = persistent_id else: diff --git a/src/ZODB/DB.py b/src/ZODB/DB.py index 405c1399d..9954f613e 100644 --- a/src/ZODB/DB.py +++ b/src/ZODB/DB.py @@ -23,6 +23,7 @@ from ZODB.broken import find_global from ZODB.utils import z64 from ZODB.Connection import Connection +from ZODB._compat import pickle, BytesIO import ZODB.serialize import transaction.weakset @@ -36,17 +37,6 @@ from persistent.TimeStamp import TimeStamp import six -try: - import cPickle as pickle -except ImportError: - # Py3 - import pickle - -try: - from cStringIO import StringIO as BytesIO -except ImportError: - # Py3 - from io import BytesIO logger = logging.getLogger('ZODB.DB') diff --git a/src/ZODB/ExportImport.py b/src/ZODB/ExportImport.py index f2442af84..5463d79f2 100644 --- a/src/ZODB/ExportImport.py +++ b/src/ZODB/ExportImport.py @@ -12,30 +12,21 @@ # ############################################################################## """Support for database export and import.""" -import os -from tempfile import TemporaryFile import logging -import six +import os import sys +from tempfile import TemporaryFile + +import six from ZODB.blob import Blob from ZODB.interfaces import IBlobStorage from ZODB.POSException import ExportError from ZODB.serialize import referencesf, _protocol from ZODB.utils import p64, u64, cp, mktemp +from ZODB._compat import pickle, BytesIO -try: - from cStringIO import StringIO as BytesIO -except ImportError: - # Py3 - from io import BytesIO - -try: - from cPickle import Unpickler, Pickler -except ImportError: - # Py3 - from pickle import Unpickler, Pickler logger = logging.getLogger('ZODB.ExportImport') @@ -176,11 +167,11 @@ def persistent_load(ooid): blob_filename = None pfile = BytesIO(data) - unpickler = Unpickler(pfile) + unpickler = pickle.Unpickler(pfile) unpickler.persistent_load = persistent_load newp = BytesIO() - pickler = Pickler(newp, _protocol) + pickler = pickle.Pickler(newp, _protocol) if sys.version_info[0] < 3: pickler.inst_persistent_id = persistent_id else: diff --git a/src/ZODB/FileStorage/FileStorage.py b/src/ZODB/FileStorage/FileStorage.py index e40a20879..017505ff2 100644 --- a/src/ZODB/FileStorage/FileStorage.py +++ b/src/ZODB/FileStorage/FileStorage.py @@ -15,9 +15,23 @@ """ from __future__ import print_function -from persistent.TimeStamp import TimeStamp +import binascii +import contextlib +import errno +import logging +import os +import threading +import time from struct import pack, unpack + +import six +import zope.interface +from persistent.TimeStamp import TimeStamp from zc.lockfile import LockFile + +import ZODB.blob +import ZODB.interfaces +import ZODB.utils from ZODB.FileStorage.format import CorruptedError, CorruptedDataError from ZODB.FileStorage.format import FileStorageFormatter, DataHeader from ZODB.FileStorage.format import TRANS_HDR, TRANS_HDR_LEN @@ -27,31 +41,7 @@ from ZODB import BaseStorage, ConflictResolution, POSException from ZODB.POSException import UndoError, POSKeyError, MultipleUndoErrors from ZODB.utils import p64, u64, z64, as_bytes, as_text - -import binascii -import contextlib -import errno -import logging -import os -import six -import threading -import time -import ZODB.blob -import ZODB.interfaces -import zope.interface -import ZODB.utils - -try: - from cPickle import Pickler, loads -except ImportError: - # Py3 - from pickle import Pickler, loads - -try: - # Py3 - from base64 import decodebytes, encodebytes -except ImportError: - from base64 import decodestring as decodebytes, encodestring as encodebytes +from ZODB._compat import pickle, decodebytes, encodebytes # Not all platforms have fsync @@ -379,7 +369,7 @@ def _restore_index(self): if not self._is_read_only: # Save the converted index. f = open(index_name, 'wb') - p = Pickler(f, 1) + p = pickle.Pickler(f, 1) info['index'] = index p.dump(info) f.close() @@ -1011,7 +1001,7 @@ def history(self, oid, size=1, filter=None): th = self._read_txn_header(h.tloc) if th.ext: - d = loads(th.ext) + d = pickle.loads(th.ext) else: d = {} @@ -1852,7 +1842,7 @@ def __next__(self): e = {} if h.elen: try: - e = loads(h.ext) + e = pickle.loads(h.ext) except: pass @@ -1994,7 +1984,7 @@ def _readnext(self): e = {} if el: try: - e = loads(self.file.read(el)) + e = pickle.loads(self.file.read(el)) except: pass d = {'id': encodebytes(tid).rstrip(), diff --git a/src/ZODB/_compat.py b/src/ZODB/_compat.py new file mode 100644 index 000000000..dbc4cddc5 --- /dev/null +++ b/src/ZODB/_compat.py @@ -0,0 +1,65 @@ +############################################################################## +# +# Copyright (c) 2013 Zope Foundation and Contributors. +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE +# +############################################################################## + +try: + # Python 2.x + import cPickle as pickle + IMPORT_MAPPING = {} + NAME_MAPPING = {} +except ImportError: + # Python 3.x: can't use stdlib's pickle because + # http://bugs.python.org/issue6784 + ## import zodbpickle as pickle + import pickle + from _compat_pickle import IMPORT_MAPPING, NAME_MAPPING + + +# XXX: overridable Unpickler.find_global as used in serialize.py? +# XXX: consistent spelling of inst_persistent_id/persistent_id? +# e.g. StorageTestBase and probably elsewhere + + +try: + # Python 2.x + # XXX: why not just import BytesIO from io? + from cStringIO import StringIO as BytesIO +except ImportError: + # Python 3.x + from io import BytesIO + + +try: + # Python 3.x + from base64 import decodebytes, encodebytes +except ImportError: + # Python 2.x + from base64 import decodestring as decodebytes, encodestring as encodebytes + + +# Python 3.x: ``hasattr()`` swallows only AttributeError. +def py2_hasattr(obj, name): + try: + getattr(obj, name) + except: + return False + return True + + +try: + # Py2: simply reexport the builtin + long = long +except NameError: + # Py3 + long = int + diff --git a/src/ZODB/blob.py b/src/ZODB/blob.py index 8881e9dce..847d8b035 100644 --- a/src/ZODB/blob.py +++ b/src/ZODB/blob.py @@ -26,28 +26,19 @@ import weakref import zope.interface +import persistent import ZODB.interfaces from ZODB.interfaces import BlobError from ZODB import utils, serialize from ZODB.POSException import POSKeyError -import persistent +from ZODB._compat import BytesIO -try: - import cPickle -except ImportError: - # Py3 - import pickle as cPickle - -try: - from cStringIO import StringIO as BytesIO -except ImportError: - # Py3 - from io import BytesIO if sys.version_info[0] >= 3: from io import FileIO as file + logger = logging.getLogger('ZODB.blob') BLOB_SUFFIX = ".blob" diff --git a/src/ZODB/broken.py b/src/ZODB/broken.py index b0b4a2ee3..9ed7aae7c 100644 --- a/src/ZODB/broken.py +++ b/src/ZODB/broken.py @@ -15,18 +15,13 @@ """ import sys -import persistent +import persistent import zope.interface import ZODB.interfaces +from ZODB._compat import pickle, IMPORT_MAPPING, NAME_MAPPING -try: - # Python 3 - import _compat_pickle -except ImportError: - # Python 2 - _compat_pickle = None broken_cache = {} @@ -87,12 +82,7 @@ class Broken(object): >>> r[2] {'x': 1} - >>> try: - ... import cPickle - ... except ImportError: - ... # Py3 - ... import pickle as cPickle - >>> a2 = cPickle.loads(cPickle.dumps(a, 1)) + >>> a2 = pickle.loads(pickle.dumps(a, 1)) >>> a2 >>> a2.__Broken_newargs__ @@ -196,12 +186,10 @@ def find_global(modulename, globalname, >>> broken_cache.clear() """ - if _compat_pickle is not None: - if (modulename, globalname) in _compat_pickle.NAME_MAPPING: - modulename, globalname = _compat_pickle.NAME_MAPPING[ - (modulename, globalname)] - if modulename in _compat_pickle.IMPORT_MAPPING: - modulename = _compat_pickle.IMPORT_MAPPING[modulename] + if (modulename, globalname) in NAME_MAPPING: + modulename, globalname = NAME_MAPPING[(modulename, globalname)] + if modulename in IMPORT_MAPPING: + modulename = IMPORT_MAPPING[modulename] # short circuit common case: try: diff --git a/src/ZODB/fsIndex.py b/src/ZODB/fsIndex.py index ba8854c8f..4c0f97675 100644 --- a/src/ZODB/fsIndex.py +++ b/src/ZODB/fsIndex.py @@ -44,11 +44,7 @@ from BTrees.OOBTree import OOBTree import six -try: - import cPickle as pickle -except ImportError: - # Py3 - import pickle +from ZODB._compat import pickle # convert between numbers and six-byte strings diff --git a/src/ZODB/fsrecover.py b/src/ZODB/fsrecover.py index 438d1f3ed..bdcd9ded1 100644 --- a/src/ZODB/fsrecover.py +++ b/src/ZODB/fsrecover.py @@ -71,12 +71,6 @@ import time from struct import unpack -try: - from cPickle import loads -except ImportError: - # Py3 - from pickle import loads - try: import ZODB except ImportError: @@ -89,6 +83,7 @@ import ZODB.FileStorage from ZODB.utils import u64, as_text from ZODB.FileStorage import TransactionRecord +from ZODB._compat import pickle from persistent.TimeStamp import TimeStamp @@ -149,9 +144,9 @@ def read_txn_header(f, pos, file_size, outp, ltid): user = f.read(ul) description = f.read(dl) if el: - try: e=loads(f.read(el)) - except: e={} - else: e={} + try: e = pickle.loads(f.read(el)) + except: e = {} + else: e = {} result = TransactionRecord(tid, status, user, description, e, pos, tend, f, tpos) diff --git a/src/ZODB/fstools.py b/src/ZODB/fstools.py index ffc068859..1ba7837d2 100644 --- a/src/ZODB/fstools.py +++ b/src/ZODB/fstools.py @@ -23,14 +23,9 @@ from ZODB.FileStorage.format import TRANS_HDR, DATA_HDR, TRANS_HDR_LEN from ZODB.FileStorage.format import DATA_HDR_LEN from ZODB.utils import u64 +from ZODB._compat import pickle from persistent.TimeStamp import TimeStamp -try: - import cPickle -except ImportError: - # Py3 - import pickle as cPickle - class TxnHeader: """Object representing a transaction record header. @@ -70,7 +65,7 @@ def read_meta(self): self.descr = self._file.read(self.descr_len) if self.ext_len: self._ext = self._file.read(self.ext_len) - self.ext = cPickle.loads(self._ext) + self.ext = pickle.loads(self._ext) def get_data_offset(self): return (self._pos + TRANS_HDR_LEN + self.user_len + self.descr_len diff --git a/src/ZODB/scripts/analyze.py b/src/ZODB/scripts/analyze.py index d1fba2225..cc64d0465 100755 --- a/src/ZODB/scripts/analyze.py +++ b/src/ZODB/scripts/analyze.py @@ -2,15 +2,16 @@ # Based on a transaction analyzer by Matt Kromer. from __future__ import print_function -import pickle + import sys + from ZODB.FileStorage import FileStorage +from ZODB._compat import BytesIO + +# We must not use cPickle on Python 2: cPickle.Unpickler cannot be +# subclassed. +import pickle -try: - from cStringIO import StringIO as BytesIO -except ImportError: - # Py3 - from io import BytesIO class FakeError(Exception): def __init__(self, module, name): diff --git a/src/ZODB/serialize.py b/src/ZODB/serialize.py index 4bad83426..807b30032 100644 --- a/src/ZODB/serialize.py +++ b/src/ZODB/serialize.py @@ -140,18 +140,7 @@ class of an object, a new record with new class metadata would be from persistent.wref import WeakRefMarker, WeakRef from ZODB import broken from ZODB.POSException import InvalidObjectReference - -try: - import cPickle as pickle -except ImportError: - # Py3 - import pickle - -try: - from cStringIO import StringIO as BytesIO -except ImportError: - # Py3 - from io import BytesIO +from ZODB._compat import pickle, BytesIO if sys.version_info[0] < 3: _Unpickler = pickle.Unpickler diff --git a/src/ZODB/tests/PackableStorage.py b/src/ZODB/tests/PackableStorage.py index dd4f324a2..df4a80d0a 100644 --- a/src/ZODB/tests/PackableStorage.py +++ b/src/ZODB/tests/PackableStorage.py @@ -14,6 +14,10 @@ """Run some tests relevant for storages that support pack().""" from __future__ import print_function +import doctest +import sys +import time + from persistent import Persistent from persistent.mapping import PersistentMapping from ZODB import DB @@ -22,25 +26,12 @@ from ZODB.tests.MinPO import MinPO from ZODB.tests.MTStorage import TestThread from ZODB.tests.StorageTestBase import snooze -import doctest -import sys -import time +from ZODB._compat import pickle, BytesIO import transaction import ZODB.interfaces import ZODB.tests.util import zope.testing.setupstack -try: - import cPickle as pickle -except ImportError: - # Py3 - import pickle - -try: - from cStringIO import StringIO as BytesIO -except ImportError: - # Py3 - from io import BytesIO ZERO = b'\0'*8 diff --git a/src/ZODB/tests/StorageTestBase.py b/src/ZODB/tests/StorageTestBase.py index 3b28efe66..bfb29fae8 100644 --- a/src/ZODB/tests/StorageTestBase.py +++ b/src/ZODB/tests/StorageTestBase.py @@ -25,20 +25,9 @@ from ZODB.utils import u64 from ZODB.tests.MinPO import MinPO +from ZODB._compat import pickle, BytesIO import ZODB.tests.util -try: - from cPickle import Pickler, Unpickler -except ImportError: - # Py3 - from pickle import Pickler, Unpickler - -try: - from cStringIO import StringIO as BytesIO -except ImportError: - # Py3 - from io import BytesIO - ZERO = b'\0'*8 @@ -61,7 +50,7 @@ def _persistent_id(obj): def zodb_pickle(obj): """Create a pickle in the format expected by ZODB.""" f = BytesIO() - p = Pickler(f, 1) + p = pickle.Pickler(f, 1) if sys.version_info[0] < 3: p.inst_persistent_id = _persistent_id else: @@ -87,7 +76,7 @@ def persistent_load(pid): def zodb_unpickle(data): """Unpickle an object stored using the format expected by ZODB.""" f = BytesIO(data) - u = Unpickler(f) + u = pickle.Unpickler(f) u.persistent_load = persistent_load klass_info = u.load() if isinstance(klass_info, tuple): diff --git a/src/ZODB/tests/testFileStorage.py b/src/ZODB/tests/testFileStorage.py index a3ac8a12b..5c8159e90 100644 --- a/src/ZODB/tests/testFileStorage.py +++ b/src/ZODB/tests/testFileStorage.py @@ -32,13 +32,7 @@ from ZODB.tests import RevisionStorage, PersistentStorage, MTStorage from ZODB.tests import ReadOnlyStorage, RecoveryStorage from ZODB.tests.StorageTestBase import MinPO, zodb_pickle - - -try: - import cPickle -except ImportError: - # Py3 - import pickle as cPickle +from ZODB._compat import pickle class FileStorageTests( @@ -97,7 +91,7 @@ def convert_index_to_dict(self): newindex = dict(index) data['index'] = newindex - cPickle.dump(data, open('FileStorageTests.fs.index', 'wb'), 1) + pickle.dump(data, open('FileStorageTests.fs.index', 'wb'), 1) return index def check_conversion_to_fsIndex(self, read_only=False): diff --git a/src/ZODB/tests/testPersistentMapping.py b/src/ZODB/tests/testPersistentMapping.py index a6dce4c10..9e7c53606 100644 --- a/src/ZODB/tests/testPersistentMapping.py +++ b/src/ZODB/tests/testPersistentMapping.py @@ -21,18 +21,14 @@ """ import unittest +import sys import transaction from transaction import Transaction + import ZODB from ZODB.MappingStorage import MappingStorage -import sys - -try: - import cPickle -except ImportError: - # Py3 - import pickle as cPickle +from ZODB._compat import pickle try: import cStringIO @@ -89,8 +85,9 @@ def TODO_checkNewPicklesAreSafe(self): # MappingStorage stores serialno + pickle in its _index. root_pickle = s._index['\000' * 8][8:] + # XXX not BytesIO really? f = cStringIO.StringIO(root_pickle) - u = cPickle.Unpickler(f) + u = pickle.Unpickler(f) klass_info = u.load() klass = find_global(*klass_info[0]) inst = klass.__new__(klass) diff --git a/src/ZODB/tests/testSerialize.py b/src/ZODB/tests/testSerialize.py index b39d758ec..709202636 100644 --- a/src/ZODB/tests/testSerialize.py +++ b/src/ZODB/tests/testSerialize.py @@ -17,18 +17,7 @@ import ZODB.tests.util from ZODB import serialize - -try: - import cPickle as pickle -except ImportError: - # Py3 - import pickle - -try: - from cStringIO import StringIO as BytesIO -except ImportError: - # Py3 - from io import BytesIO +from ZODB._compat import pickle, BytesIO class ClassWithNewargs(int): diff --git a/src/ZODB/tests/testUtils.py b/src/ZODB/tests/testUtils.py index eba4bff26..984d57f2e 100644 --- a/src/ZODB/tests/testUtils.py +++ b/src/ZODB/tests/testUtils.py @@ -20,12 +20,8 @@ from zope.testing import renormalizing from ZODB.utils import U64, p64, u64 +from ZODB._compat import pickle, long -try: - long -except NameError: - # Py3 - long = int NUM = 100 @@ -77,11 +73,6 @@ def checkConflictErrorDoesntImport(self): from ZODB.serialize import ObjectWriter from ZODB.POSException import ConflictError from ZODB.tests.MinPO import MinPO - try: - import cPickle as pickle - except ImportError: - # Py3 - import pickle obj = MinPO() data = ObjectWriter().serialize(obj) diff --git a/src/ZODB/tests/testfsIndex.py b/src/ZODB/tests/testfsIndex.py index 5503f4d18..309768e86 100644 --- a/src/ZODB/tests/testfsIndex.py +++ b/src/ZODB/tests/testfsIndex.py @@ -220,12 +220,8 @@ def fsIndex_save_and_load(): If we save the data in the old format, we can still read it: - >>> try: - ... import cPickle - ... except ImportError: - ... # Py3 - ... import pickle as cPickle - >>> cPickle.dump(dict(pos=42, index=index), open('old', 'wb'), 1) + >>> from ZODB._compat import pickle + >>> pickle.dump(dict(pos=42, index=index), open('old', 'wb'), 1) >>> info = fsIndex.load('old') >>> info['pos'] 42 diff --git a/src/ZODB/utils.py b/src/ZODB/utils.py index 15ff0ca5f..076dd39e3 100644 --- a/src/ZODB/utils.py +++ b/src/ZODB/utils.py @@ -11,29 +11,19 @@ # FOR A PARTICULAR PURPOSE # ############################################################################## +import os +import struct import sys import time -import struct -from struct import pack, unpack -from binascii import hexlify, unhexlify import warnings +from binascii import hexlify, unhexlify +from struct import pack, unpack from tempfile import mkstemp -import os - -try: - import cPickle as pickle -except ImportError: - # Py3 - import pickle -try: - from cStringIO import StringIO as BytesIO -except ImportError: - # Py3 - from io import BytesIO +from persistent.TimeStamp import TimeStamp +from ZODB._compat import pickle, BytesIO -from persistent.TimeStamp import TimeStamp __all__ = ['z64', 'p64',