diff --git a/CHANGES.rst b/CHANGES.rst index 05ba480..7750828 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,15 @@ ``zodbpickle`` Changelog ======================== +0.6.0 (unreleased) +------------------ + +- Add support for PyPy. + +- Restore the ``noload`` behaviour from Python 2.6 and provide the + ``noload`` method on the non-C-accelerated unpicklers under PyPy and + Python 2. + 0.5.2 (2013-08-17) ------------------ diff --git a/setup.py b/setup.py index a3a998f..65be9f8 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ setup( name='zodbpickle', - version='0.5.2', + version='0.6.0.dev0', description='Fork of Python 3 pickle module.', author='Python and Zope Foundation', author_email='zodb-dev@zope.org', diff --git a/src/zodbpickle/_pickle_27.c b/src/zodbpickle/_pickle_27.c index 19b3343..aa4fac7 100644 --- a/src/zodbpickle/_pickle_27.c +++ b/src/zodbpickle/_pickle_27.c @@ -5325,32 +5325,67 @@ noload_extension(Unpicklerobject *self, int nbytes) return 0; } +static int +do_noload_append(Unpicklerobject *self, Py_ssize_t x) +{ + PyObject *list = 0; + Py_ssize_t len; + + len=self->stack->length; + if (!( len >= x && x > 0 )) return stackUnderflow(); + /* nothing to do */ + if (len==x) return 0; + + list=self->stack->data[x-1]; + if (list == Py_None) { + return Pdata_clear(self->stack, x); + } + else { + return do_append(self, x); + } + +} + static int noload_append(Unpicklerobject *self) { - return Pdata_clear(self->stack, self->stack->length - 1); + return do_noload_append(self, self->stack->length - 1); } static int noload_appends(Unpicklerobject *self) { - Py_ssize_t i; - if ((i = marker(self)) < 0) return -1; - return Pdata_clear(self->stack, i); + return do_noload_append(self, marker(self)); +} + +static int +do_noload_setitems(Unpicklerobject *self, Py_ssize_t x) +{ + PyObject *dict = 0; + Py_ssize_t len; + + if (!( (len=self->stack->length) >= x + && x > 0 )) return stackUnderflow(); + + dict=self->stack->data[x-1]; + if (dict == Py_None) { + return Pdata_clear(self->stack, x); + } + else { + return do_setitems(self, x); + } } static int noload_setitem(Unpicklerobject *self) { - return Pdata_clear(self->stack, self->stack->length - 2); + return do_noload_setitems(self, self->stack->length - 2); } static int noload_setitems(Unpicklerobject *self) { - Py_ssize_t i; - if ((i = marker(self)) < 0) return -1; - return Pdata_clear(self->stack, i); + return do_noload_setitems(self, marker(self)); } static PyObject * diff --git a/src/zodbpickle/_pickle_32.c b/src/zodbpickle/_pickle_32.c index ff9b09c..2e559da 100644 --- a/src/zodbpickle/_pickle_32.c +++ b/src/zodbpickle/_pickle_32.c @@ -1889,10 +1889,10 @@ raw_unicode_escape(const Py_UNICODE *s, Py_ssize_t size) #else const Py_ssize_t expandsize = 6; #endif - + if (size > PY_SSIZE_T_MAX / expandsize) return PyErr_NoMemory(); - + repr = PyByteArray_FromStringAndSize(NULL, expandsize * size); if (repr == NULL) return NULL; @@ -3042,7 +3042,7 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) use_newobj = 0; } else { - use_newobj = PyUnicode_Check(name_str) && + use_newobj = PyUnicode_Check(name_str) && PyUnicode_Compare(name_str, newobj_str) == 0; Py_DECREF(name_str); } @@ -3147,7 +3147,7 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) return -1; if (state) { - if (save(self, state, 0) < 0 || + if (save(self, state, 0) < 0 || _Pickler_Write(self, &build_op, 1) < 0) return -1; } @@ -3401,7 +3401,7 @@ Pickler_dump(PicklerObject *self, PyObject *args) Developers often forget to call __init__() in their subclasses, which would trigger a segfault without this check. */ if (self->write == NULL) { - PyErr_Format(PicklingError, + PyErr_Format(PicklingError, "Pickler.__init__() was not called by %s.__init__()", Py_TYPE(self)->tp_name); return NULL; @@ -3881,7 +3881,7 @@ static PyTypeObject Pickler_Type = { 0, /*tp_is_gc*/ }; -/* Temporary helper for calling self.find_class(). +/* Temporary helper for calling self.find_class(). XXX: It would be nice to able to avoid Python function call overhead, by using directly the C version of find_class(), when find_class() is not @@ -3934,7 +3934,7 @@ load_int(UnpicklerObject *self) return bad_readline(); errno = 0; - /* XXX: Should the base argument of strtol() be explicitly set to 10? + /* XXX: Should the base argument of strtol() be explicitly set to 10? XXX(avassalotti): Should this uses PyOS_strtol()? */ x = strtol(s, &endptr, 0); @@ -5125,7 +5125,7 @@ do_setitems(UnpicklerObject *self, Py_ssize_t x) return stack_underflow(); if (len == x) /* nothing to do */ return 0; - if ((len - x) % 2 != 0) { + if ((len - x) % 2 != 0) { /* Currupt or hostile pickle -- we never write one like this. */ PyErr_SetString(UnpicklingError, "odd number of items for SETITEMS"); return -1; @@ -5482,7 +5482,7 @@ Unpickler_load(UnpicklerObject *self) not call Unpickler.__init__(). Here, we simply ensure that self->read is not NULL. */ if (self->read == NULL) { - PyErr_Format(UnpicklingError, + PyErr_Format(UnpicklingError, "Unpickler.__init__() was not called by %s.__init__()", Py_TYPE(self)->tp_name); return NULL; @@ -5575,32 +5575,67 @@ noload_extension(UnpicklerObject *self, int nbytes) return 0; } +static int +do_noload_append(UnpicklerObject *self, Py_ssize_t x) +{ + PyObject *list = 0; + Py_ssize_t len; + + len=Py_SIZE(self->stack); + if (!( len >= x && x > 0 )) return stack_underflow(); + /* nothing to do */ + if (len==x) return 0; + + list=self->stack->data[x-1]; + if (list == Py_None) { + return Pdata_clear(self->stack, x); + } + else { + return do_append(self, x); + } + +} + static int noload_append(UnpicklerObject *self) { - return Pdata_clear(self->stack, Py_SIZE(self->stack) - 1); + return do_noload_append(self, Py_SIZE(self->stack) - 1); } static int noload_appends(UnpicklerObject *self) { - int i; - if ((i = marker(self)) < 0) return -1; - return Pdata_clear(self->stack, i); + return do_noload_append(self, marker(self)); +} + +static int +do_noload_setitems(UnpicklerObject *self, Py_ssize_t x) +{ + PyObject *dict = 0; + Py_ssize_t len; + + if (!( (len=Py_SIZE(self->stack)) >= x + && x > 0 )) return stackUnderflow(); + + dict=self->stack->data[x-1]; + if (dict == Py_None) { + return Pdata_clear(self->stack, x); + } + else { + return do_setitems(self, x); + } } static int noload_setitem(UnpicklerObject *self) { - return Pdata_clear(self->stack, Py_SIZE(self->stack) - 2); + return do_noload_setitems(self, Py_SIZE(self->stack) - 2); } static int noload_setitems(UnpicklerObject *self) { - int i; - if ((i = marker(self)) < 0) return -1; - return Pdata_clear(self->stack, i); + return do_noload_setitems(self, marker(self)); } static PyObject * @@ -6018,7 +6053,7 @@ Unpickler_find_class(UnpicklerObject *self, PyObject *args) global = PyObject_GetAttr(module, global_name); Py_DECREF(module); } - else { + else { global = PyObject_GetAttr(module, global_name); } return global; @@ -6198,7 +6233,7 @@ Unpickler_init(UnpicklerObject *self, PyObject *args, PyObject *kwds) * intentional, as these should be treated as black-box implementation details. * * We do, however, have to implement pickling/unpickling support because of - * real-world code like cvs2svn. + * real-world code like cvs2svn. */ typedef struct { diff --git a/src/zodbpickle/_pickle_33.c b/src/zodbpickle/_pickle_33.c index a05d40f..a8e5725 100644 --- a/src/zodbpickle/_pickle_33.c +++ b/src/zodbpickle/_pickle_33.c @@ -5632,32 +5632,67 @@ noload_extension(UnpicklerObject *self, int nbytes) return 0; } +static int +do_noload_append(UnpicklerObject *self, Py_ssize_t x) +{ + PyObject *list = 0; + Py_ssize_t len; + + len=Py_SIZE(self->stack); + if (!( len >= x && x > 0 )) return stack_underflow(); + /* nothing to do */ + if (len==x) return 0; + + list=self->stack->data[x-1]; + if (list == Py_None) { + return Pdata_clear(self->stack, x); + } + else { + return do_append(self, x); + } + +} + static int noload_append(UnpicklerObject *self) { - return Pdata_clear(self->stack, Py_SIZE(self->stack) - 1); + return do_noload_append(self, Py_SIZE(self->stack) - 1); } static int noload_appends(UnpicklerObject *self) { - int i; - if ((i = marker(self)) < 0) return -1; - return Pdata_clear(self->stack, i); + return do_noload_append(self, marker(self)); +} + +static int +do_noload_setitems(UnpicklerObject *self, Py_ssize_t x) +{ + PyObject *dict = 0; + Py_ssize_t len; + + if (!( (len=Py_SIZE(self->stack)) >= x + && x > 0 )) return stack_underflow(); + + dict=self->stack->data[x-1]; + if (dict == Py_None) { + return Pdata_clear(self->stack, x); + } + else { + return do_setitems(self, x); + } } static int noload_setitem(UnpicklerObject *self) { - return Pdata_clear(self->stack, Py_SIZE(self->stack) - 2); + return do_noload_setitems(self, Py_SIZE(self->stack) - 2); } static int noload_setitems(UnpicklerObject *self) { - int i; - if ((i = marker(self)) < 0) return -1; - return Pdata_clear(self->stack, i); + return do_noload_setitems(self, marker(self)); } static PyObject * diff --git a/src/zodbpickle/pickle_2.py b/src/zodbpickle/pickle_2.py index 96bdeef..7803e70 100644 --- a/src/zodbpickle/pickle_2.py +++ b/src/zodbpickle/pickle_2.py @@ -882,6 +882,25 @@ def load(self): except _Stop, stopinst: return stopinst.value + def noload(self): + """Read a pickled object representation from the open file. + + If the object was an intrinsic type such as a literal list, dict + or tuple, return it. Otherwise (if the object was an instance), + return nothing useful. + """ + self.mark = object() # any new unique object + self.stack = [] + self.append = self.stack.append + read = self.read + dispatch = self.nl_dispatch + try: + while 1: + key = read(1) + dispatch[key](self) + except _Stop, stopinst: + return stopinst.value + # Return largest index k such that self.stack[k] is self.mark. # If the stack doesn't contain a mark, eventually raises IndexError. # This could be sped by maintaining another stack, of indices at which @@ -1289,6 +1308,101 @@ def load_stop(self): raise _Stop(value) dispatch[STOP] = load_stop + nl_dispatch = dispatch.copy() + + def noload_obj(self): + # Stack is ... markobject classobject arg1 arg2 ... + k = self.marker() + klass = self.stack.pop(k+1) + nl_dispatch[OBJ[0]] = noload_obj + + def noload_inst(self): + self.readline() # skip module + self.readline()[:-1] # skip name + k = self.marker() + klass = self.stack.pop(k+1) + self.append(None) + nl_dispatch[INST[0]] = noload_inst + + def noload_newobj(self): + self.stack.pop() # skip args + self.stack.pop() # skip cls + self.stack.append(None) + nl_dispatch[NEWOBJ[0]] = noload_newobj + + def noload_global(self): + self.readline() # skip module + self.readline()[:-1] # skip name + self.append(None) + nl_dispatch[GLOBAL[0]] = noload_global + + def noload_append(self): + if self.stack[-2] is not None: + self.load_append() + else: + self.stack.pop() + nl_dispatch[APPEND[0]] = noload_append + + def noload_appends(self): + stack = self.stack + mark = self.marker() + list = stack[mark - 1] + if list is not None: + list.extend(stack[mark + 1:]) + del self.stack[mark:] + nl_dispatch[APPENDS[0]] = noload_appends + + def noload_setitem(self): + if self.stack[-3] is not None: + self.load_setitem() + else: + self.stack.pop() # skip value + self.stack.pop() # skip key + nl_dispatch[SETITEM[0]] = noload_setitem + + def noload_setitems(self): + stack = self.stack + mark = self.marker() + dict = stack[mark - 1] + if dict is not None: + for i in range(mark + 1, len(stack), 2): + dict[stack[i]] = stack[i + 1] + + del stack[mark:] + nl_dispatch[SETITEMS[0]] = noload_setitems + + def noload_reduce(self): + self.stack.pop() # skip args + self.stack.pop() # skip func + self.stack.append(None) + nl_dispatch[REDUCE[0]] = noload_reduce + + def noload_build(self): + state = self.stack.pop() + nl_dispatch[BUILD[0]] = noload_build + + def noload_ext1(self): + code = ord(self.read(1)) + self.get_extension(code) + self.stack.pop() + self.stack.append(None) + nl_dispatch[EXT1[0]] = noload_ext1 + + def noload_ext2(self): + code = mloads(b'i' + self.read(2) + b'\000\000') + self.get_extension(code) + self.stack.pop() + self.stack.append(None) + nl_dispatch[EXT2[0]] = noload_ext2 + + def noload_ext4(self): + code = mloads(b'i' + self.read(4)) + self.get_extension(code) + self.stack.pop() + self.stack.append(None) + nl_dispatch[EXT4[0]] = noload_ext4 + + # Helper class for load_inst/load_obj class _EmptyClass: diff --git a/src/zodbpickle/pickle_3.py b/src/zodbpickle/pickle_3.py index 67729a2..3dc478e 100644 --- a/src/zodbpickle/pickle_3.py +++ b/src/zodbpickle/pickle_3.py @@ -848,7 +848,9 @@ def load(self): def noload(self): """Read a pickled object representation from the open file. - Don't return anything useful, just go through the motions. + If the object was an intrinsic type such as a literal list, dict + or tuple, return it. Otherwise (if the object was an instance), + return nothing useful. """ # Check whether Unpickler was initialized correctly. This is # only needed to mimic the behavior of _pickle.Unpickler.dump(). @@ -1336,22 +1338,38 @@ def noload_global(self): nl_dispatch[GLOBAL[0]] = noload_global def noload_append(self): - self.stack.pop() # skip value + if self.stack[-2] is not None: + self.load_append() + else: + self.stack.pop() nl_dispatch[APPEND[0]] = noload_append def noload_appends(self): + stack = self.stack mark = self.marker() + list = stack[mark - 1] + if list is not None: + list.extend(stack[mark + 1:]) del self.stack[mark:] nl_dispatch[APPENDS[0]] = noload_appends def noload_setitem(self): - self.stack.pop() # skip value - self.stack.pop() # skip key + if self.stack[-3] is not None: + self.load_setitem() + else: + self.stack.pop() # skip value + self.stack.pop() # skip key nl_dispatch[SETITEM[0]] = noload_setitem def noload_setitems(self): + stack = self.stack mark = self.marker() - del self.stack[mark:] + dict = stack[mark - 1] + if dict is not None: + for i in range(mark + 1, len(stack), 2): + dict[stack[i]] = stack[i + 1] + + del stack[mark:] nl_dispatch[SETITEMS[0]] = noload_setitems def noload_reduce(self): diff --git a/src/zodbpickle/tests/pickletester_2.py b/src/zodbpickle/tests/pickletester_2.py index cf357ba..74d54d2 100644 --- a/src/zodbpickle/tests/pickletester_2.py +++ b/src/zodbpickle/tests/pickletester_2.py @@ -1,3 +1,4 @@ +import io import unittest import StringIO import cStringIO @@ -1233,6 +1234,17 @@ def test_bin_persistence(self): self.assertEqual(self.id_count, 5) self.assertEqual(self.load_count, 5) + +REDUCE_A = 'reduce_A' + +class AAA(object): + def __reduce__(self): + return str, (REDUCE_A,) + +class BBB(object): + pass + + class AbstractPicklerUnpicklerObjectTests(unittest.TestCase): pickler_class = None @@ -1346,6 +1358,124 @@ def test_reusing_unpickler_objects(self): f.seek(0) self.assertEqual(unpickler.load(), data2) + def test_noload_object(self): + global _NOLOAD_OBJECT + after = {} + _NOLOAD_OBJECT = object() + aaa = AAA() + bbb = BBB() + ccc = 1 + ddd = 1.0 + eee = ('eee', 1) + fff = ['fff'] + ggg = {'ggg': 0} + unpickler = self.unpickler_class + f = io.BytesIO() + pickler = self.pickler_class(f, protocol=2) + pickler.dump(_NOLOAD_OBJECT) + after['_NOLOAD_OBJECT'] = f.tell() + pickler.dump(aaa) + after['aaa'] = f.tell() + pickler.dump(bbb) + after['bbb'] = f.tell() + pickler.dump(ccc) + after['ccc'] = f.tell() + pickler.dump(ddd) + after['ddd'] = f.tell() + pickler.dump(eee) + after['eee'] = f.tell() + pickler.dump(fff) + after['fff'] = f.tell() + pickler.dump(ggg) + after['ggg'] = f.tell() + + f.seek(0) + unpickler = self.unpickler_class(f) + unpickler.noload() # read past _NOLOAD_OBJECT + + self.assertEqual(f.tell(), after['_NOLOAD_OBJECT']) + noload = unpickler.noload() # read past aaa + self.assertEqual(noload, None) + self.assertEqual(f.tell(), after['aaa']) + + unpickler.noload() # read past bbb + self.assertEqual(f.tell(), after['bbb']) + + noload = unpickler.noload() # read past ccc + self.assertEqual(noload, ccc) + self.assertEqual(f.tell(), after['ccc']) + + noload = unpickler.noload() # read past ddd + self.assertEqual(noload, ddd) + self.assertEqual(f.tell(), after['ddd']) + + noload = unpickler.noload() # read past eee + self.assertEqual(noload, eee) + self.assertEqual(f.tell(), after['eee']) + + noload = unpickler.noload() # read past fff + self.assertEqual(noload, fff) + self.assertEqual(f.tell(), after['fff']) + + noload = unpickler.noload() # read past ggg + self.assertEqual(noload, ggg) + self.assertEqual(f.tell(), after['ggg']) + + def test_functional_noload_dict_subclass(self): + """noload() doesn't break or produce any output given a dict subclass""" + # See http://bugs.python.org/issue1101399 + o = MyDict() + o['x'] = 1 + f = io.BytesIO() + pickler = self.pickler_class(f, protocol=2) + pickler.dump(o) + f.seek(0) + unpickler = self.unpickler_class(f) + noload = unpickler.noload() + self.assertEqual(noload, None) + + + def test_functional_noload_list_subclass(self): + """noload() doesn't break or produce any output given a list subclass""" + # See http://bugs.python.org/issue1101399 + o = MyList() + o.append(1) + f = io.BytesIO() + pickler = self.pickler_class(f, protocol=2) + pickler.dump(o) + f.seek(0) + unpickler = self.unpickler_class(f) + noload = unpickler.noload() + self.assertEqual(noload, None) + + def test_functional_noload_dict(self): + """noload() implements the Python 2.6 behaviour and fills in dicts""" + # See http://bugs.python.org/issue1101399 + o = dict() + o['x'] = 1 + f = io.BytesIO() + pickler = self.pickler_class(f, protocol=2) + pickler.dump(o) + f.seek(0) + unpickler = self.unpickler_class(f) + noload = unpickler.noload() + self.assertEqual(noload, o) + + + def test_functional_noload_list(self): + """noload() implements the Python 2.6 behaviour and fills in lists""" + # See http://bugs.python.org/issue1101399 + o = list() + o.append(1) + f = io.BytesIO() + pickler = self.pickler_class(f, protocol=2) + pickler.dump(o) + f.seek(0) + unpickler = self.unpickler_class(f) + noload = unpickler.noload() + self.assertEqual(noload, o) + + class BigmemPickleTests(unittest.TestCase): # Memory requirements: 1 byte per character for input strings, 1 byte diff --git a/src/zodbpickle/tests/pickletester_3.py b/src/zodbpickle/tests/pickletester_3.py index f40b900..0680991 100644 --- a/src/zodbpickle/tests/pickletester_3.py +++ b/src/zodbpickle/tests/pickletester_3.py @@ -1816,21 +1816,88 @@ def test_noload_object(self): unpickler = self.unpickler_class(f) unpickler.noload() # read past _NOLOAD_OBJECT self.assertEqual(f.tell(), after['_NOLOAD_OBJECT']) - unpickler.noload() # read past aaa + + noload = unpickler.noload() # read past aaa + self.assertEqual(noload, None) self.assertEqual(f.tell(), after['aaa']) + unpickler.noload() # read past bbb self.assertEqual(f.tell(), after['bbb']) - unpickler.noload() # read past ccc + + noload = unpickler.noload() # read past ccc + self.assertEqual(noload, ccc) self.assertEqual(f.tell(), after['ccc']) - unpickler.noload() # read past ddd + + noload = unpickler.noload() # read past ddd + self.assertEqual(noload, ddd) self.assertEqual(f.tell(), after['ddd']) - unpickler.noload() # read past eee + + noload = unpickler.noload() # read past eee + self.assertEqual(noload, eee) self.assertEqual(f.tell(), after['eee']) - unpickler.noload() # read past fff + + noload = unpickler.noload() # read past fff + self.assertEqual(noload, fff) self.assertEqual(f.tell(), after['fff']) - unpickler.noload() # read past ggg + + noload = unpickler.noload() # read past ggg + self.assertEqual(noload, ggg) self.assertEqual(f.tell(), after['ggg']) + def test_functional_noload_dict_subclass(self): + """noload() doesn't break or produce any output given a dict subclass""" + # See http://bugs.python.org/issue1101399 + o = MyDict() + o['x'] = 1 + f = io.BytesIO() + pickler = self.pickler_class(f, protocol=2) + pickler.dump(o) + f.seek(0) + unpickler = self.unpickler_class(f) + noload = unpickler.noload() + self.assertEqual(noload, None) + + + def test_functional_noload_list_subclass(self): + """noload() doesn't break or produce any output given a list subclass""" + # See http://bugs.python.org/issue1101399 + o = MyList() + o.append(1) + f = io.BytesIO() + pickler = self.pickler_class(f, protocol=2) + pickler.dump(o) + f.seek(0) + unpickler = self.unpickler_class(f) + noload = unpickler.noload() + self.assertEqual(noload, None) + + def test_functional_noload_dict(self): + """noload() implements the Python 2.6 behaviour and fills in dicts""" + # See http://bugs.python.org/issue1101399 + o = dict() + o['x'] = 1 + f = io.BytesIO() + pickler = self.pickler_class(f, protocol=2) + pickler.dump(o) + f.seek(0) + unpickler = self.unpickler_class(f) + noload = unpickler.noload() + self.assertEqual(noload, o) + + + def test_functional_noload_list(self): + """noload() implements the Python 2.6 behaviour and fills in lists""" + # See http://bugs.python.org/issue1101399 + o = list() + o.append(1) + f = io.BytesIO() + pickler = self.pickler_class(f, protocol=2) + pickler.dump(o) + f.seek(0) + unpickler = self.unpickler_class(f) + noload = unpickler.noload() + self.assertEqual(noload, o) + # Tests for dispatch_table attribute