Applied patches and all tests work!

src/zodbpickle/_pickle.c

@@ -4161,6 +4161,18 @@ load_binfloat(UnpicklerObject *self)
     return 0;
 }
 
+/* Returns a new reference */
+static PyObject *
+decode_string(UnpicklerObject *self, PyObject *value)
+{
+    if (strcmp(self->encoding, "bytes") == 0) {
+        Py_INCREF(value);
+        return value;
+    } else {
+        return PyUnicode_FromEncodedObject(value, self->encoding, self->errors);
+    }
+}
+
 static int
 load_string(UnpicklerObject *self)
 {
@@ -4203,7 +4215,8 @@ load_string(UnpicklerObject *self)
     free(s);
     if (bytes == NULL)
         return -1;
-    str = PyUnicode_FromEncodedObject(bytes, self->encoding, self->errors);
+
+    str = decode_string(self, bytes);
     Py_DECREF(bytes);
     if (str == NULL)
         return -1;
@@ -4267,7 +4280,7 @@ load_short_binbytes(UnpicklerObject *self)
 static int
 load_binstring(UnpicklerObject *self)
 {
-    PyObject *str;
+    PyObject *bytes, *str;
     Py_ssize_t x;
     char *s;
 
@@ -4284,8 +4297,12 @@ load_binstring(UnpicklerObject *self)
     if (_Unpickler_Read(self, &s, x) < 0)
         return -1;
 
-    /* Convert Python 2.x strings to unicode. */
-    str = PyUnicode_Decode(s, x, self->encoding, self->errors);
+    bytes = PyBytes_FromStringAndSize(s, x);
+    if (bytes == NULL)
+        return -1;
+
+    str = decode_string(self, bytes);
+    Py_DECREF(bytes);
     if (str == NULL)
         return -1;
 
@@ -4296,7 +4313,7 @@ load_binstring(UnpicklerObject *self)
 static int
 load_short_binstring(UnpicklerObject *self)
 {
-    PyObject *str;
+    PyObject *bytes, *str;
     Py_ssize_t x;
     char *s;
 
@@ -4308,8 +4325,12 @@ load_short_binstring(UnpicklerObject *self)
     if (_Unpickler_Read(self, &s, x) < 0)
         return -1;
 
-    /* Convert Python 2.x strings to unicode. */
-    str = PyUnicode_Decode(s, x, self->encoding, self->errors);
+    bytes = PyBytes_FromStringAndSize(s, x);
+    if (bytes == NULL)
+        return -1;
+
+    str = decode_string(self, bytes);
+    Py_DECREF(bytes);
     if (str == NULL)
         return -1;
 
@@ -5633,7 +5654,8 @@ PyDoc_STRVAR(Unpickler_doc,
 "map the old Python 2.x names to the new names used in Python 3.x.  The\n"
 "*encoding* and *errors* tell pickle how to decode 8-bit string\n"
 "instances pickled by Python 2.x; these default to 'ASCII' and\n"
-"'strict', respectively.\n");
+"'strict', respectively. *encoding* can be 'bytes' to read 8-bit string\n"
+"instances as byte objects.\n");
 
 static int
 Unpickler_init(UnpicklerObject *self, PyObject *args, PyObject *kwds)

src/zodbpickle/pickle.py

@@ -806,7 +806,8 @@ def __init__(self, file, *, fix_imports=True,
         map the old Python 2.x names to the new names used in Python 3.x.  The
         *encoding* and *errors* tell pickle how to decode 8-bit string
         instances pickled by Python 2.x; these default to 'ASCII' and
-        'strict', respectively.
+        'strict', respectively. *encoding* can be 'bytes' to read 8-bit string
+        instances as bytes objects.
         """
         self.readline = file.readline
         self.read = file.read
@@ -943,6 +944,12 @@ def load_binfloat(self, unpack=struct.unpack):
         self.append(unpack('>d', self.read(8))[0])
     dispatch[BINFLOAT[0]] = load_binfloat
 
+    def decode_string(self, value):
+        if self.encoding == "bytes":
+            return value
+        else:
+            return value.decode(self.encoding, self.errors)
+
     def load_string(self):
         orig = self.readline()
         rep = orig[:-1]
@@ -954,15 +961,13 @@ def load_string(self):
                 break
         else:
             raise ValueError("insecure string pickle: %r" % orig)
-        self.append(codecs.escape_decode(rep)[0]
-                    .decode(self.encoding, self.errors))
+        self.append(self.decode_string(codecs.escape_decode(rep)[0]))
     dispatch[STRING[0]] = load_string
 
     def load_binstring(self):
         len = mloads(b'i' + self.read(4))
         data = self.read(len)
-        value = str(data, self.encoding, self.errors)
-        self.append(value)
+        self.append(self.decode_string(data))
     dispatch[BINSTRING[0]] = load_binstring
 
     def load_binbytes(self):
@@ -981,9 +986,8 @@ def load_binunicode(self):
 
     def load_short_binstring(self):
         len = ord(self.read(1))
-        data = bytes(self.read(len))
-        value = str(data, self.encoding, self.errors)
-        self.append(value)
+        data = self.read(len)
+        self.append(self.decode_string(data))
     dispatch[SHORT_BINSTRING[0]] = load_short_binstring
 
     def load_short_binbytes(self):

src/zodbpickle/tests/pickletester.py

@@ -36,7 +36,6 @@ def count_opcode(code, pickle):
             n += 1
     return n
 
-
 class UnseekableIO(io.BytesIO):
     def peek(self, *args):
         raise NotImplementedError
@@ -1205,6 +1204,59 @@ def test_negative_32b_binput(self):
         dumped = b'\x80\x03X\x01\x00\x00\x00ar\xff\xff\xff\xff.'
         self.assertRaises(ValueError, self.loads, dumped)
 
+class AbstractBytestrTests:
+    def unpickleEqual(self, data, unpickled):
+        loaded = self.loads(data, encoding="bytes")
+        self.assertEqual(loaded, unpickled)
+
+    def test_load_str_protocol_0(self):
+        """ Test str from protocol=0
+            python 2: pickle.dumps('bytestring \x00\xa0', protocol=0) """
+        self.unpickleEqual(
+                b"S'bytestring \\x00\\xa0'\np0\n.",
+                b'bytestring \x00\xa0')
+
+    def test_load_str_protocol_1(self):
+        """ Test str from protocol=1
+        python 2: pickle.dumps('bytestring \x00\xa0', protocol=1) """
+        self.unpickleEqual(
+                b'U\rbytestring \x00\xa0q\x00.',
+                b'bytestring \x00\xa0')
+
+    def test_load_str_protocol_2(self):
+        """ Test str from protocol=2
+        python 2: pickle.dumps('bytestring \x00\xa0', protocol=2) """
+        self.unpickleEqual(
+                b'\x80\x02U\rbytestring \x00\xa0q\x00.',
+                b'bytestring \x00\xa0')
+
+    def test_load_unicode_protocol_0(self):
+        """ Test unicode with protocol=0
+        python 2: pickle.dumps(u"\u041a\u043e\u043c\u043f\u044c\u044e\u0442\u0435\u0440", protocol=0) """
+        self.unpickleEqual(
+                b'V\\u041a\\u043e\\u043c\\u043f\\u044c\\u044e\\u0442\\u0435\\u0440\np0\n.',
+                '\u041a\u043e\u043c\u043f\u044c\u044e\u0442\u0435\u0440')
+
+    def test_load_unicode_protocol_1(self):
+        """ Test unicode with protocol=1
+        python 2: pickle.dumps(u"\u041a\u043e\u043c\u043f\u044c\u044e\u0442\u0435\u0440", protocol=1) """
+        self.unpickleEqual(
+                b'X\x12\x00\x00\x00\xd0\x9a\xd0\xbe\xd0\xbc\xd0\xbf\xd1\x8c\xd1\x8e\xd1\x82\xd0\xb5\xd1\x80q\x00.',
+                '\u041a\u043e\u043c\u043f\u044c\u044e\u0442\u0435\u0440')
+
+    def test_load_unicode_protocol_2(self):
+        """ Test unicode with protocol=1
+        python 2: pickle.dumps(u"\u041a\u043e\u043c\u043f\u044c\u044e\u0442\u0435\u0440", protocol=2) """
+        self.unpickleEqual(
+                b'\x80\x02X\x12\x00\x00\x00\xd0\x9a\xd0\xbe\xd0\xbc\xd0\xbf\xd1\x8c\xd1\x8e\xd1\x82\xd0\xb5\xd1\x80q\x00.',
+                '\u041a\u043e\u043c\u043f\u044c\u044e\u0442\u0435\u0440')
+
+    def test_load_long_str_protocol_1(self):
+        """ Test long str with protocol=1
+        python 2: pickle.dumps('x'*300, protocol=1) """
+        self.unpickleEqual(
+                b'T,\x01\x00\x00xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxq\x00.',
+                b'x'*300)
 
 class BigmemPickleTests:
 

src/zodbpickle/tests/test_pickle.py

@@ -10,6 +10,7 @@
 from .pickletester import AbstractPicklerUnpicklerObjectTests
 from .pickletester import AbstractDispatchTableTests
 from .pickletester import BigmemPickleTests
+from .pickletester import AbstractBytestrTests
 
 from zodbpickle import pickle
 
@@ -24,14 +25,14 @@ class PickleTests(AbstractPickleModuleTests, unittest.TestCase):
     pass
 
 
-class PyPicklerTests(AbstractPickleTests, unittest.TestCase):
+class PyPicklerBase(AbstractPickleTests):
 
     pickler = pickle._Pickler
     unpickler = pickle._Unpickler
 
-    def dumps(self, arg, proto=None):
+    def dumps(self, arg, proto=None, **kwds):
         f = io.BytesIO()
-        p = self.pickler(f, proto)
+        p = self.pickler(f, proto, **kwds)
         p.dump(arg)
         f.seek(0)
         return bytes(f.read())
@@ -41,6 +42,12 @@ def loads(self, buf, **kwds):
         u = self.unpickler(f, **kwds)
         return u.load()
 
+class PyPicklerTests(PyPicklerBase, AbstractPickleTests, unittest.TestCase):
+    pass
+
+class PyPicklerBytestrTests(PyPicklerBase, AbstractBytestrTests,
+                            unittest.TestCase):
+    pass
 
 class InMemoryPickleTests(AbstractPickleTests, BigmemPickleTests,
                           unittest.TestCase):
@@ -102,6 +109,10 @@ class CPicklerTests(PyPicklerTests):
         pickler = _pickle.Pickler
         unpickler = _pickle.Unpickler
 
+    class CPicklerBytestrTests(PyPicklerBytestrTests):
+        pickler = _pickle.Pickler
+        unpickler = _pickle.Unpickler
+
     class CPersPicklerTests(PyPersPicklerTests):
         pickler = _pickle.Pickler
         unpickler = _pickle.Unpickler
@@ -131,9 +142,10 @@ def get_dispatch_table(self):
 
 def test_main():
     tests = [PickleTests, PyPicklerTests, PyPersPicklerTests,
-             PyDispatchTableTests, PyChainDispatchTableTests]
+             PyDispatchTableTests, PyChainDispatchTableTests,
+             PyPicklerBytestrTests]
     if has_c_implementation:
-        tests.extend([CPicklerTests, CPersPicklerTests,
+        tests.extend([CPicklerTests, CPicklerBytestrTests, CPersPicklerTests,
                       CDumpPickle_LoadPickle, DumpPickle_CLoadPickle,
                       PyPicklerUnpicklerObjectTests,
                       CPicklerUnpicklerObjectTests,