63 changes: 35 additions & 28 deletions msgpack/_packer.pyx
Original file line number Diff line number Diff line change
@@ -1,19 +1,16 @@
# coding: utf-8
#cython: embedsignature=True, c_string_encoding=ascii

from cpython cimport *
from cpython.version cimport PY_MAJOR_VERSION
from cpython.exc cimport PyErr_WarnEx
from cpython.bytearray cimport PyByteArray_Check, PyByteArray_CheckExact

from msgpack.exceptions import PackValueError, PackOverflowError
from msgpack import ExtType
cdef ExtType

from . import ExtType


cdef extern from "Python.h":

int PyMemoryView_Check(object obj)
int PyByteArray_Check(object obj)
int PyByteArray_CheckExact(object obj)
char* PyUnicode_AsUTF8AndSize(object obj, Py_ssize_t *l) except NULL


Expand Down Expand Up @@ -41,6 +38,9 @@ cdef extern from "pack.h":
int msgpack_pack_ext(msgpack_packer* pk, char typecode, size_t l)
int msgpack_pack_unicode(msgpack_packer* pk, object o, long long limit)

cdef extern from "buff_converter.h":
object buff_to_buff(char *, Py_ssize_t)

cdef int DEFAULT_RECURSE_LIMIT=511
cdef long long ITEM_LIMIT = (2**32)-1

Expand Down Expand Up @@ -118,7 +118,7 @@ cdef class Packer(object):
bint use_single_float=False, bint autoreset=True, bint use_bin_type=False,
bint strict_types=False):
if encoding is not None:
PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated.", 1)
PyErr_WarnEx(DeprecationWarning, "encoding is deprecated.", 1)
self.use_float = use_single_float
self.strict_types = strict_types
self.autoreset = autoreset
Expand Down Expand Up @@ -162,7 +162,7 @@ cdef class Packer(object):
cdef Py_buffer view

if nest_limit < 0:
raise PackValueError("recursion limit exceeded.")
raise ValueError("recursion limit exceeded.")

while True:
if o is None:
Expand All @@ -188,7 +188,7 @@ cdef class Packer(object):
default_used = True
continue
else:
raise PackOverflowError("Integer value out of range")
raise OverflowError("Integer value out of range")
elif PyInt_CheckExact(o) if strict_types else PyInt_Check(o):
longval = o
ret = msgpack_pack_long(&self.pk, longval)
Expand All @@ -200,9 +200,9 @@ cdef class Packer(object):
dval = o
ret = msgpack_pack_double(&self.pk, dval)
elif PyBytesLike_CheckExact(o) if strict_types else PyBytesLike_Check(o):
L = len(o)
L = Py_SIZE(o)
if L > ITEM_LIMIT:
raise PackValueError("%s is too large" % type(o).__name__)
PyErr_Format(ValueError, b"%.200s object is too large", Py_TYPE(o).tp_name)
rawval = o
ret = msgpack_pack_bin(&self.pk, L)
if ret == 0:
Expand All @@ -211,12 +211,12 @@ cdef class Packer(object):
if self.encoding == NULL and self.unicode_errors == NULL:
ret = msgpack_pack_unicode(&self.pk, o, ITEM_LIMIT);
if ret == -2:
raise PackValueError("unicode string is too large")
raise ValueError("unicode string is too large")
else:
o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors)
L = len(o)
L = Py_SIZE(o)
if L > ITEM_LIMIT:
raise PackValueError("unicode string is too large")
raise ValueError("unicode string is too large")
ret = msgpack_pack_raw(&self.pk, L)
if ret == 0:
rawval = o
Expand All @@ -225,18 +225,18 @@ cdef class Packer(object):
d = <dict>o
L = len(d)
if L > ITEM_LIMIT:
raise PackValueError("dict is too large")
raise ValueError("dict is too large")
ret = msgpack_pack_map(&self.pk, L)
if ret == 0:
for k, v in d.iteritems():
for k, v in d.items():
ret = self._pack(k, nest_limit-1)
if ret != 0: break
ret = self._pack(v, nest_limit-1)
if ret != 0: break
elif not strict_types and PyDict_Check(o):
L = len(o)
if L > ITEM_LIMIT:
raise PackValueError("dict is too large")
raise ValueError("dict is too large")
ret = msgpack_pack_map(&self.pk, L)
if ret == 0:
for k, v in o.items():
Expand All @@ -250,25 +250,25 @@ cdef class Packer(object):
rawval = o.data
L = len(o.data)
if L > ITEM_LIMIT:
raise PackValueError("EXT data is too large")
raise ValueError("EXT data is too large")
ret = msgpack_pack_ext(&self.pk, longval, L)
ret = msgpack_pack_raw_body(&self.pk, rawval, L)
elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)):
L = len(o)
L = Py_SIZE(o)
if L > ITEM_LIMIT:
raise PackValueError("list is too large")
raise ValueError("list is too large")
ret = msgpack_pack_array(&self.pk, L)
if ret == 0:
for v in o:
ret = self._pack(v, nest_limit-1)
if ret != 0: break
elif PyMemoryView_Check(o):
if PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) != 0:
raise PackValueError("could not get buffer for memoryview")
raise ValueError("could not get buffer for memoryview")
L = view.len
if L > ITEM_LIMIT:
PyBuffer_Release(&view);
raise PackValueError("memoryview is too large")
raise ValueError("memoryview is too large")
ret = msgpack_pack_bin(&self.pk, L)
if ret == 0:
ret = msgpack_pack_raw_body(&self.pk, <char*>view.buf, L)
Expand All @@ -278,7 +278,7 @@ cdef class Packer(object):
default_used = 1
continue
else:
raise TypeError("can't serialize %r" % (o,))
PyErr_Format(TypeError, b"can not serialize '%.200s' object", Py_TYPE(o).tp_name)
return ret

cpdef pack(self, object obj):
Expand All @@ -301,7 +301,7 @@ cdef class Packer(object):

def pack_array_header(self, long long size):
if size > ITEM_LIMIT:
raise PackValueError
raise ValueError
cdef int ret = msgpack_pack_array(&self.pk, size)
if ret == -1:
raise MemoryError
Expand All @@ -314,7 +314,7 @@ cdef class Packer(object):

def pack_map_header(self, long long size):
if size > ITEM_LIMIT:
raise PackValueError
raise ValueError
cdef int ret = msgpack_pack_map(&self.pk, size)
if ret == -1:
raise MemoryError
Expand Down Expand Up @@ -349,9 +349,16 @@ cdef class Packer(object):
return buf

def reset(self):
"""Clear internal buffer."""
"""Reset internal buffer.
This method is usaful only when autoreset=False.
"""
self.pk.length = 0

def bytes(self):
"""Return buffer content."""
"""Return internal buffer contents as bytes object"""
return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)

def getbuffer(self):
"""Return view of internal buffer."""
return buff_to_buff(self.pk.buf, self.pk.length)
216 changes: 121 additions & 95 deletions msgpack/_unpacker.pyx

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion msgpack/_version.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
version = (0, 5, 6)
version = (0, 6, 2)
28 changes: 28 additions & 0 deletions msgpack/buff_converter.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
#include "Python.h"

/* cython does not support this preprocessor check => write it in raw C */
#if PY_MAJOR_VERSION == 2
static PyObject *
buff_to_buff(char *buff, Py_ssize_t size)
{
return PyBuffer_FromMemory(buff, size);
}

#elif (PY_MAJOR_VERSION == 3) && (PY_MINOR_VERSION >= 3)
static PyObject *
buff_to_buff(char *buff, Py_ssize_t size)
{
return PyMemoryView_FromMemory(buff, size, PyBUF_READ);
}
#else
static PyObject *
buff_to_buff(char *buff, Py_ssize_t size)
{
Py_buffer pybuf;
if (PyBuffer_FillInfo(&pybuf, NULL, buff, size, 1, PyBUF_FULL_RO) == -1) {
return NULL;
}

return PyMemoryView_FromBuffer(&pybuf);
}
#endif
45 changes: 26 additions & 19 deletions msgpack/exceptions.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
class UnpackException(Exception):
"""Deprecated. Use Exception instead to catch all exception during unpacking."""
"""Base class for some exceptions raised while unpacking.
NOTE: unpack may raise exception other than subclass of
UnpackException. If you want to catch all error, catch
Exception instead.
"""


class BufferFull(UnpackException):
Expand All @@ -10,32 +15,34 @@ class OutOfData(UnpackException):
pass


class UnpackValueError(UnpackException, ValueError):
"""Deprecated. Use ValueError instead."""
class FormatError(ValueError, UnpackException):
"""Invalid msgpack format"""


class ExtraData(UnpackValueError):
def __init__(self, unpacked, extra):
self.unpacked = unpacked
self.extra = extra

def __str__(self):
return "unpack(b) received extra data."
class StackError(ValueError, UnpackException):
"""Too nested"""


class PackException(Exception):
"""Deprecated. Use Exception instead to catch all exception during packing."""
# Deprecated. Use ValueError instead
UnpackValueError = ValueError


class PackValueError(PackException, ValueError):
"""PackValueError is raised when type of input data is supported but it's value is unsupported.
class ExtraData(UnpackValueError):
"""ExtraData is raised when there is trailing data.
Deprecated. Use ValueError instead.
This exception is raised while only one-shot (not streaming)
unpack.
"""

def __init__(self, unpacked, extra):
self.unpacked = unpacked
self.extra = extra

def __str__(self):
return "unpack(b) received extra data."

class PackOverflowError(PackValueError, OverflowError):
"""PackOverflowError is raised when integer value is out of range of msgpack support [-2**31, 2**32).

Deprecated. Use ValueError instead.
"""
# Deprecated. Use Exception instead to catch all exception during packing.
PackException = Exception
PackValueError = ValueError
PackOverflowError = OverflowError
324 changes: 190 additions & 134 deletions msgpack/fallback.py

Large diffs are not rendered by default.

13 changes: 3 additions & 10 deletions msgpack/pack_template.h
Original file line number Diff line number Diff line change
Expand Up @@ -566,24 +566,17 @@ if(sizeof(unsigned long long) == 2) {

static inline int msgpack_pack_float(msgpack_packer* x, float d)
{
union { float f; uint32_t i; } mem;
mem.f = d;
unsigned char buf[5];
buf[0] = 0xca; _msgpack_store32(&buf[1], mem.i);
buf[0] = 0xca;
_PyFloat_Pack4(d, &buf[1], 0);
msgpack_pack_append_buffer(x, buf, 5);
}

static inline int msgpack_pack_double(msgpack_packer* x, double d)
{
union { double f; uint64_t i; } mem;
mem.f = d;
unsigned char buf[9];
buf[0] = 0xcb;
#if defined(__arm__) && !(__ARM_EABI__) // arm-oabi
// https://github.com/msgpack/msgpack-perl/pull/1
mem.i = (mem.i & 0xFFFFFFFFUL) << 32UL | (mem.i >> 32UL);
#endif
_msgpack_store64(&buf[1], mem.i);
_PyFloat_Pack8(d, &buf[1], 0);
msgpack_pack_append_buffer(x, buf, 9);
}

Expand Down
5 changes: 5 additions & 0 deletions msgpack/unpack.h
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ typedef struct unpack_user {
bool use_list;
bool raw;
bool has_pairs_hook;
bool strict_map_key;
PyObject *object_hook;
PyObject *list_hook;
PyObject *ext_hook;
Expand Down Expand Up @@ -188,6 +189,10 @@ static inline int unpack_callback_map(unpack_user* u, unsigned int n, msgpack_un

static inline int unpack_callback_map_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v)
{
if (u->strict_map_key && !PyUnicode_CheckExact(k) && !PyBytes_CheckExact(k)) {
PyErr_Format(PyExc_ValueError, "%.100s is not allowed for map key", Py_TYPE(k)->tp_name);
return -1;
}
if (u->has_pairs_hook) {
msgpack_unpack_object item = PyTuple_Pack(2, k, v);
if (!item)
Expand Down
43 changes: 9 additions & 34 deletions msgpack/unpack_template.h
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize
goto _fixed_trail_again

#define start_container(func, count_, ct_) \
if(top >= MSGPACK_EMBED_STACK_SIZE) { goto _failed; } /* FIXME */ \
if(top >= MSGPACK_EMBED_STACK_SIZE) { ret = -3; goto _end; } \
if(construct_cb(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \
if((count_) == 0) { obj = stack[top].obj; \
if (construct_cb(func##_end)(user, &obj) < 0) { goto _failed; } \
Expand All @@ -132,27 +132,6 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize
stack[top].size = count_; \
stack[top].count = 0; \
++top; \
/*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \
/*printf("stack push %d\n", top);*/ \
/* FIXME \
if(top >= stack_size) { \
if(stack_size == MSGPACK_EMBED_STACK_SIZE) { \
size_t csize = sizeof(unpack_stack) * MSGPACK_EMBED_STACK_SIZE; \
size_t nsize = csize * 2; \
unpack_stack* tmp = (unpack_stack*)malloc(nsize); \
if(tmp == NULL) { goto _failed; } \
memcpy(tmp, ctx->stack, csize); \
ctx->stack = stack = tmp; \
ctx->stack_size = stack_size = MSGPACK_EMBED_STACK_SIZE * 2; \
} else { \
size_t nsize = sizeof(unpack_stack) * ctx->stack_size * 2; \
unpack_stack* tmp = (unpack_stack*)realloc(ctx->stack, nsize); \
if(tmp == NULL) { goto _failed; } \
ctx->stack = stack = tmp; \
ctx->stack_size = stack_size = stack_size * 2; \
} \
} \
*/ \
goto _header_again

#define NEXT_CS(p) ((unsigned int)*p & 0x1f)
Expand Down Expand Up @@ -229,7 +208,8 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize
case 0xdf: // map 32
again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01));
default:
goto _failed;
ret = -2;
goto _end;
}
SWITCH_RANGE(0xa0, 0xbf) // FixRaw
again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero);
Expand All @@ -239,7 +219,8 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize
start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY);

SWITCH_RANGE_DEFAULT
goto _failed;
ret = -2;
goto _end;
SWITCH_RANGE_END
// end CS_HEADER

Expand All @@ -262,17 +243,11 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize
_msgpack_load32(uint32_t,n)+1,
_ext_zero);
case CS_FLOAT: {
union { uint32_t i; float f; } mem;
mem.i = _msgpack_load32(uint32_t,n);
push_fixed_value(_float, mem.f); }
double f = _PyFloat_Unpack4((unsigned char*)n, 0);
push_fixed_value(_float, f); }
case CS_DOUBLE: {
union { uint64_t i; double f; } mem;
mem.i = _msgpack_load64(uint64_t,n);
#if defined(__arm__) && !(__ARM_EABI__) // arm-oabi
// https://github.com/msgpack/msgpack-perl/pull/1
mem.i = (mem.i & 0xFFFFFFFFUL) << 32UL | (mem.i >> 32UL);
#endif
push_fixed_value(_double, mem.f); }
double f = _PyFloat_Unpack8((unsigned char*)n, 0);
push_fixed_value(_double, f); }
case CS_UINT_8:
push_fixed_value(_uint8, *(uint8_t*)n);
case CS_UINT_16:
Expand Down
2 changes: 1 addition & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
Cython==0.27.3
Cython~=0.29.13
21 changes: 10 additions & 11 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,7 @@ def build_extension(self, ext):
if have_cython:
class Sdist(sdist):
def __init__(self, *args, **kwargs):
for src in glob('msgpack/*.pyx'):
cythonize(src)
cythonize('msgpack/_cmsgpack.pyx')
sdist.__init__(self, *args, **kwargs)
else:
Sdist = sdist
Expand All @@ -85,14 +84,8 @@ def __init__(self, *args, **kwargs):

ext_modules = []
if not hasattr(sys, 'pypy_version_info'):
ext_modules.append(Extension('msgpack._packer',
sources=['msgpack/_packer.cpp'],
libraries=libraries,
include_dirs=['.'],
define_macros=macros,
))
ext_modules.append(Extension('msgpack._unpacker',
sources=['msgpack/_unpacker.cpp'],
ext_modules.append(Extension('msgpack._cmsgpack',
sources=['msgpack/_cmsgpack.cpp'],
libraries=libraries,
include_dirs=['.'],
define_macros=macros,
Expand Down Expand Up @@ -120,7 +113,13 @@ def __init__(self, *args, **kwargs):
packages=['msgpack'],
description=desc,
long_description=long_desc,
url='http://msgpack.org/',
long_description_content_type="text/x-rst",
url='https://msgpack.org/',
project_urls = {
'Documentation': 'https://msgpack-python.readthedocs.io/',
'Source': 'https://github.com/msgpack/msgpack-python',
'Tracker': 'https://github.com/msgpack/msgpack-python/issues',
},
license='Apache 2.0',
classifiers=[
'Programming Language :: Python :: 2',
Expand Down
5 changes: 4 additions & 1 deletion test/test_buffer.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,10 @@
def test_unpack_buffer():
from array import array
buf = array('b')
buf.fromstring(packb((b'foo', b'bar')))
try:
buf.frombytes(packb((b'foo', b'bar')))
except AttributeError: # PY2
buf.fromstring(packb((b'foo', b'bar')))
obj = unpackb(buf, use_list=1)
assert [b'foo', b'bar'] == obj

Expand Down
44 changes: 38 additions & 6 deletions test/test_except.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# coding: utf-8

from pytest import raises
from msgpack import packb, unpackb
from msgpack import packb, unpackb, Unpacker, FormatError, StackError, OutOfData

import datetime

Expand All @@ -19,13 +19,45 @@ def test_raise_on_find_unsupported_value():
def test_raise_from_object_hook():
def hook(obj):
raise DummyException

raises(DummyException, unpackb, packb({}), object_hook=hook)
raises(DummyException, unpackb, packb({'fizz': 'buzz'}), object_hook=hook)
raises(DummyException, unpackb, packb({'fizz': 'buzz'}), object_pairs_hook=hook)
raises(DummyException, unpackb, packb({'fizz': {'buzz': 'spam'}}), object_hook=hook)
raises(DummyException, unpackb, packb({'fizz': {'buzz': 'spam'}}), object_pairs_hook=hook)
raises(DummyException, unpackb, packb({"fizz": "buzz"}), object_hook=hook)
raises(DummyException, unpackb, packb({"fizz": "buzz"}), object_pairs_hook=hook)
raises(DummyException, unpackb, packb({"fizz": {"buzz": "spam"}}), object_hook=hook)
raises(
DummyException,
unpackb,
packb({"fizz": {"buzz": "spam"}}),
object_pairs_hook=hook,
)


def test_invalidvalue():
incomplete = b"\xd9\x97#DL_" # raw8 - length=0x97
with raises(ValueError):
unpackb(incomplete)

with raises(OutOfData):
unpacker = Unpacker()
unpacker.feed(incomplete)
unpacker.unpack()

with raises(FormatError):
unpackb(b"\xc1") # (undefined tag)

with raises(FormatError):
unpackb(b"\x91\xc1") # fixarray(len=1) [ (undefined tag) ]

with raises(StackError):
unpackb(b"\x91" * 3000) # nested fixarray(len=1)


def test_strict_map_key():
valid = {u"unicode": 1, b"bytes": 2}
packed = packb(valid, use_bin_type=True)
assert valid == unpackb(packed, raw=False, strict_map_key=True)

invalid = {42: 1}
packed = packb(invalid, use_bin_type=True)
with raises(ValueError):
unpackb(b'\xd9\x97#DL_')
unpackb(packed, raw=False, strict_map_key=True)
10 changes: 8 additions & 2 deletions test/test_extension.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,15 +40,21 @@ def default(obj):
print('default called', obj)
if isinstance(obj, array.array):
typecode = 123 # application specific typecode
data = obj.tostring()
try:
data = obj.tobytes()
except AttributeError:
data = obj.tostring()
return ExtType(typecode, data)
raise TypeError("Unknown type object %r" % (obj,))

def ext_hook(code, data):
print('ext_hook called', code, data)
assert code == 123
obj = array.array('d')
obj.fromstring(data)
try:
obj.frombytes(data)
except AttributeError: # PY2
obj.fromstring(data)
return obj

obj = [42, b'hello', array.array('d', [1.1, 2.2, 3.3])]
Expand Down
25 changes: 24 additions & 1 deletion test/test_limits.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,6 @@ def test_max_ext_len():
unpacker.unpack()



# PyPy fails following tests because of constant folding?
# https://bugs.pypy.org/issue1721
#@pytest.mark.skipif(True, reason="Requires very large memory.")
Expand Down Expand Up @@ -134,3 +133,27 @@ def test_max_ext_len():
# x.append(0)
# with pytest.raises(ValueError):
# packb(x)


# auto max len

def test_auto_max_array_len():
packed = b'\xde\x00\x06zz'
with pytest.raises(UnpackValueError):
unpackb(packed, raw=False)

unpacker = Unpacker(max_buffer_size=5, raw=False)
unpacker.feed(packed)
with pytest.raises(UnpackValueError):
unpacker.unpack()

def test_auto_max_map_len():
# len(packed) == 6 -> max_map_len == 3
packed = b'\xde\x00\x04zzz'
with pytest.raises(UnpackValueError):
unpackb(packed, raw=False)

unpacker = Unpacker(max_buffer_size=6, raw=False)
unpacker.feed(packed)
with pytest.raises(UnpackValueError):
unpacker.unpack()
35 changes: 26 additions & 9 deletions test/test_pack.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,15 @@
# coding: utf-8
from __future__ import absolute_import, division, print_function, unicode_literals

from collections import OrderedDict
from io import BytesIO
import struct

import pytest
from pytest import raises, xfail

from msgpack import packb, unpackb, Unpacker, Packer
from msgpack import packb, unpackb, Unpacker, Packer, pack

from collections import OrderedDict
from io import BytesIO

def check(data, use_list=False):
re = unpackb(packb(data), use_list=use_list)
Expand Down Expand Up @@ -47,7 +49,8 @@ def testPackUTF32(): # deprecated
"Русский текст",
]
for td in test_data:
re = unpackb(packb(td, encoding='utf-32'), use_list=1, encoding='utf-32')
with pytest.deprecated_call():
re = unpackb(packb(td, encoding='utf-32'), use_list=1, encoding='utf-32')
assert re == td
except LookupError as e:
xfail(e)
Expand All @@ -67,19 +70,23 @@ def testPackByteArrays():
check(td)

def testIgnoreUnicodeErrors(): # deprecated
re = unpackb(packb(b'abc\xeddef'), encoding='utf-8', unicode_errors='ignore', use_list=1)
with pytest.deprecated_call():
re = unpackb(packb(b'abc\xeddef'), encoding='utf-8', unicode_errors='ignore', use_list=1)
assert re == "abcdef"

def testStrictUnicodeUnpack():
with raises(UnicodeDecodeError):
unpackb(packb(b'abc\xeddef'), raw=False, use_list=1)
packed = packb(b'abc\xeddef')
with pytest.raises(UnicodeDecodeError):
unpackb(packed, raw=False, use_list=1)

def testStrictUnicodePack(): # deprecated
with raises(UnicodeEncodeError):
packb("abc\xeddef", encoding='ascii', unicode_errors='strict')
with pytest.deprecated_call():
packb("abc\xeddef", encoding='ascii', unicode_errors='strict')

def testIgnoreErrorsPack(): # deprecated
re = unpackb(packb("abcФФФdef", encoding='ascii', unicode_errors='ignore'), raw=False, use_list=1)
with pytest.deprecated_call():
re = unpackb(packb("abcФФФdef", encoding='ascii', unicode_errors='ignore'), raw=False, use_list=1)
assert re == "abcdef"

def testDecodeBinary():
Expand Down Expand Up @@ -148,3 +155,13 @@ def test_pairlist():
packed = packer.pack_map_pairs(pairlist)
unpacked = unpackb(packed, object_pairs_hook=list)
assert pairlist == unpacked

def test_get_buffer():
packer = Packer(autoreset=0, use_bin_type=True)
packer.pack([1, 2])
strm = BytesIO()
strm.write(packer.getbuffer())
written = strm.getvalue()

expected = packb([1, 2], use_bin_type=True)
assert written == expected
29 changes: 0 additions & 29 deletions test/test_unpack_raw.py

This file was deleted.

8 changes: 4 additions & 4 deletions tox.ini
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[tox]
envlist = {py27,py35,py36}-{c,pure},{pypy,pypy3}-pure,py27-x86,py34-x86
envlist = {py27,py35,py36,py37}-{c,pure},{pypy,pypy3}-pure,py27-x86,py34-x86

[variants:pure]
setenv=
Expand All @@ -11,7 +11,7 @@ deps=

changedir=test
commands=
c,x86: python -c 'from msgpack import _packer, _unpacker'
c,x86: python -c 'from msgpack import _cmsgpack'
c,x86: py.test
pure: py.test

Expand All @@ -23,7 +23,7 @@ deps=
changedir=test
commands=
python -c 'import sys; print(hex(sys.maxsize))'
python -c 'from msgpack import _packer, _unpacker'
python -c 'from msgpack import _cmsgpack'
py.test

[testenv:py34-x86]
Expand All @@ -34,5 +34,5 @@ deps=
changedir=test
commands=
python -c 'import sys; print(hex(sys.maxsize))'
python -c 'from msgpack import _packer, _unpacker'
python -c 'from msgpack import _cmsgpack'
py.test