Skip to content
Permalink
Browse files

bpo-33578: Fix getstate/setstate for CJK decoder (GH-10290)

Previous version was casting to Py_ssize_t incorrectly
and exhibited unexpected behavior on big-endian systems.
  • Loading branch information
libcthorne authored and methane committed Nov 2, 2018
1 parent 318ab63 commit 488c0a6cdf09e21774e63c2a430ecc0de804d147
Showing with 26 additions and 5 deletions.
  1. +4 −0 Lib/test/test_multibytecodec.py
  2. +22 −5 Modules/cjkcodecs/multibytecodec.c
@@ -271,6 +271,10 @@ def test_state_methods(self):
pending4, _ = decoder.getstate()
self.assertEqual(pending4, b'')

# Ensure state values are preserved correctly
decoder.setstate((b'abc', 123456789))
self.assertEqual(decoder.getstate(), (b'abc', 123456789))

def test_setstate_validates_input(self):
decoder = codecs.getincrementaldecoder('euc_jp')()
self.assertRaises(TypeError, decoder.setstate, 123)
@@ -1218,14 +1218,24 @@ _multibytecodec_MultibyteIncrementalDecoder_getstate_impl(MultibyteIncrementalDe
/*[clinic end generated code: output=255009c4713b7f82 input=4006aa49bddbaa75]*/
{
PyObject *buffer;
PyObject *statelong;

buffer = PyBytes_FromStringAndSize((const char *)self->pending,
self->pendingsize);
if (buffer == NULL) {
return NULL;
}

return make_tuple(buffer, (Py_ssize_t)*self->state.c);
statelong = (PyObject *)_PyLong_FromByteArray(self->state.c,
sizeof(self->state.c),
1 /* little-endian */ ,
0 /* unsigned */ );
if (statelong == NULL) {
Py_DECREF(buffer);
return NULL;
}

return Py_BuildValue("NN", buffer, statelong);
}

/*[clinic input]
@@ -1240,16 +1250,23 @@ _multibytecodec_MultibyteIncrementalDecoder_setstate_impl(MultibyteIncrementalDe
/*[clinic end generated code: output=106b2fbca3e2dcc2 input=e5d794e8baba1a47]*/
{
PyObject *buffer;
PyLongObject *statelong;
Py_ssize_t buffersize;
char *bufferstr;
unsigned long long flag;
unsigned char statebytes[8];

if (!PyArg_ParseTuple(state, "SK;setstate(): illegal state argument",
&buffer, &flag))
if (!PyArg_ParseTuple(state, "SO!;setstate(): illegal state argument",
&buffer, &PyLong_Type, &statelong))
{
return NULL;
}

if (_PyLong_AsByteArray(statelong, statebytes, sizeof(statebytes),
1 /* little-endian */ ,
0 /* unsigned */ ) < 0) {
return NULL;
}

buffersize = PyBytes_Size(buffer);
if (buffersize == -1) {
return NULL;
@@ -1266,7 +1283,7 @@ _multibytecodec_MultibyteIncrementalDecoder_setstate_impl(MultibyteIncrementalDe
}
self->pendingsize = buffersize;
memcpy(self->pending, bufferstr, self->pendingsize);
memcpy(self->state.c, (unsigned char *)&flag, sizeof(flag));
memcpy(self->state.c, statebytes, sizeof(statebytes));

Py_RETURN_NONE;
}

0 comments on commit 488c0a6

Please sign in to comment.
You can’t perform that action at this time.