diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7e08d672..55593199 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,6 +61,7 @@ jobs: - "3.8" - "3.9" - "3.10" + - "3.11-dev" - "pypy-3.7" - "pypy-3.8" - "pypy-3.9" diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 72fdbe91..dcfe9ee0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,10 @@ Changelog .. This document is user facing. Please word the changes in such a way .. that users understand how the changes affect the new version. +version 1.1.0 +----------------- ++ Added tests and support for Python 3.11. + version 1.0.1 ------------------ + Fixed failing tests and wheel builds for PyPy. diff --git a/setup.py b/setup.py index 4fd59db3..fd6a6a49 100644 --- a/setup.py +++ b/setup.py @@ -161,7 +161,7 @@ def build_isa_l(compiler_command: str, compiler_options: str): setup( name="isal", - version="1.0.1", + version="1.1.0", description="Faster zlib and gzip compatible compression and " "decompression by providing python bindings for the ISA-L " "library.", @@ -188,6 +188,7 @@ def build_isa_l(compiler_command: str, compiler_options: str): "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Programming Language :: C", diff --git a/src/isal/__init__.py b/src/isal/__init__.py index e2ef7b9e..bf69f2a0 100644 --- a/src/isal/__init__.py +++ b/src/isal/__init__.py @@ -27,4 +27,4 @@ "__version__" ] -__version__ = "1.0.1" +__version__ = "1.1.0" diff --git a/src/isal/_isalmodule.c b/src/isal/_isalmodule.c index 92b8b317..a429d3a7 100644 --- a/src/isal/_isalmodule.c +++ b/src/isal/_isalmodule.c @@ -1,12 +1,15 @@ -// Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -// 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 -// Python Software Foundation; All Rights Reserved +/* +Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 +Python Software Foundation; All Rights Reserved -// This file is part of python-isal which is distributed under the -// PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2. +This file is part of python-isal which is distributed under the +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2. -// This file is not originally from the CPython distribution. But it does contain mostly example code -// from the Python docs. Also dual licensing just for this one file seemed silly. +This file is not originally from the CPython distribution. But it does +contain mostly example code from the Python docs. Also dual licensing just +for this one file seemed silly. +*/ #define PY_SSIZE_T_CLEAN #include @@ -24,20 +27,19 @@ static struct PyModuleDef _isal_module = { PyMODINIT_FUNC PyInit__isal(void) { - PyObject *m; - - m = PyModule_Create(&_isal_module); - if (m == NULL) + PyObject *m = PyModule_Create(&_isal_module); + if (m == NULL) { return NULL; - + } PyModule_AddIntMacro(m, ISAL_MAJOR_VERSION); PyModule_AddIntMacro(m, ISAL_MINOR_VERSION); PyModule_AddIntMacro(m, ISAL_PATCH_VERSION); PyObject *isal_version = PyUnicode_FromFormat( "%d.%d.%d", ISAL_MAJOR_VERSION, ISAL_MINOR_VERSION, ISAL_PATCH_VERSION); - if (isal_version == NULL) + if (isal_version == NULL) { return NULL; + } PyModule_AddObject(m, "ISAL_VERSION", isal_version); return m; } diff --git a/src/isal/igzip.py b/src/isal/igzip.py index f905483a..ac951690 100644 --- a/src/isal/igzip.py +++ b/src/isal/igzip.py @@ -167,7 +167,7 @@ def __init__(self, filename=None, mode=None, 0) if self.mode == READ: raw = _IGzipReader(self.fileobj) - self._buffer = io.BufferedReader(raw, buffer_size=READ_BUFFER_SIZE) + self._buffer = io.BufferedReader(raw) def __repr__(self): s = repr(self.fileobj) diff --git a/src/isal/igzip_libmodule.c b/src/isal/igzip_libmodule.c index ccb3ab8d..c7c531a0 100644 --- a/src/isal/igzip_libmodule.c +++ b/src/isal/igzip_libmodule.c @@ -1,45 +1,47 @@ -// Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -// 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 -// Python Software Foundation; All Rights Reserved - -// This file is part of python-isal which is distributed under the -// PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2. - -// This file was modified from Cpython Modules/bz2module.c file from the 3.9 -// branch. - -// Changes compared to CPython: -// - The BZ2Decompressor has been used as a basis for IgzipDecompressor. -// Functionality is almost the same. IgzipDecompressor does have a more -// elaborate __init__ to set settings. It also implements decompress_buf more -// akin to how decompression is implemented in isal_shared.h -// - Constants were added that are particular to igzip_lib. -// - Argument parsers were written using th CPython API rather than argument -// clinic. +/* +Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 +Python Software Foundation; All Rights Reserved + +This file is part of python-isal which is distributed under the +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2. + +This file was modified from Cpython Modules/bz2module.c file from the 3.9 +branch. + +Changes compared to CPython: +- The BZ2Decompressor has been used as a basis for IgzipDecompressor. + Functionality is almost the same. IgzipDecompressor does have a more + elaborate __init__ to set settings. It also implements decompress_buf more + akin to how decompression is implemented in isal_shared.h +- Constants were added that are particular to igzip_lib. +- Argument parsers were written using th CPython API rather than argument + clinic. +*/ #include "isal_shared.h" typedef struct { PyObject_HEAD - struct inflate_state state; - char eof; /* T_BOOL expects a char */ PyObject *unused_data; PyObject *zdict; - char needs_input; uint8_t *input_buffer; Py_ssize_t input_buffer_size; - /* inflate_state>avail_in is only 32 bit, so we store the true length separately. Conversion and looping is encapsulated in decompress_buf() */ Py_ssize_t avail_in_real; + char eof; /* T_BOOL expects a char */ + char needs_input; + /* Struct inflate state contains a massive buffer at the end. Put it at + the end of the IgzipDecompressor so members can be accessed easily. */ + struct inflate_state state; } IgzipDecompressor; static void IgzipDecompressor_dealloc(IgzipDecompressor *self) { - if(self->input_buffer != NULL) - PyMem_Free(self->input_buffer); + PyMem_Free(self->input_buffer); Py_CLEAR(self->unused_data); Py_CLEAR(self->zdict); Py_TYPE(self)->tp_free((PyObject *)self); @@ -55,23 +57,23 @@ decompress_buf(IgzipDecompressor *self, Py_ssize_t max_length) /* data_size is strictly positive, but because we repeatedly have to compare against max_length and PyBytes_GET_SIZE we declare it as signed */ - PyObject * RetVal = NULL; + PyObject *RetVal = NULL; Py_ssize_t hard_limit; Py_ssize_t obuflen; int err; - // In Python 3.10 sometimes sys.maxsize is passed by default. In those cases - // we do want to use DEF_BUF_SIZE as start buffer. + /* In Python 3.10 sometimes sys.maxsize is passed by default. In those cases + we do want to use DEF_BUF_SIZE as start buffer. */ if ((max_length < 0) || max_length == PY_SSIZE_T_MAX) { hard_limit = PY_SSIZE_T_MAX; obuflen = DEF_BUF_SIZE; } else { - // Assume that decompressor is used in file decompression with a fixed - // block size of max_length. In that case we will reach max_length almost - // always (except at the end of the file). So it makes sense to allocate - // max_length. + /* Assume that decompressor is used in file decompression with a fixed + block size of max_length. In that case we will reach max_length almost + always (except at the end of the file). So it makes sense to allocate + max_length. */ hard_limit = max_length; obuflen = max_length; if (obuflen > DEF_MAX_INITIAL_BUF_SIZE){ @@ -102,8 +104,10 @@ decompress_buf(IgzipDecompressor *self, Py_ssize_t max_length) isal_inflate_error(err); goto error; } - } while (self->state.avail_out == 0 && self->state.block_state != ISAL_BLOCK_FINISH); - } while(self->avail_in_real != 0 && self->state.block_state != ISAL_BLOCK_FINISH); + } while (self->state.avail_out == 0 && + self->state.block_state != ISAL_BLOCK_FINISH); + } while(self->avail_in_real != 0 && + self->state.block_state != ISAL_BLOCK_FINISH); if (self->state.block_state == ISAL_BLOCK_FINISH) self->eof = 1; @@ -183,13 +187,14 @@ decompress(IgzipDecompressor *self, uint8_t *data, size_t len, Py_ssize_t max_le self->needs_input = 0; Py_ssize_t bytes_in_bitbuffer = bitbuffer_size(&(self->state)); if (self->avail_in_real + bytes_in_bitbuffer > 0) { - PyObject * new_data = PyBytes_FromStringAndSize( + PyObject *new_data = PyBytes_FromStringAndSize( NULL, self->avail_in_real + bytes_in_bitbuffer); if (new_data == NULL) goto error; - char * new_data_ptr = PyBytes_AS_STRING(new_data); + char *new_data_ptr = PyBytes_AS_STRING(new_data); bitbuffer_copy(&(self->state), new_data_ptr, bytes_in_bitbuffer); - memcpy(new_data_ptr + bytes_in_bitbuffer, self->state.next_in, self->avail_in_real); + memcpy(new_data_ptr + bytes_in_bitbuffer, self->state.next_in, + self->avail_in_real); Py_XSETREF(self->unused_data, new_data); } } @@ -208,7 +213,7 @@ decompress(IgzipDecompressor *self, uint8_t *data, size_t len, Py_ssize_t max_le /* Discard buffer if it's too small (resizing it may needlessly copy the current contents) */ if (self->input_buffer != NULL && - self->input_buffer_size < self->avail_in_real) { + self->input_buffer_size < self->avail_in_real) { PyMem_Free(self->input_buffer); self->input_buffer = NULL; } @@ -257,13 +262,14 @@ PyDoc_STRVAR(igzip_lib_compress__doc__, " the header and trailer are controlled by the flag parameter."); #define IGZIP_LIB_COMPRESS_METHODDEF \ - {"compress", (PyCFunction)(void(*)(void))igzip_lib_compress, METH_VARARGS|METH_KEYWORDS, igzip_lib_compress__doc__} + {"compress", (PyCFunction)(void(*)(void))igzip_lib_compress, \ + METH_VARARGS|METH_KEYWORDS, igzip_lib_compress__doc__} static PyObject * igzip_lib_compress(PyObject *module, PyObject *args, PyObject *kwargs) { - char *keywords[] = {"", "level", "flag", "mem_level", "hist_bits", NULL}; - char *format ="y*|iiii:compress"; + static char *keywords[] = {"", "level", "flag", "mem_level", "hist_bits", NULL}; + static char *format ="y*|iiii:compress"; Py_buffer data = {NULL, NULL}; int level = ISAL_DEFAULT_COMPRESSION; int flag = COMP_DEFLATE; @@ -298,13 +304,14 @@ PyDoc_STRVAR(igzip_lib_decompress__doc__, " The initial output buffer size."); #define IGZIP_LIB_DECOMPRESS_METHODDEF \ - {"decompress", (PyCFunction)(void(*)(void))igzip_lib_decompress, METH_VARARGS|METH_KEYWORDS, igzip_lib_decompress__doc__} + {"decompress", (PyCFunction)(void(*)(void))igzip_lib_decompress, \ + METH_VARARGS|METH_KEYWORDS, igzip_lib_decompress__doc__} static PyObject * igzip_lib_decompress(PyObject *module, PyObject *args, PyObject *kwargs) { - char *keywords[] = {"", "flag", "hist_bits", "bufsize", NULL}; - char *format ="y*|iin:decompress"; + static char *keywords[] = {"", "flag", "hist_bits", "bufsize", NULL}; + static char *format ="y*|iin:decompress"; Py_buffer data = {NULL, NULL}; int flag = DECOMP_DEFLATE; int hist_bits = ISAL_DEF_MAX_HIST_BITS; @@ -315,7 +322,7 @@ igzip_lib_decompress(PyObject *module, PyObject *args, PyObject *kwargs) &data, &flag, &hist_bits, &bufsize)) { return NULL; } - PyObject * return_value = igzip_lib_decompress_impl(&data, flag, hist_bits, bufsize); + PyObject *return_value = igzip_lib_decompress_impl(&data, flag, hist_bits, bufsize); PyBuffer_Release(&data); return return_value; } @@ -340,13 +347,16 @@ PyDoc_STRVAR(igzip_lib_IgzipDecompressor_decompress__doc__, "the unused_data attribute."); #define IGZIP_LIB_IGZIPDECOMPRESSOR_DECOMPRESS_METHODDEF \ - {"decompress", (PyCFunction)(void(*)(void))igzip_lib_IgzipDecompressor_decompress, METH_VARARGS|METH_KEYWORDS, igzip_lib_IgzipDecompressor_decompress__doc__} + {"decompress", (PyCFunction)(void(*)(void))igzip_lib_IgzipDecompressor_decompress, \ + METH_VARARGS|METH_KEYWORDS, igzip_lib_IgzipDecompressor_decompress__doc__} static PyObject * -igzip_lib_IgzipDecompressor_decompress(IgzipDecompressor *self, PyObject *args, PyObject *kwargs) +igzip_lib_IgzipDecompressor_decompress(IgzipDecompressor *self, + PyObject *args, + PyObject *kwargs) { - char *keywords[] = {"", "max_length", NULL}; - char *format = "y*|n:decompress"; + static char *keywords[] = {"", "max_length", NULL}; + static char *format = "y*|n:decompress"; Py_buffer data = {NULL, NULL}; Py_ssize_t max_length = -1; @@ -383,8 +393,8 @@ igzip_lib_IgzipDecompressor__new__(PyTypeObject *type, PyObject *args, PyObject *kwargs) { - char *keywords[] = {"flag", "hist_bits", "zdict", NULL}; - char *format = "|iiO:IgzipDecompressor"; + static char *keywords[] = {"flag", "hist_bits", "zdict", NULL}; + static char *format = "|iiO:IgzipDecompressor"; int flag = ISAL_DEFLATE; int hist_bits = ISAL_DEF_MAX_HIST_BITS; PyObject *zdict = NULL; @@ -458,8 +468,9 @@ static PyMemberDef IgzipDecompressor_members[] = { READONLY, IgzipDecompressor_unused_data__doc__}, {"needs_input", T_BOOL, offsetof(IgzipDecompressor, needs_input), READONLY, IgzipDecompressor_needs_input_doc}, - {"crc", T_UINT, offsetof(IgzipDecompressor, state) + offsetof(struct inflate_state, crc), READONLY, - IgzipDecompressor_crc_doc}, + {"crc", T_UINT, + offsetof(IgzipDecompressor, state) + offsetof(struct inflate_state, crc), + READONLY, IgzipDecompressor_crc_doc}, {NULL} }; @@ -561,9 +572,7 @@ static struct PyModuleDef igzip_lib_module = { PyMODINIT_FUNC PyInit_igzip_lib(void) { - PyObject *m; - - m = PyModule_Create(&igzip_lib_module); + PyObject *m = PyModule_Create(&igzip_lib_module); if (m == NULL) return NULL; @@ -580,10 +589,12 @@ PyInit_igzip_lib(void) return NULL; } - if (PyType_Ready(&IgzipDecompressor_Type) != 0) + if (PyType_Ready(&IgzipDecompressor_Type) != 0) { return NULL; + } Py_INCREF(&IgzipDecompressor_Type); - if (PyModule_AddObject(m, "IgzipDecompressor", (PyObject *)&IgzipDecompressor_Type) < 0) { + if (PyModule_AddObject(m, "IgzipDecompressor", + (PyObject *)&IgzipDecompressor_Type) < 0) { return NULL; } diff --git a/src/isal/isal_shared.h b/src/isal/isal_shared.h index c7bb3aa0..bd35f089 100644 --- a/src/isal/isal_shared.h +++ b/src/isal/isal_shared.h @@ -1,23 +1,24 @@ -// Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -// 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 -// Python Software Foundation; All Rights Reserved - -// This file is part of python-isal which is distributed under the -// PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2. - -// This file was modified from Cpython Modules/zlibmodule.c file from the 3.9 -// branch. This is because the BlocksBuffer used in Python 3.10 and higher is -// not available in python 3.7-3.9 which this project supports. - -// Changes compared to CPython: -// - igzip_lib.compress and igzip_lib.decompress are equivalent to -// zlib.compress and zlib.decompress except that these use a 'flag' and -// 'hist_bits' argument to set compression headers and trailers and window -// size respectively. The igzip_lib functions also offer more control by -// allowing to set no header, but include the trailer. -// - This file also includes some utility functions to set parameters on ISA-L -// structs. - +/* +Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 +Python Software Foundation; All Rights Reserved + +This file is part of python-isal which is distributed under the +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2. + +This file was modified from Cpython Modules/zlibmodule.c file from the 3.9 +branch. This is because the BlocksBuffer used in Python 3.10 and higher is +not available in python 3.7-3.9 which this project supports. + +Changes compared to CPython: +- igzip_lib.compress and igzip_lib.decompress are equivalent to + zlib.compress and zlib.decompress except that these use a 'flag' and + 'hist_bits' argument to set compression headers and trailers and window + size respectively. The igzip_lib functions also offer more control by + allowing to set no header, but include the trailer. +- This file also includes some utility functions to set parameters on ISA-L + structs. +*/ #define PY_SSIZE_T_CLEAN #include #include "structmember.h" // PyMemberDef @@ -82,10 +83,11 @@ static const uint32_t LEVEL_BUF_SIZES[24] = { ISAL_DEF_LVL3_EXTRA_LARGE }; -static int mem_level_to_bufsize(int compression_level, int mem_level, - uint32_t * bufsize) +static inline int mem_level_to_bufsize(int compression_level, int mem_level, + uint32_t *bufsize) { - if (compression_level < 0 || compression_level > 3 || mem_level < MEM_LEVEL_DEFAULT || mem_level > MEM_LEVEL_EXTRA_LARGE) { + if (compression_level < 0 || compression_level > 3 || + mem_level < MEM_LEVEL_DEFAULT || mem_level > MEM_LEVEL_EXTRA_LARGE) { *bufsize = 0; return -1; } *bufsize = LEVEL_BUF_SIZES[compression_level * 6 + mem_level]; @@ -94,37 +96,77 @@ static int mem_level_to_bufsize(int compression_level, int mem_level, static void isal_deflate_error(int err) { - const char * msg = NULL; - if (err == COMP_OK) return; - else if (err == INVALID_FLUSH) msg = "Invalid flush type"; - else if (err == INVALID_PARAM) msg = "Invalid parameter"; - else if (err == STATELESS_OVERFLOW) msg = "Not enough room in output buffer"; - else if (err == ISAL_INVALID_OPERATION) msg = "Invalid operation"; - else if (err == ISAL_INVALID_STATE) msg = "Invalid state"; - else if (err == ISAL_INVALID_LEVEL) msg = "Invalid compression level."; - else if (err == ISAL_INVALID_LEVEL_BUF) msg = "Level buffer too small."; - else msg = "Unknown Error"; - + const char *msg = NULL; + switch (err) { + case COMP_OK: return; + case INVALID_FLUSH: + msg = "Invalid flush type"; + break; + case INVALID_PARAM: + msg = "Invalid parameter"; + break; + case STATELESS_OVERFLOW: + msg = "Not enough room in output buffer"; + break; + case ISAL_INVALID_OPERATION: + msg = "Invalid operation"; + break; + case ISAL_INVALID_STATE: + msg = "Invalid state"; + break; + case ISAL_INVALID_LEVEL: + msg = "Invalid compression level."; + break; + case ISAL_INVALID_LEVEL_BUF: + msg = "Level buffer too small."; + break; + default: + msg = "Unknown Error"; + } PyErr_Format(IsalError, "Error %d %s", err, msg); } static void isal_inflate_error(int err){ - const char * msg = NULL; - if (err == ISAL_DECOMP_OK) return; - else if (err == ISAL_END_INPUT) msg = "End of input reached"; - else if (err == ISAL_OUT_OVERFLOW) msg = "End of output reached"; - else if (err == ISAL_NAME_OVERFLOW) msg = "End of gzip name buffer reached"; - else if (err == ISAL_COMMENT_OVERFLOW) msg = "End of gzip comment buffer reached"; - else if (err == ISAL_EXTRA_OVERFLOW) msg = "End of extra buffer reached"; - else if (err == ISAL_NEED_DICT) msg = "Dictionary needed to continue"; - else if (err == ISAL_INVALID_BLOCK) msg = "Invalid deflate block found"; - else if (err == ISAL_INVALID_SYMBOL) msg = "Invalid deflate symbol found"; - else if (err == ISAL_INVALID_LOOKBACK) msg = "Invalid lookback distance found"; - else if (err == ISAL_INVALID_WRAPPER) msg = "Invalid gzip/zlib wrapper found"; - else if (err == ISAL_UNSUPPORTED_METHOD) msg = "Gzip/zlib wrapper specifies unsupported compress method"; - else if (err == ISAL_INCORRECT_CHECKSUM) msg = "Incorrect checksum found"; - else msg = "Unknown error"; - + const char *msg = NULL; + switch (err){ + case ISAL_DECOMP_OK: + return; + case ISAL_END_INPUT: + msg = "End of input reached"; + break; + case ISAL_NAME_OVERFLOW: + msg = "End of gzip name buffer reached"; + break; + case ISAL_COMMENT_OVERFLOW: + msg = "End of gzip comment buffer reached"; + break; + case ISAL_EXTRA_OVERFLOW: + msg = "End of extra buffer reached"; + break; + case ISAL_NEED_DICT: + msg = "Dictionary needed to continue"; + break; + case ISAL_INVALID_BLOCK: + msg = "Invalid deflate block found"; + break; + case ISAL_INVALID_SYMBOL: + msg = "Invalid deflate symbol found"; + break; + case ISAL_INVALID_LOOKBACK: + msg = "Invalid lookback distance found"; + break; + case ISAL_INVALID_WRAPPER: + msg = "Invalid gzip/zlib wrapper found"; + break; + case ISAL_UNSUPPORTED_METHOD: + msg = "Gzip/zlib wrapper specifies unsupported compress method"; + break; + case ISAL_INCORRECT_CHECKSUM: + msg = "Incorrect checksum found"; + break; + default: + msg = "Unknown error"; + } PyErr_Format(IsalError, "Error %d %s", err, msg); } @@ -135,7 +177,7 @@ static void isal_inflate_error(int err){ * @param state An inflate_state * @return size_t */ -static size_t bitbuffer_size(struct inflate_state *state){ +static inline size_t bitbuffer_size(struct inflate_state *state){ return state->read_in_length / 8; } @@ -157,12 +199,12 @@ static int bitbuffer_copy(struct inflate_state *state, char *to, size_t n){ int remainder = bits_in_buffer % 8; // Shift the 8-byte bitbuffer read_in so that the bytes are aligned. uint64_t remaining_bytes = state->read_in >> remainder; - char * remaining_bytes_ptr = (char *)(&remaining_bytes); - memcpy(to, remaining_bytes_ptr, n); + // memcpy works because of little-endianness + memcpy(to, &remaining_bytes, n); return 0; } -static void +static inline void arrange_input_buffer(uint32_t *avail_in, Py_ssize_t *remains) { *avail_in = (uint32_t)Py_MIN((size_t)*remains, UINT32_MAX); @@ -208,7 +250,7 @@ arrange_output_buffer_with_maximum(uint32_t *avail_out, return length; } -static Py_ssize_t +static inline Py_ssize_t arrange_output_buffer(uint32_t *avail_out, uint8_t **next_out, PyObject **buffer, @@ -268,7 +310,8 @@ igzip_lib_compress_impl(Py_buffer *data, else zst.flush = NO_FLUSH; do { - obuflen = arrange_output_buffer(&(zst.avail_out), &(zst.next_out), &RetVal, obuflen); + obuflen = arrange_output_buffer(&(zst.avail_out), &(zst.next_out), + &RetVal, obuflen); if (obuflen < 0) { PyErr_SetString(PyExc_MemoryError, "Unsufficient memory for buffer allocation"); diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index 0f8dbcf8..6b4cf969 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -1,27 +1,28 @@ -// Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -// 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 -// Python Software Foundation; All Rights Reserved - -// This file is part of python-isal which is distributed under the -// PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2. - -// This file was modified from Cpython's Modules/zlibmodule.c file. -// Changes compared to CPython: -// - All zlib naming changed to isal_zlib -// - Including a few constants that are more specific to the ISA-L library -// (ISAL_DEFAULT_COMPRESSION etc). -// - Zlib to ISA-L conversion functions were included. -// - All compression and checksum functions from zlib replaced with ISA-L -// compatible functions. -// - No locks in Compress and Decompress objects. These were deemed unnecessary -// as the ISA-L functions do not allocate memory, unlike the zlib -// counterparts. -// - zlib.compress also has a 'wbits' argument. This change was included in -// Python 3.11. It allows for faster gzip compression by using -// isal_zlib.compress(data, wbits=31). -// - Argument parsers were written using th CPython API rather than argument -// clinic. - +/* +Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 +Python Software Foundation; All Rights Reserved + +This file is part of python-isal which is distributed under the +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2. + +This file was modified from Cpython's Modules/zlibmodule.c file. +Changes compared to CPython: +- All zlib naming changed to isal_zlib +- Including a few constants that are more specific to the ISA-L library + (ISAL_DEFAULT_COMPRESSION etc). +- Zlib to ISA-L conversion functions were included. +- All compression and checksum functions from zlib replaced with ISA-L + compatible functions. +- No locks in Compress and Decompress objects. These were deemed unnecessary + as the ISA-L functions do not allocate memory, unlike the zlib + counterparts. +- zlib.compress also has a 'wbits' argument. This change was included in + Python 3.11. It allows for faster gzip compression by using + isal_zlib.compress(data, wbits=31). +- Argument parsers were written using th CPython API rather than argument + clinic. +*/ #include "isal_shared.h" @@ -109,26 +110,28 @@ static const int ZLIB_MEM_LEVEL_TO_ISAL[10] = { MEM_LEVEL_MEDIUM, // 4-6 -> MEDIUM MEM_LEVEL_MEDIUM, MEM_LEVEL_MEDIUM, - MEM_LEVEL_LARGE, // 7-8 LARGE. The zlib module default = 8. Large is the ISA-L default value. + // 7-8 LARGE. The zlib module default = 8. Large is the ISA-L default value. + MEM_LEVEL_LARGE, MEM_LEVEL_LARGE, MEM_LEVEL_EXTRA_LARGE, // 9 -> EXTRA_LARGE. }; -static int zlib_mem_level_to_isal(int mem_level) { +static inline int zlib_mem_level_to_isal(int mem_level) { if (mem_level < 1 || mem_level > 9) { - PyErr_Format(PyExc_ValueError, - "Invalid mem level: %d. Mem level should be between 1 and 9"); + PyErr_Format( + PyExc_ValueError, + "Invalid mem level: %d. Mem level should be between 1 and 9", + mem_level + ); return -1;} return ZLIB_MEM_LEVEL_TO_ISAL[mem_level]; } -static int +static inline int data_is_gzip(Py_buffer *data){ - if (data->len < 2) - return 0; uint8_t *buf = (uint8_t *)data->buf; - return (buf[0] == 31 && buf[1] == 139); + return (data->len > 1) && (buf[0] == 31) && (buf[1] == 139); } @@ -144,7 +147,8 @@ PyDoc_STRVAR(isal_zlib_adler32__doc__, "The returned checksum is an integer."); #define ISAL_ZLIB_ADLER32_METHODDEF \ - {"adler32", (PyCFunction)(void(*)(void))isal_zlib_adler32, METH_FASTCALL, isal_zlib_adler32__doc__} + {"adler32", (PyCFunction)(void(*)(void))isal_zlib_adler32, METH_FASTCALL, \ + isal_zlib_adler32__doc__} static PyObject * isal_zlib_adler32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) @@ -188,7 +192,8 @@ PyDoc_STRVAR(zlib_crc32__doc__, "The returned checksum is an integer."); #define ISAL_ZLIB_CRC32_METHODDEF \ - {"crc32", (PyCFunction)(void(*)(void))isal_zlib_crc32, METH_FASTCALL, zlib_crc32__doc__} + {"crc32", (PyCFunction)(void(*)(void))isal_zlib_crc32, METH_FASTCALL, \ + zlib_crc32__doc__} static PyObject * isal_zlib_crc32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) @@ -233,13 +238,14 @@ PyDoc_STRVAR(zlib_compress__doc__, " The window buffer size and container format."); #define ISAL_ZLIB_COMPRESS_METHODDEF \ - {"compress", (PyCFunction)(void(*)(void))isal_zlib_compress, METH_VARARGS|METH_KEYWORDS, zlib_compress__doc__} + {"compress", (PyCFunction)(void(*)(void))isal_zlib_compress, \ + METH_VARARGS|METH_KEYWORDS, zlib_compress__doc__} static PyObject * isal_zlib_compress(PyObject *module, PyObject *args, PyObject *kwargs) { - char *keywords[] = {"", "level", "wbits", NULL}; - char *format ="y*|ii:isal_zlib.compress"; + static char *keywords[] = {"", "level", "wbits", NULL}; + static char *format ="y*|ii:isal_zlib.compress"; Py_buffer data = {NULL, NULL}; int level = ISAL_DEFAULT_COMPRESSION; int wbits = ISAL_DEF_MAX_HIST_BITS; @@ -276,15 +282,16 @@ PyDoc_STRVAR(zlib_decompress__doc__, " The initial output buffer size."); #define ISAL_ZLIB_DECOMPRESS_METHODDEF \ - {"decompress", (PyCFunction)(void(*)(void))isal_zlib_decompress, METH_VARARGS|METH_KEYWORDS, zlib_decompress__doc__} + {"decompress", (PyCFunction)(void(*)(void))isal_zlib_decompress, \ + METH_VARARGS|METH_KEYWORDS, zlib_decompress__doc__} static PyObject * isal_zlib_decompress(PyObject *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - char *keywords[] = {"", "wbits", "bufsize", NULL}; - char *format ="y*|in:isal_zlib.decompress"; + static char *keywords[] = {"", "wbits", "bufsize", NULL}; + static char *format ="y*|in:isal_zlib.decompress"; Py_buffer data = {NULL, NULL}; int wbits = ISAL_DEF_MAX_HIST_BITS; Py_ssize_t bufsize = DEF_BUF_SIZE; @@ -315,10 +322,11 @@ isal_zlib_decompress(PyObject *module, PyObject *args, PyObject *kwargs) typedef struct { PyObject_HEAD - struct isal_zstream zst; - int is_initialised; - uint8_t * level_buf; + uint8_t *level_buf; PyObject *zdict; + int is_initialised; + // isal_zstream should be at the bottom as it contains buffers inside the struct. + struct isal_zstream zst; } compobject; static void @@ -347,13 +355,14 @@ newcompobject() typedef struct { PyObject_HEAD - struct inflate_state zst; PyObject *unused_data; PyObject *unconsumed_tail; - char eof; + PyObject *zdict; int is_initialised; int method_set; - PyObject *zdict; + char eof; + // inflate_state should be at the bottom as it contains buffers inside the struct. + struct inflate_state zst; } decompobject; static void @@ -433,7 +442,8 @@ isal_zlib_compressobj_impl(PyObject *module, int level, int method, int wbits, if (strategy != Z_DEFAULT_STRATEGY){ err = PyErr_WarnEx( PyExc_UserWarning, - "Only one strategy is supported when using isal_zlib. Using the default strategy.", + "Only one strategy is supported when using isal_zlib. Using the " + "default strategy.", 1); if (err == -1) // Warning was turned into an exception. @@ -454,7 +464,8 @@ isal_zlib_compressobj_impl(PyObject *module, int level, int method, int wbits, if (mem_level_to_bufsize( level, isal_mem_level, &level_buf_size) == -1) { PyErr_Format(PyExc_ValueError, - "Invalid compression level: %d. Compression level should be between 0 and 3", + "Invalid compression level: %d. Compression level should " + "be between 0 and 3", level); goto error; } @@ -606,7 +617,7 @@ save_unconsumed_input(decompobject *self, Py_buffer *data, int err) new_data = PyBytes_FromStringAndSize(NULL, new_size); if (new_data == NULL) return -1; - char * new_data_ptr = PyBytes_AS_STRING(new_data); + char *new_data_ptr = PyBytes_AS_STRING(new_data); memcpy(new_data_ptr, PyBytes_AS_STRING(self->unused_data), old_size); bitbuffer_copy(&(self->zst), new_data_ptr + old_size, bytes_in_bitbuffer); @@ -688,7 +699,8 @@ isal_zlib_Decompress_decompress_impl(decompobject *self, Py_buffer *data, goto abort; } - } while (self->zst.avail_out == 0 && self->zst.block_state != ISAL_BLOCK_FINISH); + } while (self->zst.avail_out == 0 && + self->zst.block_state != ISAL_BLOCK_FINISH); } while (self->zst.block_state != ISAL_BLOCK_FINISH && ibuflen != 0); @@ -800,7 +812,8 @@ isal_zlib_Decompress_flush_impl(decompobject *self, Py_ssize_t length) do { length = arrange_output_buffer(&(self->zst.avail_out), - &(self->zst.next_out), &RetVal, length); + &(self->zst.next_out), + &RetVal, length); if (length < 0) goto abort; @@ -811,7 +824,8 @@ isal_zlib_Decompress_flush_impl(decompobject *self, Py_ssize_t length) goto abort; } - } while (self->zst.avail_out == 0 && self->zst.block_state != ISAL_BLOCK_FINISH); + } while (self->zst.avail_out == 0 && + self->zst.block_state != ISAL_BLOCK_FINISH); } while (self->zst.block_state != ISAL_BLOCK_FINISH && ibuflen != 0); @@ -866,14 +880,16 @@ PyDoc_STRVAR(isal_zlib_compressobj__doc__, " containing subsequences that are likely to occur in the input data."); #define ISAL_ZLIB_COMPRESSOBJ_METHODDEF \ - {"compressobj", (PyCFunction)(void(*)(void))isal_zlib_compressobj, METH_VARARGS|METH_KEYWORDS, isal_zlib_compressobj__doc__} + {"compressobj", (PyCFunction)(void(*)(void))isal_zlib_compressobj, \ + METH_VARARGS|METH_KEYWORDS, isal_zlib_compressobj__doc__} static PyObject * isal_zlib_compressobj(PyObject *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - char *keywords[] = {"level", "method", "wbits", "memLevel", "strategy", "zdict", NULL}; - char *format = "|iiiiiy*:compressobj"; + static char *keywords[] = {"level", "method", "wbits", "memLevel", + "strategy", "zdict", NULL}; + static char *format = "|iiiiiy*:compressobj"; int level = ISAL_DEFAULT_COMPRESSION; int method = Z_DEFLATED; int wbits = ISAL_DEF_MAX_HIST_BITS; @@ -886,7 +902,8 @@ isal_zlib_compressobj(PyObject *module, PyObject *args, PyObject *kwargs) &level, &method, &wbits, &memLevel, &strategy, &zdict)) { return NULL; } - return_value = isal_zlib_compressobj_impl(module, level, method, wbits, memLevel, strategy, &zdict); + return_value = isal_zlib_compressobj_impl(module, level, method, wbits, + memLevel, strategy, &zdict); PyBuffer_Release(&zdict); return return_value; } @@ -904,13 +921,14 @@ PyDoc_STRVAR(isal_zlib_decompressobj__doc__, " dictionary as used by the compressor that produced the input data."); #define ISAL_ZLIB_DECOMPRESSOBJ_METHODDEF \ - {"decompressobj", (PyCFunction)(void(*)(void))isal_zlib_decompressobj, METH_VARARGS|METH_KEYWORDS, isal_zlib_decompressobj__doc__} + {"decompressobj", (PyCFunction)(void(*)(void))isal_zlib_decompressobj, \ + METH_VARARGS|METH_KEYWORDS, isal_zlib_decompressobj__doc__} static PyObject * isal_zlib_decompressobj(PyObject *module, PyObject *args, PyObject *kwargs) { - char *keywords[] = {"wbits", "zdict", NULL}; - char *format = "|iO:decompressobj"; + static char *keywords[] = {"wbits", "zdict", NULL}; + static char *format = "|iO:decompressobj"; int wbits = ISAL_DEF_MAX_HIST_BITS; PyObject *zdict = NULL; @@ -936,7 +954,8 @@ PyDoc_STRVAR(isal_zlib_Compress_compress__doc__, "Call the flush() method to clear these buffers."); #define ISAL_ZLIB_COMPRESS_COMPRESS_METHODDEF \ - {"compress", (PyCFunction)(void(*)(void))isal_zlib_Compress_compress, METH_O, isal_zlib_Compress_compress__doc__} + {"compress", (PyCFunction)(void(*)(void))isal_zlib_Compress_compress, \ + METH_O, isal_zlib_Compress_compress__doc__} static PyObject * @@ -969,14 +988,15 @@ PyDoc_STRVAR(isal_zlib_Decompress_decompress__doc__, "Call the flush() method to clear these buffers."); #define ISAL_ZLIB_DECOMPRESS_DECOMPRESS_METHODDEF \ - {"decompress", (PyCFunction)(void(*)(void))isal_zlib_Decompress_decompress, METH_VARARGS|METH_KEYWORDS, isal_zlib_Decompress_decompress__doc__} + {"decompress", (PyCFunction)(void(*)(void))isal_zlib_Decompress_decompress, \ + METH_VARARGS|METH_KEYWORDS, isal_zlib_Decompress_decompress__doc__} static PyObject * isal_zlib_Decompress_decompress(decompobject *self, PyObject *args, PyObject *kwargs) { - char *keywords[] = {"", "max_length", NULL}; - char *format = "y*|n:decompress"; + static char *keywords[] = {"", "max_length", NULL}; + static char *format = "y*|n:decompress"; Py_buffer data = {NULL, NULL}; Py_ssize_t max_length = 0; @@ -984,7 +1004,8 @@ isal_zlib_Decompress_decompress(decompobject *self, PyObject *args, PyObject *kw args, kwargs, format, keywords, &data, &max_length)) { return NULL; } - PyObject *return_value = isal_zlib_Decompress_decompress_impl(self, &data, max_length); + PyObject *return_value = isal_zlib_Decompress_decompress_impl(self, &data, + max_length); PyBuffer_Release(&data); return return_value; } @@ -1002,11 +1023,15 @@ PyDoc_STRVAR(isal_zlib_Compress_flush__doc__, " can still be compressed."); #define ISAL_ZLIB_COMPRESS_FLUSH_METHODDEF \ - {"flush", (PyCFunction)(void(*)(void))isal_zlib_Compress_flush, METH_FASTCALL|METH_KEYWORDS, isal_zlib_Compress_flush__doc__} + {"flush", (PyCFunction)(void(*)(void))isal_zlib_Compress_flush, \ + METH_FASTCALL|METH_KEYWORDS, isal_zlib_Compress_flush__doc__} static PyObject * -isal_zlib_Compress_flush(compobject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +isal_zlib_Compress_flush(compobject *self, + PyObject *const *args, + Py_ssize_t nargs, + PyObject *kwnames) { Py_ssize_t mode; if (nargs == 0) { @@ -1045,7 +1070,8 @@ PyDoc_STRVAR(isal_zlib_Decompress_flush__doc__, #define ISAL_ZLIB_DECOMPRESS_FLUSH_METHODDEF \ - {"flush", (PyCFunction)(void(*)(void))isal_zlib_Decompress_flush, METH_FASTCALL, isal_zlib_Decompress_flush__doc__} + {"flush", (PyCFunction)(void(*)(void))isal_zlib_Decompress_flush, \ + METH_FASTCALL, isal_zlib_Decompress_flush__doc__} static PyObject * isal_zlib_Decompress_flush(decompobject *self, PyObject *const *args, Py_ssize_t nargs) @@ -1171,11 +1197,10 @@ static struct PyModuleDef isal_zlib_module = { PyMODINIT_FUNC PyInit_isal_zlib(void) { - PyObject *m; - - m = PyModule_Create(&isal_zlib_module); - if (m == NULL) + PyObject *m = PyModule_Create(&isal_zlib_module); + if (m == NULL) { return NULL; + } PyObject *igzip_lib_module = PyImport_ImportModule("isal.igzip_lib"); if (igzip_lib_module == NULL) { @@ -1220,7 +1245,6 @@ PyInit_isal_zlib(void) PyModule_AddIntConstant(m, "DEFLATED", Z_DEFLATED); PyModule_AddIntMacro(m, DEF_MEM_LEVEL); PyModule_AddIntMacro(m, DEF_BUF_SIZE); - // compression levels // No compression is not supported by ISA-L. Throw an error if chosen. // PyModule_AddIntMacro(m, Z_NO_COMPRESSION); PyModule_AddIntConstant(m, "Z_BEST_SPEED", ISAL_DEF_MIN_LEVEL);