From 4f414bc1cfb26b17aa3b7e459b006157fcda7175 Mon Sep 17 00:00:00 2001 From: Sergey Dryabzhinsky Date: Thu, 23 Mar 2023 20:08:51 +0300 Subject: [PATCH 1/3] Don't fail if number of workers bigger than zstd can take --- src/python-zstd.c | 6 ++++-- src/python-zstd.h | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/python-zstd.c b/src/python-zstd.c index 75ffceb..2fea2cb 100644 --- a/src/python-zstd.c +++ b/src/python-zstd.c @@ -93,8 +93,10 @@ static PyObject *py_zstd_compress_mt(PyObject* self, PyObject *args) if (0 == threads) threads = UTIL_countPhysicalCores(); /* If threads more than 200 - raise Error. */ if (threads > ZSTDMT_NBWORKERS_MAX) { - PyErr_Format(ZstdError, "Bad threads count - more than %d: %d", ZSTDMT_NBWORKERS_MAX, threads); - return NULL; + threads = ZSTDMT_NBWORKERS_MAX; + // do not fail here, due auto thread counter + //PyErr_Format(ZstdError, "Bad threads count - more than %d: %d", ZSTDMT_NBWORKERS_MAX, threads); + //return NULL; } dest_size = (Py_ssize_t)ZSTD_compressBound(source_size); diff --git a/src/python-zstd.h b/src/python-zstd.h index 71557bc..87cb3f3 100644 --- a/src/python-zstd.h +++ b/src/python-zstd.h @@ -58,7 +58,7 @@ #endif #ifndef ZSTDMT_NBWORKERS_MAX -#define ZSTDMT_NBWORKERS_MAX 200 +#define ZSTDMT_NBWORKERS_MAX ((sizeof(void*)==4) /*32-bit*/ ? 64 : 256) #endif /* --== Negative fast compression levels only since 1.3.4 ==-- */ From 7b4b17b09630ed01c0494a6a31b38caef408c879 Mon Sep 17 00:00:00 2001 From: Sergey Dryabzhinsky Date: Thu, 23 Mar 2023 20:26:35 +0300 Subject: [PATCH 2/3] Add two functions to get available number of threads Prepare for 1.5.4-r1 version --- PKG-INFO | 4 ++-- README.rst | 13 +++++++++++++ setup.py | 2 +- src/python-zstd.c | 21 +++++++++++++++++++++ src/python-zstd.h | 2 ++ tests/base.py | 2 +- 6 files changed, 40 insertions(+), 4 deletions(-) diff --git a/PKG-INFO b/PKG-INFO index fd8b29e..424f1f7 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,12 +1,12 @@ Metadata-Version: 1.1 Name: zstd -Version: 1.5.4.0 +Version: 1.5.4.1 Summary: Simple python bindings to Yann Collet ZSTD compression library Home-page: https://github.com/sergey-dryabzhinsky/python-zstd Author: Sergey Dryabzhinsky Author-email: sergey.dryabzhinsky@gmail.com License: BSD -Download-URL: https://github.com/sergey-dryabzhinsky/python-zstd/archive/v1.5.4.0.tar.gz +Download-URL: https://github.com/sergey-dryabzhinsky/python-zstd/archive/v1.5.4.1.tar.gz Description: Simple ZSTandarD bindings for Python Keywords: zstd,zstandard,compression Platform: POSIX diff --git a/README.rst b/README.rst index f17d48b..a3d41bf 100644 --- a/README.rst +++ b/README.rst @@ -130,6 +130,9 @@ ZSTD_compress (data[, level, threads]): string|bytes Aliases: *compress(...)*, *dumps(...)* + Exception if: + - level bigger than max level + Since: 0.1 ZSTD_uncompress (data): string|bytes @@ -161,6 +164,16 @@ ZSTD_version_number (): int Since: 1.3.4.3 +ZSTD_threads_count (): int + Returns ZSTD determined CPU cores count. + + Since: 1.5.4.1 + +ZSTD_max_threads_count (): int + Returns ZSTD library determined maximum working threads count. + + Since: 1.5.4.1 + ZSTD_external (): int Returns 0 of 1 if ZSTD library build as external. diff --git a/setup.py b/setup.py index 98de726..f39e922 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ # Package version PKG_VERSION = VERSION # Minor versions -PKG_VERSION += ("0",) +PKG_VERSION += ("1",) PKG_VERSION_STR = ".".join([str(x) for x in PKG_VERSION]) ### diff --git a/src/python-zstd.c b/src/python-zstd.c index 2fea2cb..b418178 100644 --- a/src/python-zstd.c +++ b/src/python-zstd.c @@ -224,6 +224,25 @@ static PyObject *py_zstd_library_external(PyObject* self, PyObject *args) } +/** + * Returns ZSTD determined threads count, int + */ +static PyObject *py_zstd_threads_count(PyObject* self, PyObject *args) +{ + int32_t threads = UTIL_countPhysicalCores(); + + return Py_BuildValue("i", threads); +} + +/** + * Returns ZSTD determined max threads count, int + */ +static PyObject *py_zstd_max_threads_count(PyObject* self, PyObject *args) +{ + return Py_BuildValue("i", ZSTDMT_NBWORKERS_MAX); +} + + static PyMethodDef ZstdMethods[] = { {"ZSTD_compress", py_zstd_compress_mt, METH_VARARGS, COMPRESS_DOCSTRING}, {"ZSTD_uncompress", py_zstd_uncompress, METH_VARARGS, UNCOMPRESS_DOCSTRING}, @@ -235,6 +254,8 @@ static PyMethodDef ZstdMethods[] = { {"version", py_zstd_module_version, METH_NOARGS, VERSION_DOCSTRING}, {"ZSTD_version", py_zstd_library_version, METH_NOARGS, ZSTD_VERSION_DOCSTRING}, {"ZSTD_version_number", py_zstd_library_version_int, METH_NOARGS, ZSTD_INT_VERSION_DOCSTRING}, + {"ZSTD_threads_count", py_zstd_threads_count, METH_NOARGS, ZSTD_THREADS_COUNT_DOCSTRING}, + {"ZSTD_max_threads_count", py_zstd_max_threads_count, METH_NOARGS, ZSTD_MAX_THREADS_COUNT_DOCSTRING}, {"ZSTD_external", py_zstd_library_external, METH_NOARGS, ZSTD_EXTERNAL_DOCSTRING}, {NULL, NULL, 0, NULL} }; diff --git a/src/python-zstd.h b/src/python-zstd.h index 87cb3f3..05b0ae7 100644 --- a/src/python-zstd.h +++ b/src/python-zstd.h @@ -108,6 +108,8 @@ Raises a zstd.Error exception if any error occurs." #define ZSTD_VERSION_DOCSTRING "ZSTD_version(): string -- Returns ZSTD library version as string." #define ZSTD_INT_VERSION_DOCSTRING "ZSTD_version_number(): int -- Returns ZSTD library version as integer.\n Format of the number is: major * 100*100 + minor * 100 + release." #define ZSTD_EXTERNAL_DOCSTRING "ZSTD_external(): int -- Returns 0 or 1 if ZSTD library build as external." +#define ZSTD_THREADS_COUNT_DOCSTRING "ZSTD_threads_count(): int -- Returns ZSTD determined CPU cores count in integer." +#define ZSTD_MAX_THREADS_COUNT_DOCSTRING "ZSTD_max_threads_count(): int -- Returns ZSTD library determined maximum working threads count in integer." #if PYZSTD_LEGACY > 0 #define COMPRESS_OLD_DOCSTRING "compress_old(string[, level]): "PY_BYTESTR_TYPE" -- Compress string, old version, returning the compressed data.\n\nUses custom format. Not compatible with streaming or origin compression tools.\n\nRaises a zstd.Error exception if any error occurs.\n\n@deprecated" diff --git a/tests/base.py b/tests/base.py index 2006866..043477a 100644 --- a/tests/base.py +++ b/tests/base.py @@ -43,7 +43,7 @@ class BaseTestZSTD(unittest.TestCase): VERSION = "1.5.4" VERSION_INT = 10504 VERSION_INT_MIN = 1 * 100*100 + 0 * 1*100 + 0 - PKG_VERSION = "1.5.4.0" + PKG_VERSION = "1.5.4.1" def helper_version(self): self.assertEqual(self.PKG_VERSION, zstd.version()) From e43012b3a2c1ba1a34afd94b5bbab1f094f2848e Mon Sep 17 00:00:00 2001 From: Sergey Dryabzhinsky Date: Thu, 23 Mar 2023 20:31:46 +0300 Subject: [PATCH 3/3] Update readme --- README.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.rst b/README.rst index a3d41bf..357834a 100644 --- a/README.rst +++ b/README.rst @@ -133,6 +133,11 @@ ZSTD_compress (data[, level, threads]): string|bytes Exception if: - level bigger than max level + Max number of threads: + - 32bit system: 64 + - 64bit system: 256 + If provided bigger number - siletly set maximum number (since 1.5.4.1) + Since: 0.1 ZSTD_uncompress (data): string|bytes