Merge pull request #93 from sergey-dryabzhinsky/issue-92-threads-uppe…

…r-limit

Don't fail if number of workers bigger than zstd can take

PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 1.1
 Name: zstd
-Version: 1.5.4.0
+Version: 1.5.4.1
 Summary: Simple python bindings to Yann Collet ZSTD compression library
 Home-page: https://github.com/sergey-dryabzhinsky/python-zstd
 Author: Sergey Dryabzhinsky
 Author-email: sergey.dryabzhinsky@gmail.com
 License: BSD
-Download-URL: https://github.com/sergey-dryabzhinsky/python-zstd/archive/v1.5.4.0.tar.gz
+Download-URL: https://github.com/sergey-dryabzhinsky/python-zstd/archive/v1.5.4.1.tar.gz
 Description: Simple ZSTandarD bindings for Python
 Keywords: zstd,zstandard,compression
 Platform: POSIX

README.rst

@@ -130,6 +130,14 @@ ZSTD_compress (data[, level, threads]): string|bytes
 
   Aliases: *compress(...)*, *dumps(...)*
 
+  Exception if:
+  - level bigger than max level
+
+  Max number of threads:
+  - 32bit system: 64
+  - 64bit system: 256
+  If provided bigger number - siletly set maximum number (since 1.5.4.1)
+
   Since: 0.1
 
 ZSTD_uncompress (data): string|bytes
@@ -161,6 +169,16 @@ ZSTD_version_number (): int
 
   Since: 1.3.4.3
 
+ZSTD_threads_count (): int
+  Returns ZSTD determined CPU cores count.
+
+  Since: 1.5.4.1
+
+ZSTD_max_threads_count (): int
+  Returns ZSTD library determined maximum working threads count.
+
+  Since: 1.5.4.1
+
 ZSTD_external (): int
   Returns 0 of 1 if ZSTD library build as external.
 

setup.py

@@ -14,7 +14,7 @@
 # Package version
 PKG_VERSION = VERSION
 # Minor versions
-PKG_VERSION += ("0",)
+PKG_VERSION += ("1",)
 PKG_VERSION_STR = ".".join([str(x) for x in PKG_VERSION])
 
 ###

src/python-zstd.c

@@ -93,8 +93,10 @@ static PyObject *py_zstd_compress_mt(PyObject* self, PyObject *args)
     if (0 == threads) threads = UTIL_countPhysicalCores();
     /* If threads more than 200 - raise Error. */
     if (threads > ZSTDMT_NBWORKERS_MAX) {
-        PyErr_Format(ZstdError, "Bad threads count - more than %d: %d", ZSTDMT_NBWORKERS_MAX, threads);
-        return NULL;
+        threads = ZSTDMT_NBWORKERS_MAX;
+        // do not fail here, due auto thread counter
+        //PyErr_Format(ZstdError, "Bad threads count - more than %d: %d", ZSTDMT_NBWORKERS_MAX, threads);
+        //return NULL;
     }
 
     dest_size = (Py_ssize_t)ZSTD_compressBound(source_size);
@@ -222,6 +224,25 @@ static PyObject *py_zstd_library_external(PyObject* self, PyObject *args)
 }
 
 
+/**
+ * Returns ZSTD determined threads count, int
+ */
+static PyObject *py_zstd_threads_count(PyObject* self, PyObject *args)
+{
+    int32_t threads = UTIL_countPhysicalCores();
+
+    return Py_BuildValue("i", threads);
+}
+
+/**
+ * Returns ZSTD determined max threads count, int
+ */
+static PyObject *py_zstd_max_threads_count(PyObject* self, PyObject *args)
+{
+    return Py_BuildValue("i", ZSTDMT_NBWORKERS_MAX);
+}
+
+
 static PyMethodDef ZstdMethods[] = {
     {"ZSTD_compress",  py_zstd_compress_mt, METH_VARARGS, COMPRESS_DOCSTRING},
     {"ZSTD_uncompress",  py_zstd_uncompress, METH_VARARGS, UNCOMPRESS_DOCSTRING},
@@ -233,6 +254,8 @@ static PyMethodDef ZstdMethods[] = {
     {"version",  py_zstd_module_version, METH_NOARGS, VERSION_DOCSTRING},
     {"ZSTD_version",  py_zstd_library_version, METH_NOARGS, ZSTD_VERSION_DOCSTRING},
     {"ZSTD_version_number",  py_zstd_library_version_int, METH_NOARGS, ZSTD_INT_VERSION_DOCSTRING},
+    {"ZSTD_threads_count",  py_zstd_threads_count, METH_NOARGS, ZSTD_THREADS_COUNT_DOCSTRING},
+    {"ZSTD_max_threads_count",  py_zstd_max_threads_count, METH_NOARGS, ZSTD_MAX_THREADS_COUNT_DOCSTRING},
     {"ZSTD_external",  py_zstd_library_external, METH_NOARGS, ZSTD_EXTERNAL_DOCSTRING},
     {NULL, NULL, 0, NULL}
 };

src/python-zstd.h

@@ -58,7 +58,7 @@
 #endif
 
 #ifndef ZSTDMT_NBWORKERS_MAX
-#define ZSTDMT_NBWORKERS_MAX 200
+#define ZSTDMT_NBWORKERS_MAX ((sizeof(void*)==4) /*32-bit*/ ? 64 : 256)
 #endif
 
 /* --== Negative fast compression levels only since 1.3.4 ==-- */
@@ -108,6 +108,8 @@ Raises a zstd.Error exception if any error occurs."
 #define ZSTD_VERSION_DOCSTRING  "ZSTD_version(): string -- Returns ZSTD library version as string."
 #define ZSTD_INT_VERSION_DOCSTRING  "ZSTD_version_number(): int -- Returns ZSTD library version as integer.\n Format of the number is: major * 100*100 + minor * 100 + release."
 #define ZSTD_EXTERNAL_DOCSTRING  "ZSTD_external(): int -- Returns 0 or 1 if ZSTD library build as external."
+#define ZSTD_THREADS_COUNT_DOCSTRING  "ZSTD_threads_count(): int -- Returns ZSTD determined CPU cores count in integer."
+#define ZSTD_MAX_THREADS_COUNT_DOCSTRING  "ZSTD_max_threads_count(): int -- Returns ZSTD library determined maximum working threads count in integer."
 
 #if PYZSTD_LEGACY > 0
 #define COMPRESS_OLD_DOCSTRING      "compress_old(string[, level]): "PY_BYTESTR_TYPE" -- Compress string, old version, returning the compressed data.\n\nUses custom format. Not compatible with streaming or origin compression tools.\n\nRaises a zstd.Error exception if any error occurs.\n\n@deprecated"

tests/base.py

@@ -43,7 +43,7 @@ class BaseTestZSTD(unittest.TestCase):
     VERSION = "1.5.4"
     VERSION_INT = 10504
     VERSION_INT_MIN = 1 * 100*100 + 0 * 1*100 + 0
-    PKG_VERSION = "1.5.4.0"
+    PKG_VERSION = "1.5.4.1"
 
     def helper_version(self):
         self.assertEqual(self.PKG_VERSION, zstd.version())