Improved thread testing

libpymath.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: libpymath
-Version: 0.1.4
+Version: 0.1.5
 Summary: A general purpose Python math module
 Home-page: https://www.github.com/pencilcaseman/gpc
 Author: Toby Davis

libpymath.egg-info/SOURCES.txt

@@ -13,6 +13,8 @@ libpymath/LibPyMathModules/matrix/__init__.py
 libpymath/LibPyMathModules/matrix/doubleFunctions.h
 libpymath/LibPyMathModules/matrix/matrixModule.c
 libpymath/matrix/__init__.py
+libpymath/matrix/_threadInfo.py
+libpymath/matrix/_threadSetup.py
 libpymath/matrix/matrix.py
 libpymath/testModule/__init__.py
 libpymath/testModule/testScript.py

libpymath/LibPyMathModules/matrix/matrixModule.c

@@ -19,7 +19,7 @@ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */
 
-// #define PY_SSIZE_T_CLEAN
+#define PY_SSIZE_T_CLEAN
 
 #include <Python.h>
 #include <structmember.h>

libpymath/matrix/_threadInfo.py

@@ -0,0 +1,4 @@
+LPM_CORES = 4
+LPM_THREADS = 32
+LPM_IS_HYPERTHREADING = True
+LPM_OPTIMAL_MATRIX_THREADS = 5

libpymath/matrix/_threadSetup.py

@@ -0,0 +1,71 @@
+
+import sys
+import os
+from libpymath.core import matrix as _matrix
+from time import time
+import math
+
+# Min 4 cores for thread test purposes -- speed may increase with more cores
+LPM_CORES = min(os.cpu_count(), 4)
+LPM_THREADS = 4
+try:
+    if sys.platform == 'win32':
+        LPM_THREADS = (int)(os.environ['NUMBER_OF_PROCESSORS'])
+    else:
+        LPM_THREADS = (int)(os.popen('grep -c cores /proc/cpuinfo').read())
+except:
+    print("Number of threads available unknown. Assuming 4")
+
+LPM_IS_HYPERTHREADING = LPM_CORES != LPM_THREADS
+
+# Find the optimal number of threads to use
+def _lpmFindOptimalMatrixThreads(n = 1000, verbose=False):
+    termWidth = os.get_terminal_size().columns
+
+    fastTime = 99999999999999
+    fastThreads = 0
+
+    for i in range(1, LPM_THREADS + 1):
+        # Create a matrix
+        mat = _matrix.Matrix(1000, 1000)
+
+        dt = 0
+        maxTime = 0
+        minTime = 99999999999999
+        for j in range(n):
+            start = time()
+            res = mat.matrixAddMatrixReturn(mat, i)
+            end = time()
+            t = end - start
+
+            if t < minTime:
+                minTime = t
+
+            if t > maxTime:
+                maxTime = t
+
+            dt += t
+
+            progLen = termWidth - 31
+            inc = math.ceil(n / progLen) # n // progLen
+
+            if verbose:
+                print("Testing thread {}   [{}{}]\r".format(str(i).rjust(5), "#" * math.ceil(j / inc), " " * (math.ceil(n / inc) - math.ceil(j / inc))), end="")
+                sys.stdout.flush()
+
+        if dt < fastTime:
+            fastTime = dt
+            fastThreads = i
+
+    if verbose:
+        print(" ".ljust(termWidth - 5), end="")
+
+    return fastThreads
+
+LPM_OPTIMAL_MATRIX_THREADS = _lpmFindOptimalMatrixThreads(n=500, verbose=True)
+
+with open("_threadInfo.py", "w") as out:
+    out.write("LPM_CORES = {}\n".format(LPM_CORES))
+    out.write("LPM_THREADS = {}\n".format(LPM_THREADS))
+    out.write("LPM_IS_HYPERTHREADING = {}\n".format(LPM_IS_HYPERTHREADING))
+    out.write("LPM_OPTIMAL_MATRIX_THREADS = {}\n".format(LPM_OPTIMAL_MATRIX_THREADS))

libpymath/matrix/matrix.py

@@ -19,97 +19,25 @@
 CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 """
 
+from . import _threadSetup
+from . import _threadInfo
 import libpymath.core.matrix as _matrix
 import sys
 import os
 from time import time
 
-
-# Min 4 cores for thread test purposes -- speed may increase with more cores
-lpmCores = min(os.cpu_count(), 4)
-lpmThreads = 4
-try:
-    if sys.platform == 'win32':
-        lpmThreads = (int)(os.environ['NUMBER_OF_PROCESSORS'])
-    else:
-        lpmThreads = (int)(os.popen('grep -c cores /proc/cpuinfo').read())
-except:
-    print("Number of threads available unknown. Assuming 4")
-
-lpmHyperthreading = lpmCores != lpmThreads
-
-# Find the optimal number of threads to use
-def _optimalThreads(verbose=False):
-    if verbose:
-        print("Testing speed using {} through {} threads".format(1, lpmThreads))
-
-    fastTime = 99999999999999
-    fastThreads = 0
-
-    for i in range(1, lpmThreads + 1):
-        # Create a matrix
-        mat = _matrix.Matrix(1000, 1000)
-
-        start = time()
-        for j in range(10):
-            res = mat.matrixAddMatrixReturn(mat, i)
-        end = time()
-        dt = end - start
-
-        if verbose:
-            print("{} threads >> {} seconds".format(i, dt))
-
-        if dt < fastTime:
-            fastTime = dt
-            fastThreads = i
-
-    return fastThreads
-
-lpmDefaultThreads = _optimalThreads(verbose=False)
-
-def _checkMultiprocessing():
-    global lpmThreads
-    global lpmCores
-    global lpmDefaultThreads
-    if isinstance(lpmThreads, int):
-        if lpmThreads < 1:
-            lpmThreads = 1
-    else:
-        try:
-            if sys.platform == 'win32':
-                lpmThreads = (int)(os.environ['NUMBER_OF_PROCESSORS'])
-            else:
-                lpmThreads = (int)(os.popen('grep -c cores /proc/cpuinfo').read())
-        except:
-            print("Number of threads available unknown. Assuming 4")
-
-    if isinstance(lpmCores, int):
-        if lpmCores < 1:
-            lpmCores = 1
-    else:
-        lpmCores = os.cpu_count()
-
-    if isinstance(lpmDefaultThreads, int):
-        if lpmDefaultThreads < 1:
-            lpmDefaultThreads = 1
-    else:
-        lpmDefaultThreads = _optimalThreads()
-
-
 class Matrix:
     def __init__(self, rows=None, cols=None, data=None, dtype="float64", threads=None, internal_new=False):
         self.__safe_for_unpickling__ = True
 
-        _checkMultiprocessing()
-
         if internal_new:
             self.matrix = None
             self._rows = None
             self._cols = None
             self._rowStride = None
             self._colStride = None
             self.dtype = None
-            self._threads = lpmDefaultThreads
+            self._threads = _threadInfo.LPM_OPTIMAL_MATRIX_THREADS
         else:
             self.matrix = None
             self._rows = None
@@ -128,7 +56,7 @@ def __init__(self, rows=None, cols=None, data=None, dtype="float64", threads=Non
                 else:
                     raise Exception("Invalid thread type. Must be of type <int>") from TypeError
             else:
-                self._threads = lpmDefaultThreads
+                self._threads = _threadInfo.LPM_OPTIMAL_MATRIX_THREADS
 
             if dtype is not None:
                 if dtype in ("float64", "float32", "int64", "int32", "int16"):
@@ -226,9 +154,7 @@ def __init__(self, rows=None, cols=None, data=None, dtype="float64", threads=Non
                 self.matrix = _matrix.Matrix(self._rows, self._cols)
 
     @staticmethod
-    def _internal_new(matrix, dtype="float64", threads=lpmDefaultThreads):
-        _checkMultiprocessing()
-
+    def _internal_new(matrix, dtype="float64", threads=_threadInfo.LPM_OPTIMAL_MATRIX_THREADS):
         res = Matrix(internal_new=True)
         res.matrix = matrix
         res._rows = matrix.rows
@@ -274,38 +200,24 @@ def transposed(self):
 
     def __add__(self, other):
         if isinstance(other, Matrix) and self._rows == other._rows and self._cols == other._cols:
-            _checkMultiprocessing()
-
-            start = time()
-            tmp = self.matrix.matrixAddMatrixReturn(other.matrix, self.threads)
-            print(time() - start)
-            start = time()
-            res = Matrix._internal_new(tmp, self.dtype, self.threads)
-            print(time() - start)
-            return res
-
-            # return Matrix._internal_new(self.matrix.matrixAddMatrixReturn(other.matrix, self.threads), self.dtype, self.threads)
-
+            return Matrix._internal_new(self.matrix.matrixAddMatrixReturn(other.matrix, self.threads), self.dtype, self.threads)
         else:
             raise Exception("Invalid matrix size for matrix addition") from TypeError
 
     def __sub__(self, other):
         if isinstance(other, Matrix) and self._rows == other._rows and self._cols == other._cols:
-            _checkMultiprocessing()
             return Matrix._internal_new(self.matrix.matrixSubMatrixReturn(other.matrix, self.threads), self.dtype, self.threads)
         else:
             raise Exception("Invalid matrix size for matrix subtraction") from TypeError
 
     def __mul__(self, other):
         if isinstance(other, Matrix) and self._rows == other._rows and self._cols == other._cols:
-            _checkMultiprocessing()
             return Matrix._internal_new(self.matrix.matrixMulMatrixReturn(other.matrix, self.threads), self.dtype, self.threads)
         else:
             raise Exception("Invalid matrix size for matrix multiplication") from TypeError
 
     def __truediv__(self, other):
         if isinstance(other, Matrix) and self._rows == other._rows and self._cols == other._cols:
-            _checkMultiprocessing()
             return Matrix._internal_new(self.matrix.matrixDivMatrixReturn(other.matrix, self.threads), self.dtype, self.threads)
         else:
             raise Exception("Invalid matrix size for matrix division") from TypeError

setup.py

@@ -177,7 +177,7 @@ def lgompLink():
 
 setup(
     name="libpymath",
-    version="0.1.4",
+    version="0.1.5",
     description="A general purpose Python math module",
     long_description=long_description,
     long_description_content_type='text/markdown',