Merge pull request #460 from GavinHuttley/develop

BUG: setting max_workers for MPI now uses UNIVERSE_SIZE
cogent3 · Dec 20, 2019 · de5cbac · de5cbac
2 parents aa9a90a + 6a2f916
commit de5cbac
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 3 deletions.
diff --git a/src/cogent3/app/composable.py b/src/cogent3/app/composable.py
@@ -688,4 +688,3 @@ def __str__(self):
 
     def __repr__(self):
         return str(self)
-
diff --git a/src/cogent3/util/parallel.py b/src/cogent3/util/parallel.py
@@ -141,8 +141,14 @@ def imap(f, s, max_workers=None, use_mpi=False, if_serial="raise", chunksize=Non
         elif COMM.Get_attr(MPI.UNIVERSE_SIZE) == 1 and if_serial == "warn":
             warnings.warn(UserWarning, msg=err_msg)
 
-        if not max_workers:
-            max_workers = COMM.Get_attr(MPI.UNIVERSE_SIZE) - 1
+        max_workers = max_workers or 0
+
+        if max_workers > COMM.Get_attr(MPI.UNIVERSE_SIZE):
+            warnings.warn(
+                UserWarning, msg="max_workers too large, reducing to UNIVERSE_SIZE-1"
+            )
+
+        max_workers = min(max_workers, COMM.Get_attr(MPI.UNIVERSE_SIZE) - 1)
 
         if not chunksize:
             chunksize = set_default_chunksize(s, max_workers)