Merge pull request #16 from alaniwi/devel

when max_threads=1, do not create a thread pool

esgprep/drs/context.py

@@ -49,6 +49,7 @@ def __init__(self, args):
         if args.set_key:
             self.set_keys = dict(args.set_key)
         self.threads = args.max_threads
+        self.use_pool = (self.threads > 1)
         self.project = args.project
         self.action = args.action
         if args.copy:
@@ -109,7 +110,8 @@ def __enter__(self):
         # And exclude hidden files
         self.sources.FileFilter[uuid()] = ('^\..*$', True)
         # Init threads pool
-        self.pool = ThreadPool(int(self.threads))
+        if self.use_pool:
+            self.pool = ThreadPool(int(self.threads))
         return self
 
     def _check_existing_commands_file(self):
@@ -127,8 +129,9 @@ def _check_existing_commands_file(self):
 
     def __exit__(self, *exc):
         # Close threads pool
-        self.pool.close()
-        self.pool.join()
+        if self.use_pool:
+            self.pool.close()
+            self.pool.join()
         # Decline outputs depending on the scan results
         # Raise errors when one or several files have been skipped or failed
         # Default is sys.exit(0)

esgprep/drs/main.py

@@ -8,6 +8,7 @@
 """
 
 import logging
+import itertools
 
 from ESGConfigParser.custom_exceptions import NoConfigOption
 
@@ -190,7 +191,10 @@ def run(args):
             logging.warning(msg)
         else:
             nfiles = len(ctx.sources)
-            processes = ctx.pool.imap(process, ctx.sources)
+            if ctx.use_pool:
+                processes = ctx.pool.imap(process, ctx.sources)
+            else:
+                processes = itertools.imap(process, ctx.sources)
             if ctx.pbar:
                 processes = as_pbar(processes, desc='Scanning incoming files', units='files', total=nfiles)
             # Process supplied files

esgprep/mapfile/context.py

@@ -44,6 +44,7 @@ def __init__(self, args):
         self.notes_url = args.tech_notes_url
         self.no_version = args.no_version
         self.threads = args.max_threads
+        self.use_pool = (self.threads > 1)
         self.dataset = args.dataset
         if not args.no_cleanup:
             self.clean()
@@ -109,13 +110,15 @@ def __enter__(self):
             # If --latest-symlink, --version is set to "latest"
             self.sources.PathFilter['version_filter'] = '/{}'.format(self.version)
         # Init threads pool
-        self.pool = ThreadPool(int(self.threads))
+        if self.use_pool:
+            self.pool = ThreadPool(int(self.threads))
         return self
 
     def __exit__(self, *exc):
         # Close threads pool
-        self.pool.close()
-        self.pool.join()
+        if self.use_pool:
+            self.pool.close()
+            self.pool.join()
         # Decline outputs depending on the scan results
         # Raise errors when one or several files have been skipped or failed
         # Default is sys.exit(0)

esgprep/mapfile/main.py

@@ -11,6 +11,7 @@
 import os
 import re
 from datetime import datetime
+import itertools
 
 from ESGConfigParser import interpolate
 from lockfile import LockFile
@@ -186,7 +187,10 @@ def run(args):
     with ProcessingContext(args) as ctx:
         logging.info('==> Scan started')
         nfiles = len(ctx.sources)
-        processes = ctx.pool.imap(process, ctx.sources)
+        if ctx.use_pool:
+            processes = ctx.pool.imap(process, ctx.sources)
+        else:
+            processes = itertools.imap(process, ctx.sources)
         if ctx.pbar:
             processes = as_pbar(processes, desc='Mapfile(s) generation', units='files', total=nfiles)
         # Process supplied files