diff --git a/esgprep/drs/context.py b/esgprep/drs/context.py index f90e686..5a0a663 100644 --- a/esgprep/drs/context.py +++ b/esgprep/drs/context.py @@ -49,6 +49,7 @@ def __init__(self, args): if args.set_key: self.set_keys = dict(args.set_key) self.threads = args.max_threads + self.use_pool = (self.threads > 1) self.project = args.project self.action = args.action if args.copy: @@ -109,7 +110,8 @@ def __enter__(self): # And exclude hidden files self.sources.FileFilter[uuid()] = ('^\..*$', True) # Init threads pool - self.pool = ThreadPool(int(self.threads)) + if self.use_pool: + self.pool = ThreadPool(int(self.threads)) return self def _check_existing_commands_file(self): @@ -127,8 +129,9 @@ def _check_existing_commands_file(self): def __exit__(self, *exc): # Close threads pool - self.pool.close() - self.pool.join() + if self.use_pool: + self.pool.close() + self.pool.join() # Decline outputs depending on the scan results # Raise errors when one or several files have been skipped or failed # Default is sys.exit(0) diff --git a/esgprep/drs/main.py b/esgprep/drs/main.py index 35dfca0..d34d7ea 100644 --- a/esgprep/drs/main.py +++ b/esgprep/drs/main.py @@ -8,6 +8,7 @@ """ import logging +import itertools from ESGConfigParser.custom_exceptions import NoConfigOption @@ -190,7 +191,10 @@ def run(args): logging.warning(msg) else: nfiles = len(ctx.sources) - processes = ctx.pool.imap(process, ctx.sources) + if ctx.use_pool: + processes = ctx.pool.imap(process, ctx.sources) + else: + processes = itertools.imap(process, ctx.sources) if ctx.pbar: processes = as_pbar(processes, desc='Scanning incoming files', units='files', total=nfiles) # Process supplied files diff --git a/esgprep/mapfile/context.py b/esgprep/mapfile/context.py index 6fc5529..432c99a 100644 --- a/esgprep/mapfile/context.py +++ b/esgprep/mapfile/context.py @@ -44,6 +44,7 @@ def __init__(self, args): self.notes_url = args.tech_notes_url self.no_version = args.no_version self.threads = args.max_threads + self.use_pool = (self.threads > 1) self.dataset = args.dataset if not args.no_cleanup: self.clean() @@ -109,13 +110,15 @@ def __enter__(self): # If --latest-symlink, --version is set to "latest" self.sources.PathFilter['version_filter'] = '/{}'.format(self.version) # Init threads pool - self.pool = ThreadPool(int(self.threads)) + if self.use_pool: + self.pool = ThreadPool(int(self.threads)) return self def __exit__(self, *exc): # Close threads pool - self.pool.close() - self.pool.join() + if self.use_pool: + self.pool.close() + self.pool.join() # Decline outputs depending on the scan results # Raise errors when one or several files have been skipped or failed # Default is sys.exit(0) diff --git a/esgprep/mapfile/main.py b/esgprep/mapfile/main.py index 990300f..6d30814 100644 --- a/esgprep/mapfile/main.py +++ b/esgprep/mapfile/main.py @@ -11,6 +11,7 @@ import os import re from datetime import datetime +import itertools from ESGConfigParser import interpolate from lockfile import LockFile @@ -186,7 +187,10 @@ def run(args): with ProcessingContext(args) as ctx: logging.info('==> Scan started') nfiles = len(ctx.sources) - processes = ctx.pool.imap(process, ctx.sources) + if ctx.use_pool: + processes = ctx.pool.imap(process, ctx.sources) + else: + processes = itertools.imap(process, ctx.sources) if ctx.pbar: processes = as_pbar(processes, desc='Mapfile(s) generation', units='files', total=nfiles) # Process supplied files