From d8db675e9a3725b5e761290dc6956b121c77cce7 Mon Sep 17 00:00:00 2001 From: Joost van Griethuysen Date: Fri, 16 Nov 2018 16:31:36 +0100 Subject: [PATCH] ENH: Limit thread-safe logging to python>=3.2 Thread safe logging is implemented using QueueHandler and QueueListener, which are only available in python versions >= 3.2. Therefore, only implement this for appropriate python versions. For lower versions (most notably, version 2.7), this will only affect the log files. Here, conflicts may arise due to racing conflicts, where multiple workers try to write to the log file at the same time. This may result in poorly or incorrectly formatted log entries. --- radiomics/scripts/__init__.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/radiomics/scripts/__init__.py b/radiomics/scripts/__init__.py index 83253b24..2e1de82d 100644 --- a/radiomics/scripts/__init__.py +++ b/radiomics/scripts/__init__.py @@ -357,17 +357,24 @@ def _configureLogging(args): } } + if args.jobs > 1: + # Update the logger format to include the threadname if multiprocessing + # is enabled + logging_config['formatters']['default']['format'] = \ + '[%(asctime)s] %(levelname)-.1s: (%(threadName)s) %(name)s: %(message)s' + # Set up optional logging to file if args.log_file is not None: - if args.jobs > 1: + py_version = (sys.version_info.major, sys.version_info.minor) + if args.jobs > 1 and py_version >= (3, 2): # Multiprocessing! Use a QueueHandler, FileHandler and QueueListener # to implement thread-safe logging. + + # However, QueueHandler and Listener were added in python 3.2. + # Therefore, only use this if the python version > 3.2 q = Manager().Queue(-1) threading.current_thread().setName('Main') - logging_config['formatters']['default']['format'] = \ - '[%(asctime)s] %(levelname)-.1s: (%(threadName)s) %(name)s: %(message)s' - logging_config['handlers']['logfile'] = { 'class': 'logging.handlers.QueueHandler', 'queue': q,