Merge pull request #641 from Libensemble/feature/local_close_logs

Local comms closes log files and handlers on cleanup

libensemble/comms/logs.py

@@ -101,6 +101,7 @@ def worker_logging_config(comm, worker_id=None):
     if logconfig.logger_set:
         for hdl in logger.handlers[:]:
             logger.removeHandler(hdl)
+            hdl.close()
     else:
         logger.propagate = False
         logger.setLevel(logconfig.log_level)
@@ -129,13 +130,13 @@ def manager_logging_config():
 
         # Stats logging
         # NB: Could add a specialized handler for immediate flushing
-        fh = logging.FileHandler(logconfig.stat_filename, mode='w')
-        fh.addFilter(wfilter)
-        fh.setFormatter(logging.Formatter('Worker %(worker)5d: %(message)s'))
+        fhs = logging.FileHandler(logconfig.stat_filename, mode='w')
+        fhs.addFilter(wfilter)
+        fhs.setFormatter(logging.Formatter('Worker %(worker)5d: %(message)s'))
         stat_logger = logging.getLogger(logconfig.stats_name)
         stat_logger.propagate = False
         stat_logger.setLevel(logging.DEBUG)
-        stat_logger.addHandler(fh)
+        stat_logger.addHandler(fhs)
 
         # Mirror error-logging to stderr of user-specified level
         fhe = logging.StreamHandler(stream=sys.stderr)
@@ -144,3 +145,9 @@ def manager_logging_config():
         fhe.addFilter(efilter)
         fhe.setFormatter(formatter)
         logger.addHandler(fhe)
+
+        def close_logs():
+            fh.close()
+            fhs.close()
+
+        return close_logs

libensemble/libE.py

@@ -327,12 +327,14 @@ def libE_local(sim_specs, gen_specs, exit_criteria,
     wcomms = start_proc_team(nworkers, sim_specs, gen_specs, libE_specs)
 
     if not libE_specs.get('disable_log_files', False):
-        manager_logging_config()
+        close_logs = manager_logging_config()
 
     # Set up cleanup routine to shut down worker team
     def cleanup():
         "Handler to clean up comms team."
         kill_proc_team(wcomms, timeout=libE_specs.get('worker_timeout', 1))
+        if close_logs is not None:  # logger remains set between multiple libE calls
+            close_logs()
 
     # Run generic manager
     return manager(wcomms, sim_specs, gen_specs, exit_criteria,