log unhandled exceptions to file (#480)

* log unhandled exceptions to file * remove unnecessary import
neulab · Jul 30, 2018 · 92a5fbf · 92a5fbf
1 parent 77e2baf
commit 92a5fbf
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 13 deletions.
diff --git a/xnmt/__init__.py b/xnmt/__init__.py
@@ -12,6 +12,7 @@
 import logging
 logger = logging.getLogger('xnmt')
 yaml_logger = logging.getLogger('yaml')
+file_logger = logging.getLogger('xnmt_file')
 
 import _dynet
 dyparams = _dynet.DynetParams()

diff --git a/xnmt/tee.py b/xnmt/tee.py
@@ -38,6 +38,8 @@ def format(self, record):
 
 logger = logging.getLogger('xnmt')
 logger.setLevel(min(logging._checkLevel(settings.LOG_LEVEL_CONSOLE), logging._checkLevel(settings.LOG_LEVEL_FILE)))
+logger_file = logging.getLogger('xnmt_file')
+logger_file.setLevel(min(logging._checkLevel(settings.LOG_LEVEL_CONSOLE), logging._checkLevel(settings.LOG_LEVEL_FILE)))
 
 ch_out = logging.StreamHandler(sys_std_out)
 ch_out.setLevel(settings.LOG_LEVEL_CONSOLE)
@@ -79,6 +81,7 @@ def set_out_file(out_file):
   fh.setLevel(settings.LOG_LEVEL_FILE)
   fh.setFormatter(MainFormatter())
   logger.addHandler(fh)
+  logger_file.addHandler(fh)
   yaml_fh = logging.FileHandler(f"{out_file}.yaml", mode='w', encoding="utf-8")
   yaml_fh.setLevel(logging.DEBUG)
   yaml_fh.setFormatter(YamlFormatter())
@@ -97,6 +100,9 @@ def unset_out_file():
     if isinstance(hdlr, logging.FileHandler):
       hdlr.close()
       yaml_logger.removeHandler(hdlr)
+  for hdlr in list(logger_file.handlers):
+    hdlr.close()
+    logger_file.removeHandler(hdlr)
 
 class Tee(object):
   def __init__(self, indent=0, error=False):

diff --git a/xnmt/xnmt_run_experiments.py b/xnmt/xnmt_run_experiments.py
@@ -13,11 +13,12 @@
 import datetime
 import faulthandler
 faulthandler.enable()
+import traceback
 
 import numpy as np
 from xnmt.settings import settings
 
-from xnmt import logger
+from xnmt import logger, file_logger
 from xnmt.tee import log_preamble
 from xnmt.param_collection import ParamManager
 import xnmt.tee as tee
@@ -90,22 +91,28 @@ def main(overwrite_args=None):
 
       tee.set_out_file(log_file)
 
-      model_file = glob_args.model_file
+      try:
 
-      uninitialized_exp_args.data.exp_global.commandline_args = vars(args)
+        model_file = glob_args.model_file
 
-      # Create the model
-      experiment = initialize_if_needed(uninitialized_exp_args)
-      ParamManager.param_col.model_file = experiment.exp_global.model_file
-      ParamManager.param_col.save_num_checkpoints = experiment.exp_global.save_num_checkpoints
-      ParamManager.populate()
+        uninitialized_exp_args.data.exp_global.commandline_args = vars(args)
 
-      # Run the experiment
-      eval_scores = experiment(save_fct = lambda: save_to_file(model_file, experiment))
-      results.append((experiment_name, eval_scores))
-      print_results(results)
+        # Create the model
+        experiment = initialize_if_needed(uninitialized_exp_args)
+        ParamManager.param_col.model_file = experiment.exp_global.model_file
+        ParamManager.param_col.save_num_checkpoints = experiment.exp_global.save_num_checkpoints
+        ParamManager.populate()
 
-      tee.unset_out_file()
+        # Run the experiment
+        eval_scores = experiment(save_fct = lambda: save_to_file(model_file, experiment))
+        results.append((experiment_name, eval_scores))
+        print_results(results)
+
+      except Exception as e:
+        file_logger.error(traceback.format_exc())
+        raise e
+      finally:
+        tee.unset_out_file()
 
 def print_results(results):
   print("")