added logging for resubmits

awe/aweclasses.py

@@ -467,6 +467,8 @@ def _try_duplicate_tasks(self):
     def _resubmit(self):
         invalid = self.system.filter_by_valid()
         while invalid != {}:
+            if self.verbose:
+                print("Resubmitting invalid models")
             for cid in invalid:
                 valid = self.system.get_valid_walker(cid)
                 if valid is None:

awe/workqueue.py

@@ -541,7 +541,7 @@ class WorkQueue(object):
     """
 
     # @awe.typecheck(Config)
-    def __init__(self, cfg, statslogger=None, taskoutputlogger=None, log_it=False):
+    def __init__(self, cfg, statslogger=None, taskoutputlogger=None, verbose=False, log_it=False):
         """
         Initialize a new instance of WorkQueue for managing the cctools
         work_queue.WorkQueue object with the necessary parameters for running
@@ -579,6 +579,7 @@ def __init__(self, cfg, statslogger=None, taskoutputlogger=None, log_it=False):
         self.statslogger      = statslogger      or awe.stats.StatsLogger(buffersize=42)
         self.taskoutputlogger = taskoutputlogger or awe.stats.StatsLogger(path='debug/task_output.log.gz', buffersize=42)
         self._log = log_it
+        self.verbose = verbose
 
     @property
     def empty(self):
@@ -921,6 +922,8 @@ def recv(self, marshall, mark_invalid):
             elif task and not task.result == 0:
                 # Check the task output for a bad model
                 if task.output.find('Exception: Particle coordinate is NaN'):
+                    if self.verbose:
+                        print(''.join([task.tag, ': Particle coordinate is NaN']))
                     mark_invalid(task)
                     continue
                 # Kill the task if it cannot be restarted.