From af0df024fb4fde034567fd1a1220e96bebf08a12 Mon Sep 17 00:00:00 2001
From: Jeff Kinnison <jkinniso@nd.edu>
Date: Thu, 14 Jul 2016 10:54:15 -0400
Subject: [PATCH] added logging for resubmits

---
 awe/aweclasses.py | 2 ++
 awe/workqueue.py  | 5 ++++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/awe/aweclasses.py b/awe/aweclasses.py
index 89d95d5..f912644 100644
--- a/awe/aweclasses.py
+++ b/awe/aweclasses.py
@@ -467,6 +467,8 @@ def _try_duplicate_tasks(self):
     def _resubmit(self):
         invalid = self.system.filter_by_valid()
         while invalid != {}:
+            if self.verbose:
+                print("Resubmitting invalid models")
             for cid in invalid:
                 valid = self.system.get_valid_walker(cid)
                 if valid is None:
diff --git a/awe/workqueue.py b/awe/workqueue.py
index 3a9d664..96e15e9 100644
--- a/awe/workqueue.py
+++ b/awe/workqueue.py
@@ -541,7 +541,7 @@ class WorkQueue(object):
     """
 
     # @awe.typecheck(Config)
-    def __init__(self, cfg, statslogger=None, taskoutputlogger=None, log_it=False):
+    def __init__(self, cfg, statslogger=None, taskoutputlogger=None, verbose=False, log_it=False):
         """
         Initialize a new instance of WorkQueue for managing the cctools
         work_queue.WorkQueue object with the necessary parameters for running
@@ -579,6 +579,7 @@ def __init__(self, cfg, statslogger=None, taskoutputlogger=None, log_it=False):
         self.statslogger      = statslogger      or awe.stats.StatsLogger(buffersize=42)
         self.taskoutputlogger = taskoutputlogger or awe.stats.StatsLogger(path='debug/task_output.log.gz', buffersize=42)
         self._log = log_it
+        self.verbose = verbose
 
     @property
     def empty(self):
@@ -921,6 +922,8 @@ def recv(self, marshall, mark_invalid):
             elif task and not task.result == 0:
                 # Check the task output for a bad model
                 if task.output.find('Exception: Particle coordinate is NaN'):
+                    if self.verbose:
+                        print(''.join([task.tag, ': Particle coordinate is NaN']))
                     mark_invalid(task)
                     continue
                 # Kill the task if it cannot be restarted.