Merge pull request #622 from matz-e/fix-report-errors

Provide default report file.

lobster/core/data/report.json

@@ -0,0 +1,31 @@
+{
+  "files": {
+    "info": {},
+    "output_info": {},
+    "skipped": []
+  },
+  "cache": {
+    "start_size": 0,
+    "end_size": 0,
+    "type": 2
+  },
+  "task_exit_code": 0,
+  "exe_exit_code": 0,
+  "stageout_exit_code": 0,
+  "cpu_time": 0,
+  "events_written": 0,
+  "output_size": 0,
+  "output_bare_size": 0,
+  "output_storage_element": "",
+  "task_timing": {
+    "stage_in_end": 0,
+    "prologue_end": 0,
+    "wrapper_start": 0,
+    "wrapper_ready": 0,
+    "processing_end": 0,
+    "epilogue_end": 0,
+    "stage_out_end": 0
+  },
+  "events_per_run": 0,
+  "transfers": {}
+}

lobster/core/data/task.py

@@ -949,9 +949,7 @@ def run_prologue(data, config, env):
 
 @check_execution(exitcode=199, timing='epilogue_end')
 def run_epilogue(data, config, env):
-    with open('report.json', 'w') as f:
-        json.dump(data, f, indent=2)
-        f.write('\n')
+    write_report(data)
     run_step(data, config, env, 'epilogue')
     with open('report.json', 'r') as f:
         update = json.load(f)
@@ -1071,6 +1069,9 @@ def write_zipfiles(data):
     logger.propagate = False
     logger.setLevel(logging.DEBUG)
 
+    with open('report.json', 'r') as fd:
+        data = json.load(fd)
+        data['transfers'] = defaultdict(Counter)
     data = {
         'files': {
             'info': {},

lobster/core/data/wrapper.sh

@@ -26,6 +26,11 @@ log() {
 }
 
 date +%s > t_wrapper_start
+
+# WorkQueue does not like it if we transfer report.json in with the same
+# name, so rename it here.
+mv report.json.in report.json
+
 log "startup" "wrapper started" "echo -e 'hostname: $(hostname)\nkernel: $(uname -a)'"
 
 log "trace" "tracing google" traceroute -w 1 www.google.com

lobster/core/source.py

@@ -271,6 +271,7 @@ def __setup_inputs(self):
             (self.siteconf, 'siteconf', False),
             (os.path.join(os.path.dirname(__file__), 'data', 'wrapper.sh'), 'wrapper.sh', True),
             (os.path.join(os.path.dirname(__file__), 'data', 'task.py'), 'task.py', True),
+            (os.path.join(os.path.dirname(__file__), 'data', 'report.json'), 'report.json.in', True),
             (self.parrot_bin, 'bin', True),
             (self.parrot_lib, 'lib', True),
         ]

lobster/core/task.py

@@ -208,7 +208,7 @@ def process(self, task, summary, transfers):
             files_info, files_skipped, events_written, exe_exit_code, stageout_exit_code, task_exit_code = self.process_report(task_update, transfers)
         except (ValueError, EOFError, IOError) as e:
             failed = True
-            logger.error("error processing {0}:\n{1}".format(task.tag, e))
+            logger.exception("error processing {0}:\n{1}".format(task.tag, e))
 
         # Determine true status
         if task.result != wq.WORK_QUEUE_RESULT_SUCCESS and task.result != wq.WORK_QUEUE_RESULT_OUTPUT_MISSING:

lobster/util.py

@@ -23,7 +23,7 @@
 from email.mime.text import MIMEText
 from pkg_resources import get_distribution
 
-VERSION = "1.8"
+VERSION = "1.9"
 
 logger = logging.getLogger('lobster.util')
 

setup.py

@@ -27,6 +27,7 @@
         'core/data/siteconf/PhEDEx/storage.xml',
         'core/data/merge_cfg.py',
         'core/data/merge_reports.py',
+        'core/data/report.json.in',
         'commands/data/index.html',
         'commands/data/gh.png',
         'commands/data/styles.css',