process-run123-logs

#!/usr/bin/env python
from argparse import ArgumentParser
from os.path import exists, join, dirname
from sys import exit
import json
import re
from operator import itemgetter
from pickle import load
import sys

RESULTS_RE = "^([0-9.]+)_([^ ]*) (.*) - time date.*exit: (.*)"
STEP_STATUS_RE = "Step([0-9])-(PASSED|FAILED|DAS_ERROR|TIMEOUT)"

if __name__ == "__main__":
  parser = ArgumentParser()
  parser.add_argument("input")
  args = parser.parse_args()
  if not exists(args.input):
    print "Please specify an input file"
    exit(1)

  # Takes care of the actual results.
  known_errors = {}
  known_error_file = join(dirname(args.input),"all_known_errors.json")
  summary_file = join(dirname(args.input),"summary.json")
  if exists(known_error_file):
    try: known_errors = json.load(open(known_error_file))
    except: pass
  buf = open(args.input).read()
  results = []
  summary = {"num_passed" : 0, "num_failed" : 0, "known_failed" : 0}
  for l in buf.split("\n"):
    r = re.match(RESULTS_RE, l)
    if not r:
      continue
    workflowId, workflow, steps, exit = r.groups()
    exitcode = 0
    for c in exit.split(" "):
      if re.match("^[1-9][0-9]*$",c):
        exitcode=int(c)
        summary["num_failed"]=summary["num_failed"]+1
        break
      summary["num_passed"]=summary["num_passed"]+1
    steps = steps.split(" ")
    stepResults = [re.match(STEP_STATUS_RE, s).group(2)
                   for s in steps if re.match(STEP_STATUS_RE, s)]

    failedStep  = 0
    order       = 0
    maxOrder    = 200
    if exitcode:
      for (i,x) in enumerate(stepResults):
        if x=="PASSED": continue
        failedStep = i+1
        order = i-maxOrder
        break
    known_error = 0
    known_error_msg = ""
    known_failed_step = 0
    if workflowId in known_errors:
      maxOrder2   = maxOrder/2
      known_error_msg = known_errors[workflowId]["reason"]
      known_failed_step = known_errors[workflowId]["step"]
      if str(known_failed_step).startswith("step"): known_failed_step = int(known_failed_step[4:])
      if (known_failed_step==failedStep) or (known_errors[workflowId]["exitcode"]==exitcode):
        known_error = 1
        order = order+maxOrder2
        summary["known_failed"]=summary["known_failed"]+1
        summary["num_failed"]=summary["num_failed"]-1
      else:
        known_error = -1
        if order<0: order = order-maxOrder2
        else:
          order = known_failed_step-maxOrder2-maxOrder
          summary["num_failed"]=summary["num_failed"]+1
          summary["num_passed"]=summary["num_passed"]-1

    result={
            "id": workflowId,
            "fid": float(workflowId),
            "name": workflow,
            "exitcode": exitcode,
            "known_error": known_error,
            "steps": [{"status": x} for x in stepResults],
            "order": order
           }

    if known_error!=0:
      result["known_error_message"] = known_error_msg
      result["known_failed_step"]   = known_failed_step
    results.append(result)
  json.dump(summary, open(summary_file,"w"))
  # Does the same for errors and warnings.
  warningsFile = join(dirname(args.input), "runTheMatrixMsgs.pkl")
  if exists(warningsFile):
    warnings = load(open(warningsFile))
    for result in results:
      if not str(result["id"]) in warnings:
        continue
      data = warnings[result["id"]]
      dlen = len(data["failed"])
      for (i, s) in enumerate(result["steps"]):
        if (i >= dlen): break
        try:
          s.update({"errors": data["failed"][i], "warnings": data["warning"][i]})
        except:
          sys.stderr.write("ERROR: Reading "+warningsFile+"\n")
          sys.stderr.write("ERROR: "+str(i)+","+str(result["id"])+","+str(data)+"\n")
          sys.exit(1)
  results.sort(key=itemgetter("order", "fid"))
  for r in results:
    r.pop("fid")
    r.pop("order")
  print json.dumps(results, sort_keys=True, indent=4, separators=(',', ': '))