report

#!/usr/bin/python

import json
import sys
import re
import copy
import os
import argparse
import subprocess
import numpy as np

def ensureFigDir():
  if not os.path.isdir("figures"):
    os.mkdir("figures")

def getGitRoot():
  return subprocess.Popen(['git', 'rev-parse', '--show-toplevel'],
    stdout=subprocess.PIPE).communicate()[0].rstrip()

ROOT = getGitRoot()

parser = argparse.ArgumentParser()
parser.add_argument('--no-plots', action='store_true', dest='no_plots')
parser.add_argument('input_file', nargs='?', metavar='RESULTS_FILE')
args = parser.parse_args()

BLUE = '\033[94m'
#GREEN = '\033[92m'
GREEN = '\033[38;2;20;139;20m'
#LIGHT_GREEN = '\033[38;2;138;226;52m'
LIGHT_GREEN = '\033[38;2;100;226;130m'
YELLOW = '\033[93m'
GRAY = '\033[38;2;151;155;147m'
RED = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'

class colortext:
  def __init__(self, text, color, bold=True):
    self.text = text
    self.color = color
    self.bold = bold
  def __len__(self):
    return len(self.text)
  def __str__(self):
    return (BOLD if self.bold else "") + self.color + self.text + ENDC
def green(s):
  return colortext(s, GREEN)
def red(s):
  return colortext(s, RED)
def orange(s):
  return colortext(s, YELLOW, bold=True)
def blue(s):
  return colortext(s, BLUE)
def lightgreen(s):
  return colortext(s, LIGHT_GREEN, bold=True)
def gray(s):
  return colortext(s, GRAY, bold=False)

def json_careful_loads(s):
  try:
    return json.loads(s)
  except Exception as e:
    sys.stderr.write("[ERR] Error while parsing json: {}\n".format(e))
    sys.exit(1)

def json_careful_readlines(f):
  return [ json_careful_loads(line.rstrip('\n')) for line in f ]

def safeInsert(dict, key, value):
  if key not in dict:
    dict[key] = value
  else:
    sys.stderr.write("[WARN] Key {} is already in use; trying _{} instead.\n".format(key))
    safeInsert(dict, "_" + key, value)

def reCompile(exp):
  return re.compile(exp, re.MULTILINE)

# def parseCommaInteger(s):
#   return int(s.replace(",", ""))

# local reclaimed: 32859049984
# num local: 20999
# local gc time: 4541
# promo time: 8

def parseKiB(kibStr):
  return float(int(kibStr)) * 1024.0 / 1000.0

def parseB(bytesStr):
  return float(int(bytesStr)) / 1000.0

def parseTimes(stdout):
  pat = reCompile(r"^time\s+(\d+.\d+).*$")
  return [float(x) for x in pat.findall(stdout)]

statsPatterns = \
  [ #("time", float, reCompile(r"^end-to-end\s+(\d+.\d+)s$"))
  #,
    ("space", parseKiB, reCompile(r"^\s*Maximum resident set size \(kbytes\): (\d+).*$"))
  , ("approx-race-factor", parseB, reCompile(r"^entangled bytes: (\d+).*$"))
  , ("kbytes-pinned-entangled", parseB, reCompile(r"^bytes pinned entangled: (\d+).*$"))
  # , ("num-local", int, reCompile(r"^num local: (\d+)$"))
  # , ("local-reclaimed", parseB, reCompile(r"^local reclaimed: (\d+)$"))
  # , ("local-time", int, reCompile(r"^local gc time: (\d+)$"))
  # , ("promo-time", int, reCompile(r"^promo time: (\d+)$"))
  # , ("root-reclaimed", parseB, reCompile(r"^root cc reclaimed: (\d+)$"))
  # , ("internal-reclaimed", parseB, reCompile(r"^internal cc reclaimed: (\d+)$"))
  # , ("num-root", int, reCompile(r"^num root cc: (\d+)$"))
  # , ("num-internal", int, reCompile(r"^num internal cc: (\d+)$"))
  # , ("root-time", int, reCompile(r"^root cc time: (\d+)$"))
  # , ("internal-time", int, reCompile(r"^internal cc time: (\d+)$"))
  # , ("working-set", parseCommaInteger, reCompile(r"^max bytes live: (.*) bytes$"))
  ]

def renameConfig(c):
  return c

def renameTag(t):
  return t

def displayTag(t):
  sandmarkTags = ["binarytrees5","lu-decomp","game-of-life","nbody"]
  if t in sandmarkTags:
    return "SM:" + t
  if t == "msort-ints":
    return "msort-int64"
  return t


foundTags = set()
foundProcs = set()

def parseStats(row):
  newRow = copy.deepcopy(row)
  for (name, convert, pat) in statsPatterns:
    m = pat.search(newRow['stdout'] + newRow['stderr'])
    if m:
      safeInsert(newRow, name, convert(m.group(1)))
  newRow['procs'] = int(newRow.get('procs', '1'))
  newRow['config'] = renameConfig(row['config'])
  newRow['tag'] = renameTag(row['tag'])

  allOutput = newRow['stdout'] + newRow['stderr']
  if 'multi' in newRow:
    for i in range(1, int(newRow['multi'])):
      allOutput += newRow['stdout{}'.format(i)] + newRow['stderr{}'.format(i)]

  tms = parseTimes(allOutput)
  try:
    newRow['avgtime'] = sum(tms) / len(tms)
  except:
    newRow['avgtime'] = None

  # try:
  #   newRow['space'] = float(newRow['space'])
  # except KeyError:
  #   pass

  # try:
  #   newRow['time'] = float(newRow['elapsed'])
  # except KeyError:
  #   pass

  foundTags.add(newRow['tag'])
  foundProcs.add(newRow['procs'])

  return newRow

def findTrials(data, config, tag, procs):
  result = []
  for row in data:
    if (row['config'] == config and \
        row['tag'] == tag and \
        row['procs'] == procs):
      result.append(row)
  return result

# ======================================================================

def averageTime(data, config, tag, procs, checkExpType=True):
  trials = [ r for r in findTrials(data, config, tag, procs) if (not checkExpType) or (r['exp'] == 'time') ]
  tms = [ r['avgtime'] for r in trials if 'avgtime' in r ]
  try:
    return tms[-1]
  except:
    return None

def averageSpace(data, config, tag, procs, checkExpType=True):
  trials = [ r for r in findTrials(data, config, tag, procs) if (not checkExpType) or r['exp'] == 'space' ]
  sp = [ r['space'] for r in trials if 'space' in r ]

  try:
    # sp = sp[-10:] if procs > 1 else sp[-1:]
    sp = sp[-1:]
    return sum(sp) / len(sp)
  except:
    return None

def approxRaceFactor(data, config, tag, procs):
  trials = [r for r in findTrials(data, config, tag, procs) if r['exp'] == 'stats' ]
  ee = [ r['approx-race-factor'] for r in trials if 'approx-race-factor' in r ]
  try:
    return ee[-1]
  except:
    return None

# ======================================================================

def tm(t):
  if t is None:
    return None
  if t == 0.0:
    return int(0)
  # if t > 10.0:
  #   return int(round(t))
  try:
    if t < 1.0:
      return round(t, 3)
    if t < 10.0:
      return round(t, 2)
    elif t < 100.0:
      return round(t, 1)
    else:
      return round(t)
  except TypeError:
    print ("[ERR] Got type error trying to round {}".format(repr(t)))
    return None

def ov(x):
  if x is None:
    return None
  return "{:.2f}".format(x)

def ovv(x):
  if x is None:
    return None
  return round(x, 2)

def rat(x):
  if x is None:
    return None
  if x >= 10.0:
    return str(int(round(x)))
  if x >= 1:
    return "{:.1f}".format(x)
  else:
    return "{:.2f}".format(x)

def sd(x, y):
  try:
    return x / y
  except:
    return None

def safemul(x, y):
  try:
    return x * y
  except:
    return None

def safeadd(x, y):
  try:
    return x + y
  except:
    return None

def su(x):
  if x is None:
    return None
  return str(int(round(x)))

def bu(x):
  if x is None:
    return None
  return "{:.1f}".format(x)

def noLeadZero(x):
  try:
    if "0" == x[:1]:
      return x[1:]
  except:
    pass
  return x

def sp(kb):
  if kb is None:
    return None
  if kb < 0.001:
    return "0"
  num = kb
  for unit in ['K','M','G']:
    if num < 100:
      if num < 1:
        return noLeadZero("%0.2f %s" % (num, unit))
      if num < 10:
        return noLeadZero("%1.1f %s" % (num, unit))
      return "%d %s" % (round(num), unit)
      # return "%d %s" % (int(round(num,-1)), unit)
    num = num / 1000
  return noLeadZero("%1.1f %s" % (num, 'T'))

def sfmt(xx):
  if xx is None:
    return "--"
  elif type(xx) is str:
    return xx
  elif xx < 0.01:
    return noLeadZero("{:.4f}".format(xx))
  elif xx < 0.1:
    return noLeadZero("{:.3f}".format(xx))
  elif xx < 1.0:
    return noLeadZero("{:.2f}".format(xx))
  elif xx < 10.0:
    return "{:.1f}".format(xx)
  else:
    return str(int(round(xx)))

def spg(kb):
  try:
    gb = kb / (1000.0 * 1000.0)
    if gb < .01:
      return round(gb, 4)
    elif gb < .1:
      return round(gb, 3)
    elif gb < 1.0:
      return round(gb, 2)
    elif gb < 10.0:
      return round(gb, 1)
    else:
      return round(gb, 0)
  except:
    return None

def spm(kb):
  try:
    mb = kb / (1000.0)
    if mb < .01:
      return round(mb, 4)
    elif mb < .1:
      return round(mb, 3)
    elif mb < 1.0:
      return round(mb, 2)
    elif mb < 10.0:
      return round(mb, 1)
    else:
      return round(mb, 0)
  except:
    return None

def makeBold(s):
  try:
    return "{\\bf " + s + "}"
  except Exception as e:
    sys.stderr.write("[WARN] " + str(e) + "\n")
    return "--"

def pcd(b, a):
  try:
    xx = int(round(100.0 * (b-a) / abs(a)))
    return xx
  except:
    return None

def fmtpcd(xx, highlight=True):
  try:
    xx = int(round(xx))
    result = ("+" if xx >= 0.0 else "") + ("{}\\%".format(xx))
    if highlight and (xx < 0):
      return makeBold(result)
    else:
      return result
  except Exception as e:
    sys.stderr.write("[WARN] " + str(e) + "\n")
    return "--"

def ov_to_latexpcd(ov, highlight=True):
  try:
    xx = int(round(100.0 * (ov-1.0)))
    result = ("+" if xx >= 0.0 else "") + ("{}\\%".format(xx))
    if highlight and (xx < 0):
      return makeBold(result)
    else:
      return result
  except Exception as e:
    sys.stderr.write("[WARN] " + str(e) + "\n")
    return "--"

def latexpcd(b, a, highlight=True):
  try:
    xx = pcd(b, a)
    result = ("+" if xx >= 0.0 else "") + ("{}\\%".format(xx))
    if highlight and (xx < 0):
      return makeBold(result)
    else:
      return result
  except Exception as e:
    sys.stderr.write("[WARN] " + str(e) + "\n")
    return "--"

def fmt(xx):
  if xx is None:
    return "--"
  elif type(xx) is str:
    return xx
  elif xx < 1.0:
    return noLeadZero("{:.3f}".format(xx))
  elif xx < 10.0:
    return "{:.2f}".format(xx)
  elif xx < 100.0:
    return "{:.1f}".format(xx)
  else:
    return str(int(round(xx)))

def geomean(iterable):
  try:
    a = np.array(iterable)
    return a.prod()**(1.0/len(a))
  except:
    return None

def average(iterable):
  try:
    a = np.array(iterable)
    return a.sum() * (1.0/len(a))
  except:
    return None

# =========================================================================

delimWidth = 2

def makeline(row, widths, align):
  bits = []
  i = 0
  while i < len(row):
    j = i+1
    while j < len(row) and (row[j] is None):
      j += 1
    availableWidth = sum(widths[i:j]) + delimWidth*(j-i-1)
    s = str(row[i])
    w = " " * (availableWidth - len(row[i]))
    aa = align(i)
    if aa == "l":
      ln = s + w
    elif aa == "r":
      ln = w + s
    elif aa == "c":
      ln = w[:len(w)/2] + s + w[len(w)/2:]
    else:
      raise ValueError("invalid formatter: {}".format(aa))
    bits.append(ln)
    i = j
  return (" " * delimWidth).join(bits)

def table(rows, align=None):
  numCols = max(len(row) for row in rows if not isinstance(row, str))

  widths = [0] * numCols
  for row in rows:
    # string rows are used for formatting
    if isinstance(row, str):
      continue

    i = 0
    while i < len(row):
      j = i+1
      while j < len(row) and (row[j] is None):
        j += 1
      # rw = len(stripANSI(str(row[i])))
      # rw = len(str(row[i]))
      rw = len(row[i])
      for k in xrange(i, j):
        w = (rw / (j-i)) + (1 if k < rw % (j-i) else 0)
        widths[k] = max(widths[k], w)
      i = j

  totalWidth = sum(widths) + delimWidth*(numCols-1)

  def aa(i):
    try:
      return align(i)
    except:
      return "l"

  output = []
  for row in rows:
    if row == "-" or row == "=":
      output.append(row * totalWidth)
      continue
    elif isinstance(row, str):
      raise ValueError("bad row: {}".format(row))
    output.append(makeline(row, widths, aa))

  return "\n".join(output)

# =========================================================================

def mostRecentResultsFile(suffix=""):
  files = os.listdir(os.path.join(ROOT, "results"))
  pattern = r'\d{6}-\d{6}'
  if suffix != "":
    pattern = pattern + "-" + suffix + "$"
  else:
    pattern = pattern + "$"
  # A bit of a hack. Filenames are ...YYMMDD-hhmmss, so lexicographic string
  # comparison is correct for finding the most recent (i.e. maximum) file
  mostRecent = max(p for p in files if re.match(pattern, p))
  return mostRecent

if args.input_file:
  timingsFile = args.input_file
else:
  print("[INFO] no results file argument; finding most recent")
  try:
    mostRecent = mostRecentResultsFile()
  except:
    print("[ERR] could not find most recent results file\n " + \
          "  check that these are formatted as 'YYMMSS-hhmmss'")
    sys.exit(1)
  timingsFile = os.path.join(ROOT, 'results', mostRecent)

print("[INFO] reading {}\n".format(timingsFile))
with open(timingsFile, 'r') as data:
  resultsData = json_careful_readlines(data)
D = [ parseStats(row) for row in resultsData ]
P = sorted(list(foundProcs))
maxp = max(p for p in foundProcs if p <= 72)
orderedTags = sorted(list(foundTags), key=displayTag)

foundProcs = set()
foundTags = set()

def keepTag(t):
  return True

orderedTags = [ t for t in orderedTags if keepTag(t) ]

ocamlTags = [
  "tokens",
  "msort-ints",
  "primes",
  "raytracer",
  "lu-decomp",
  "msort-strings",
  "nbody",
  "game-of-life",
  "binarytrees5"
]

# map each MPL tag to the correct C++ tag for comparison
cppTags = {
  "bfs": "bfs",
  "bignum-add": "bignum-add",
  "delaunay": "delaunay",
  "grep": "grep",
  "integrate": "integrate",
  "linearrec": "linearrec",
  "linefit": "linefit",
  "mcss": "mcss",
  "msort-ints": "msort",
  "nearest-nbrs": "nearest-nbrs",
  "primes": "primes",
  "quickhull": "quickhull",
  "sparse-mxv": "sparse-mxv",
  "tokens": "tokens",
  "wc": "wc"
}

javaTags = {
  "primes": "primes",
  "msort-ints": "sort-longs",
  "tokens": "tokens",
  "mcss": "mcss",
  "linefit": "linefit",
  "sparse-mxv": "sparse-mxv",
  "msort-strings": "msort-strings"
}

goTags = {
  "primes": "primes",
  "msort-ints": "msort",
  "tokens": "tokens",
  "mcss": "mcss",
  "linefit": "linefit",
  "sparse-mxv": "sparse-mxv",
  "msort-strings": "msort-strings"
}

sandmarkTags = [
  "binarytrees5",
  "game-of-life",
  "nbody",
  "lu-decomp"
]

# orderedTags = [ t for t in orderedTags if t not in sandmarkTags ]

# ===========================================================================

def filterSome(xs):
  return [x for x in xs if x is not None]

# def seqOverhead(tag):
#   return sd(averageTime(D, 'mpl-em', tag, 1),averageTime(D, 'mpl', tag, 1))
# def parOverhead(tag):
#   return sd(averageTime(D, 'mpl-em', tag, maxp),averageTime(D, 'mpl', tag, maxp))
# def seqSpaceOverhead(tag):
#   return sd(averageSpace(D, 'mpl-em', tag, 1),averageSpace(D, 'mpl', tag, 1))
# def parSpaceOverhead(tag):
#   return sd(averageSpace(D, 'mpl-em', tag, maxp),averageSpace(D, 'mpl', tag, maxp))

# print "geomean 1-core time overhead", geomean(filterSome([seqOverhead(t) for t in disentangledTags]))
# print "geomean {}-core time overhead".format(maxp), geomean(filterSome([parOverhead(t) for t in disentangledTags]))
# print "geomean 1-core space overhead", geomean(filterSome([seqSpaceOverhead(t) for t in disentangledTags]))
# print "geomean {}-core space overhead".format(maxp), geomean(filterSome([parSpaceOverhead(t) for t in disentangledTags]))

# ===========================================================================

# percent difference (b-a)/|a|
def color_pcd(b, a):
  try:
    xx = 100.0 * (b-a) / abs(a)
    result = ("+" if xx >= 0.0 else "") + ("{:.1f}%".format(xx))
    if xx >= 10.0:
      return red(result)
    elif xx >= 5.0:
      return orange(result)
    elif xx <= -10.0:
      return green(result)
    elif xx <= -5.0:
      return lightgreen(result)
    else:
      return gray(result)
  except:
    return None

# def sp(kb):
#   if kb is None:
#     return None
#   num = kb
#   for unit in ['K','M','G']:
#     if num < 1000:
#       return "%3.1f %s" % (num, unit)
#     num = num / 1000
#   return "%3.1f %s" % (num, 'T')

def defaultAlign(i):
  return "r" if i == 0 else "l"


# ============================================================================

sortShootout = [
  ("C++", "cpp", "msort"),
  ("MPL", "mpl", "msort-ints"),
  ("Java", "java", "sort-longs"),
  ("OCaml", "ocaml", "msort-ints"),
  ("Go", "go", "msort")
]

sortShootoutBaseline = \
  min(filterSome(
    [tm(averageTime(D, config, tag, 1)) for name,config,tag in sortShootout]
  ))

headers1 = ['System', 'T(1)', 'T({})'.format(maxp), 'SelfSU', 'SU', 'R(1)', 'R({})'.format(maxp)]
tt = [headers1, "="]
for name, config, tag in sortShootout:
  t1 = tm(averageTime(D, config, tag, 1))
  tp = tm(averageTime(D, config, tag, maxp))
  r1 = spg(averageSpace(D, config, tag, 1))
  rp = spg(averageSpace(D, config, tag, maxp))
  thisRow = [
    t1,
    tp,
    su(sd(t1,tp)),
    su(sd(sortShootoutBaseline, tp)),
    r1,
    rp
  ]
  thisRow = [name] + [str(x) if x is not None else "--" for x in thisRow]
  tt.append(thisRow)

print("SORT SHOOTOUT")
print(table(tt, defaultAlign))
print("")

# ============================================================================

tokensShootout = [
  ("C++", "cpp", "tokens"),
  ("MPL", "mpl", "tokens"),
  ("Java", "java", "tokens"),
  ("OCaml", "ocaml", "tokens"),
  ("Go", "go", "tokens")
]

tokensShootoutBaseline = \
  min(filterSome(
    [tm(averageTime(D, config, tag, 1)) for name,config,tag in tokensShootout]
  ))

headers1 = ['System', 'T(1)', 'T({})'.format(maxp), 'SelfSU', 'SU', 'R(1)', 'R({})'.format(maxp)]
tt = [headers1, "="]
for name, config, tag in tokensShootout:
  t1 = tm(averageTime(D, config, tag, 1))
  tp = tm(averageTime(D, config, tag, maxp))
  r1 = spg(averageSpace(D, config, tag, 1))
  rp = spg(averageSpace(D, config, tag, maxp))
  thisRow = [
    t1,
    tp,
    su(sd(t1,tp)),
    su(sd(tokensShootoutBaseline, tp)),
    r1,
    rp
  ]
  thisRow = [name] + [str(x) if x is not None else "--" for x in thisRow]
  tt.append(thisRow)

print("TOKENS SHOOTOUT")
print(table(tt, defaultAlign))
print("")

# ============================================================================

primesShootout = [
  ("C++", "cpp", "primes"),
  ("MPL", "mpl", "primes"),
  ("Java", "java", "primes"),
  ("OCaml", "ocaml", "primes"),
  ("Go", "go", "primes")
]

primesShootoutBaseline = \
  min(filterSome(
    [tm(averageTime(D, config, tag, 1)) for name,config,tag in primesShootout]
  ))

headers1 = ['System', 'T(1)', 'T({})'.format(maxp), 'SelfSU', 'SU', 'R(1)', 'R({})'.format(maxp)]
tt = [headers1, "="]
for name, config, tag in primesShootout:
  t1 = tm(averageTime(D, config, tag, 1))
  tp = tm(averageTime(D, config, tag, maxp))
  r1 = spg(averageSpace(D, config, tag, 1))
  rp = spg(averageSpace(D, config, tag, maxp))
  thisRow = [
    t1,
    tp,
    su(sd(t1,tp)),
    su(sd(primesShootoutBaseline, tp)),
    r1,
    rp
  ]
  thisRow = [name] + [str(x) if x is not None else "--" for x in thisRow]
  tt.append(thisRow)

print("PRIMES SHOOTOUT")
print(table(tt, defaultAlign))
print("")

# ============================================================================

headers = ['', 'T(1)', None, None, 'T({})'.format(maxp), None, None, 'R(1)', None, None, 'R({})'.format(maxp), None, None]
headers1 = ['Benchmark', 'O', 'M', 'O/M', 'O', 'M', 'O/M', 'O', 'M', 'O/M', 'O', 'M', 'O/M']
tt = [headers, "-", headers1, "="]
for tag in ocamlTags:
  mt1 = tm(averageTime(D, 'mpl', tag, 1))
  mtp = tm(averageTime(D, 'mpl', tag, maxp))
  mr1 = spg(averageSpace(D, 'mpl', tag, 1))
  mrp = spg(averageSpace(D, 'mpl', tag, maxp))

  ot1 = tm(averageTime(D, 'ocaml', tag, 1))
  otp = tm(averageTime(D, 'ocaml', tag, maxp))
  or1 = spg(averageSpace(D, 'ocaml', tag, 1))
  orp = spg(averageSpace(D, 'ocaml', tag, maxp))

  thisRow = [
    ot1,
    mt1,
    ov(sd(ot1,mt1)),
    otp,
    mtp,
    ov(sd(otp,mtp)),
    or1,
    mr1,
    ov(sd(or1,mr1)),
    orp,
    mrp,
    ov(sd(orp,mrp))
  ]
  thisRow = [displayTag(tag)] + [str(x) if x is not None else "--" for x in thisRow]
  tt.append(thisRow)

print("OCAML COMPARISON")
print(table(tt, defaultAlign))
print("")

# ============================================================================

headers = ['', 'T(1)', None, None, 'T({})'.format(maxp), None, None, 'R(1)', None, None, 'R({})'.format(maxp), None, None]
headers1 = ['Benchmark', 'C', 'M', 'M/C', 'C', 'M', 'M/C', 'C', 'M', 'M/C', 'C', 'M', 'M/C']
tt = [headers, "-", headers1, "="]
for tag in sorted(cppTags.keys()):
  mt1 = tm(averageTime(D, 'mpl', tag, 1))
  mtp = tm(averageTime(D, 'mpl', tag, maxp))
  mr1 = spg(averageSpace(D, 'mpl', tag, 1))
  mrp = spg(averageSpace(D, 'mpl', tag, maxp))

  ct1 = tm(averageTime(D, 'cpp', cppTags[tag], 1))
  ctp = tm(averageTime(D, 'cpp', cppTags[tag], maxp))
  cr1 = spg(averageSpace(D, 'cpp', cppTags[tag], 1))
  crp = spg(averageSpace(D, 'cpp', cppTags[tag], maxp))

  thisRow = [
    ct1,
    mt1,
    ov(sd(mt1,ct1)),
    ctp,
    mtp,
    ov(sd(mtp,ctp)),
    cr1,
    mr1,
    ov(sd(mr1,cr1)),
    crp,
    mrp,
    ov(sd(mrp,crp))
  ]
  thisRow = [displayTag(tag)] + [str(x) if x is not None else "--" for x in thisRow]
  tt.append(thisRow)

print("C++ COMPARISON")
print(table(tt, defaultAlign))
print("")

# ============================================================================

headers = ['', 'T(1)', None, None, 'T({})'.format(maxp), None, None, 'R(1)', None, None, 'R({})'.format(maxp), None, None]
headers1 = ['Benchmark', 'J', 'M', 'J/M', 'J', 'M', 'J/M', 'J', 'M', 'J/M', 'J', 'M', 'J/M']
tt = [headers, "-", headers1, "="]
for tag in sorted(javaTags.keys()):
  mt1 = tm(averageTime(D, 'mpl', tag, 1))
  mtp = tm(averageTime(D, 'mpl', tag, maxp))
  mr1 = spg(averageSpace(D, 'mpl', tag, 1))
  mrp = spg(averageSpace(D, 'mpl', tag, maxp))

  jt1 = tm(averageTime(D, 'java', javaTags[tag], 1))
  jtp = tm(averageTime(D, 'java', javaTags[tag], maxp))
  jr1 = spg(averageSpace(D, 'java', javaTags[tag], 1))
  jrp = spg(averageSpace(D, 'java', javaTags[tag], maxp))

  thisRow = [
    jt1,
    mt1,
    ov(sd(jt1, mt1)),
    jtp,
    mtp,
    ov(sd(jtp, mtp)),
    jr1,
    mr1,
    ov(sd(jr1, mr1)),
    jrp,
    mrp,
    ov(sd(jrp, mrp))
  ]
  thisRow = [displayTag(tag)] + [str(x) if x is not None else "--" for x in thisRow]
  tt.append(thisRow)

print("JAVA COMPARISON")
print(table(tt, defaultAlign))
print("")

# ============================================================================

headers = ['', 'T(1)', None, None, 'T({})'.format(maxp), None, None, 'R(1)', None, None, 'R({})'.format(maxp), None, None]
headers1 = ['Benchmark', 'G', 'M', 'G/M', 'G', 'M', 'G/M', 'G', 'M', 'G/M', 'G', 'M', 'G/M']
tt = [headers, "-", headers1, "="]
for tag in sorted(goTags.keys()):
  mt1 = tm(averageTime(D, 'mpl', tag, 1))
  mtp = tm(averageTime(D, 'mpl', tag, maxp))
  mr1 = spg(averageSpace(D, 'mpl', tag, 1))
  mrp = spg(averageSpace(D, 'mpl', tag, maxp))

  jt1 = tm(averageTime(D, 'go', goTags[tag], 1))
  jtp = tm(averageTime(D, 'go', goTags[tag], maxp))
  jr1 = spg(averageSpace(D, 'go', goTags[tag], 1))
  jrp = spg(averageSpace(D, 'go', goTags[tag], maxp))

  thisRow = [
    jt1,
    mt1,
    ov(sd(jt1, mt1)),
    jtp,
    mtp,
    ov(sd(jtp, mtp)),
    jr1,
    mr1,
    ov(sd(jr1, mr1)),
    jrp,
    mrp,
    ov(sd(jrp, mrp))
  ]
  thisRow = [displayTag(tag)] + [str(x) if x is not None else "--" for x in thisRow]
  tt.append(thisRow)

print("GO COMPARISON")
print(table(tt, defaultAlign))
print("")

# ============================================================================
# LaTeX figure: OCaml comparison
# ============================================================================

proc1TimeRats = []
proc72TimeRats = []
proc1SpaceRats = []
proc72SpaceRats = []

ensureFigDir()
ocamlComparisonTable = "figures/ocaml-space-time-comparison.tex"
with open(ocamlComparisonTable, 'w') as output:
  for tag in ocamlTags:
    or1 = spg(averageSpace(D, 'ocaml', tag, 1))
    orp = spg(averageSpace(D, 'ocaml', tag, maxp))
    mr1 = spg(averageSpace(D, 'mpl', tag, 1))
    mrp = spg(averageSpace(D, 'mpl', tag, maxp))

    ot1 = tm(averageTime(D, 'ocaml', tag, 1))
    otp = tm(averageTime(D, 'ocaml', tag, maxp))
    mt1 = tm(averageTime(D, 'mpl', tag, 1))
    mtp = tm(averageTime(D, 'mpl', tag, maxp))

    proc1TimeRats.append(ovv(sd(ot1, mt1)))
    proc72TimeRats.append(ovv(sd(otp, mtp)))
    proc1SpaceRats.append(ovv(sd(or1, mr1)))
    proc72SpaceRats.append(ovv(sd(orp, mrp)))

    row = \
      [ fmt(ot1)
      , fmt(mt1)
      , makeBold(fmt(ov(sd(ot1, mt1))))
      , fmt(otp)
      , fmt(mtp)
      , makeBold(fmt(ov(sd(otp, mtp))))
      , sfmt(or1)
      , sfmt(mr1)
      , makeBold(fmt(ov(sd(or1, mr1))))
      , sfmt(orp)
      , sfmt(mrp)
      , makeBold(fmt(ov(sd(orp, mrp))))
      ]

    output.write(" & ".join([displayTag(tag)] + row))
    output.write("  \\\\\n")

  output.write("\\midrule\n")
  row = [
    "", "", makeBold(fmt(ov(geomean(proc1TimeRats)))),
    "", "", makeBold(fmt(ov(geomean(proc72TimeRats)))),
    "", "", makeBold(fmt(ov(geomean(proc1SpaceRats)))),
    "", "", makeBold(fmt(ov(geomean(proc72SpaceRats))))
  ]
  output.write(" & ".join(["geomean"] + row))
  output.write("  \\\\\n")
print("[INFO] wrote to {}".format(ocamlComparisonTable))

# ============================================================================
# LaTeX figure: Java comparison
# ============================================================================

proc1TimeRats = []
proc72TimeRats = []
proc1SpaceRats = []
proc72SpaceRats = []

ensureFigDir()
javaComparisonTable = "figures/java-space-time-comparison.tex"
with open(javaComparisonTable, 'w') as output:
  for tag in sorted(javaTags.keys()):
    or1 = spg(averageSpace(D, 'java', javaTags[tag], 1))
    orp = spg(averageSpace(D, 'java', javaTags[tag], maxp))
    mr1 = spg(averageSpace(D, 'mpl', tag, 1))
    mrp = spg(averageSpace(D, 'mpl', tag, maxp))

    ot1 = tm(averageTime(D, 'java', javaTags[tag], 1))
    otp = tm(averageTime(D, 'java', javaTags[tag], maxp))
    mt1 = tm(averageTime(D, 'mpl', tag, 1))
    mtp = tm(averageTime(D, 'mpl', tag, maxp))

    proc1TimeRats.append(ovv(sd(ot1, mt1)))
    proc72TimeRats.append(ovv(sd(otp, mtp)))
    proc1SpaceRats.append(ovv(sd(or1, mr1)))
    proc72SpaceRats.append(ovv(sd(orp, mrp)))

    row = \
      [ fmt(ot1)
      , fmt(mt1)
      , makeBold(fmt(ov(sd(ot1, mt1))))
      , fmt(otp)
      , fmt(mtp)
      , makeBold(fmt(ov(sd(otp, mtp))))
      , sfmt(or1)
      , sfmt(mr1)
      , makeBold(fmt(ov(sd(or1, mr1))))
      , sfmt(orp)
      , sfmt(mrp)
      , makeBold(fmt(ov(sd(orp, mrp))))
      ]

    output.write(" & ".join([displayTag(tag)] + row))
    output.write("  \\\\\n")

  output.write("\\midrule\n")
  row = [
    "", "", makeBold(fmt(ov(geomean(proc1TimeRats)))),
    "", "", makeBold(fmt(ov(geomean(proc72TimeRats)))),
    "", "", makeBold(fmt(ov(geomean(proc1SpaceRats)))),
    "", "", makeBold(fmt(ov(geomean(proc72SpaceRats))))
  ]
  output.write(" & ".join(["geomean"] + row))
  output.write("  \\\\\n")
print("[INFO] wrote to {}".format(javaComparisonTable))

# ============================================================================
# LaTeX figure: Go comparison
# ============================================================================

proc1TimeRats = []
proc72TimeRats = []
proc1SpaceRats = []
proc72SpaceRats = []

ensureFigDir()
goComparisonTable = "figures/go-space-time-comparison.tex"
with open(goComparisonTable, 'w') as output:
  for tag in sorted(goTags.keys()):
    or1 = spg(averageSpace(D, 'go', goTags[tag], 1))
    orp = spg(averageSpace(D, 'go', goTags[tag], maxp))
    mr1 = spg(averageSpace(D, 'mpl', tag, 1))
    mrp = spg(averageSpace(D, 'mpl', tag, maxp))

    ot1 = tm(averageTime(D, 'go', goTags[tag], 1))
    otp = tm(averageTime(D, 'go', goTags[tag], maxp))
    mt1 = tm(averageTime(D, 'mpl', tag, 1))
    mtp = tm(averageTime(D, 'mpl', tag, maxp))

    proc1TimeRats.append(ovv(sd(ot1, mt1)))
    proc72TimeRats.append(ovv(sd(otp, mtp)))
    proc1SpaceRats.append(ovv(sd(or1, mr1)))
    proc72SpaceRats.append(ovv(sd(orp, mrp)))

    row = \
      [ fmt(ot1)
      , fmt(mt1)
      , makeBold(fmt(ov(sd(ot1, mt1))))
      , fmt(otp)
      , fmt(mtp)
      , makeBold(fmt(ov(sd(otp, mtp))))
      , sfmt(or1)
      , sfmt(mr1)
      , makeBold(fmt(ov(sd(or1, mr1))))
      , sfmt(orp)
      , sfmt(mrp)
      , makeBold(fmt(ov(sd(orp, mrp))))
      ]

    output.write(" & ".join([displayTag(tag)] + row))
    output.write("  \\\\\n")

  output.write("\\midrule\n")
  row = [
    "", "", makeBold(fmt(ov(geomean(proc1TimeRats)))),
    "", "", makeBold(fmt(ov(geomean(proc72TimeRats)))),
    "", "", makeBold(fmt(ov(geomean(proc1SpaceRats)))),
    "", "", makeBold(fmt(ov(geomean(proc72SpaceRats))))
  ]
  output.write(" & ".join(["geomean"] + row))
  output.write("  \\\\\n")
print("[INFO] wrote to {}".format(goComparisonTable))

# ============================================================================
# LaTeX figure: C++ comparison
# ============================================================================

proc1TimeRats = []
proc72TimeRats = []
proc1SpaceRats = []
proc72SpaceRats = []
# multiTimeRats = []

ensureFigDir()
cppComparisonTable = "figures/cpp-space-time-comparison.tex"
with open(cppComparisonTable, 'w') as output:
  for tag in sorted(cppTags.keys()):
    cr1 = spg(averageSpace(D, 'cpp', cppTags[tag], 1))
    crp = spg(averageSpace(D, 'cpp', cppTags[tag], maxp))
    mr1 = spg(averageSpace(D, 'mpl', tag, 1))
    mrp = spg(averageSpace(D, 'mpl', tag, maxp))

    ct1 = tm(averageTime(D, 'cpp', cppTags[tag], 1))
    ctp = tm(averageTime(D, 'cpp', cppTags[tag], maxp))
    mt1 = tm(averageTime(D, 'mpl', tag, 1))
    mtp = tm(averageTime(D, 'mpl', tag, maxp))

    proc1TimeRats.append(ovv(sd(mt1, ct1)))
    proc72TimeRats.append(ovv(sd(mtp, ctp)))
    proc1SpaceRats.append(ovv(sd(mr1, cr1)))
    proc72SpaceRats.append(ovv(sd(mrp, crp)))

    row = \
      [ fmt(ct1)
      , fmt(mt1)
      , makeBold(fmt(ov(sd(mt1, ct1))))
      , fmt(ctp)
      , fmt(mtp)
      , makeBold(fmt(ov(sd(mtp, ctp))))
      , sfmt(cr1)
      , sfmt(mr1)
      , makeBold(fmt(ov(sd(mr1, cr1))))
      , sfmt(crp)
      , sfmt(mrp)
      , makeBold(fmt(ov(sd(mrp, crp))))
      ]

    output.write(" & ".join([displayTag(tag)] + row))
    output.write("  \\\\\n")

  output.write("\\midrule\n")
  row = [
    "", "", makeBold(fmt(ov(geomean(proc1TimeRats)))),
    "", "", makeBold(fmt(ov(geomean(proc72TimeRats)))),
    "", "", makeBold(fmt(ov(geomean(proc1SpaceRats)))),
    "", "", makeBold(fmt(ov(geomean(proc72SpaceRats))))
  ]
  output.write(" & ".join(["geomean"] + row))
  output.write("  \\\\\n")
print("[INFO] wrote to {}".format(cppComparisonTable))

# ============================================================================
# LaTeX figure: MLton comparison
# ============================================================================

ovRats = []
suRats = []
bu1Rats = []
bupRats = []

mltonCmpTags = sorted([
  tag for tag in orderedTags
  if (averageTime(D, 'mpl', tag, maxp) is not None)
  and tag not in sandmarkTags
], key=displayTag)

ensureFigDir()
mltonCmpTable = "figures/mlton-comparison.tex"
with open(mltonCmpTable, 'w') as output:
  for tag in mltonCmpTags:
    mltont = tm(averageTime(D, 'mlton', tag, 1))
    mt1 = tm(averageTime(D, 'mpl', tag, 1))
    mtp = tm(averageTime (D, 'mpl', tag, maxp))

    mltonr = spg(averageSpace(D, 'mlton', tag, 1))
    mr1 = spg(averageSpace(D, 'mpl', tag, 1))
    mrp = spg(averageSpace (D, 'mpl', tag, maxp))

    ovRats.append(ovv(sd(mt1, mltont)))
    suRats.append(ovv(sd(mltont, mtp)))
    bu1Rats.append(ovv(sd(mr1, mltonr)))
    bupRats.append(ovv(sd(mrp, mltonr)))

    row = [
      mltont,
      mt1,
      makeBold(fmt(ov(sd(mt1, mltont)))),
      mtp,
      makeBold(fmt(su(sd(mltont, mtp)))),
      sfmt(mltonr),
      sfmt(mr1),
      makeBold(fmt(bu(sd(mr1, mltonr)))),
      sfmt(mrp),
      makeBold(fmt(bu(sd(mrp, mltonr))))
    ]

    row = [ fmt(x) for x in row ]
    output.write(" & ".join([displayTag(tag)] + row))
    output.write("  \\\\\n")

  output.write("\\midrule\n")
  row = [
    "", "", makeBold(fmt(ov(geomean(ovRats)))),
    "",     makeBold(fmt(su(geomean(suRats)))),
    "", "", makeBold(fmt(ov(geomean(bu1Rats)))),
    "",     makeBold(fmt(ov(geomean(bupRats))))
  ]
  output.write(" & ".join(["geomean"] + row))
  output.write("  \\\\\n")
print("[INFO] wrote to {}".format(mltonCmpTable))

# ============================================================================
# PDF: Speedup plot
# ============================================================================

import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

def speedupPlot(outputName, tagsSortedBySpeedups, tagsSortedByName):
  plt.figure(figsize=(7,7))
  # markers = ['o','v','^','<','>','s','*','d','D','+','x','|','','','','','']
  colors = ['blue', 'darkturquoise', 'darkgreen', 'darkviolet', 'red', 'goldenrod','dimgrey', 'brown']
  # 'black',
  markers = ['o','v','^','<','>','s','d']
  # ,'D','*','P','X'
  linestyles = ['solid', 'dashed','dashdot']

  # markers = ['.'] * len(speedupTags)
  procs = P

  fontSize = 14
  legendFontSize = 12
  markerSize = 8

  usedFull = dict()
  usedNoc = dict()

  plt.plot([0,maxp+1], [0,maxp+1], marker="", color="grey", linewidth=0.8)
  lines = []
  for tag in tagsSortedByName:
    try:
      baseline = averageTime(D, 'mlton', tag, 1)
      speedups = map((lambda p: getspeedup(tag, p)), procs)

      i = tagsSortedBySpeedups.index(tag)
      ci = i % len(colors)
      mi = i % len(markers)
      si = i % len(linestyles)

      if tuple((ci, mi, si)) in usedFull:
        otherTag = usedFull[tuple((ci, mi, si))]
        sys.stderr.write('[WARN] uh oh: {} and {} have same style\n'.format(tag, otherTag))
      else:
        usedFull[tuple((ci, mi, si))] = tag

      if tuple((mi, si)) in usedNoc:
        otherTag = usedNoc[tuple((mi, si))]
        if tuple((ci, mi, si)) not in usedFull:
          sys.stderr.write('[WARN] uh oh: {} and {} only differ by color\n'.format(tag, otherTag))
      else:
        usedNoc[tuple((mi, si))] = tag

      color = colors[ci]
      marker = markers[mi]
      linestyle = linestyles[si]
      lines.append(plt.plot(procs, speedups, linestyle=linestyle, marker=marker, markersize=markerSize, mec='black', mew=0.0, linewidth=1.3, color=color, alpha=0.8))
    except Exception as e:
      sys.stderr.write('[WARN] error while plotting speedup for {}: {}\n'.format(tag, e))

  # this sets the legend.
  font = {
    'size': legendFontSize,
    #'family' : 'normal',
    #'weight' : 'bold',
  }
  matplotlib.rc('font', **font)

  # make sure to use truetype fonts
  matplotlib.rcParams['pdf.fonttype'] = 42
  matplotlib.rcParams['ps.fonttype'] = 42

  # set legend position
  matplotlib.rcParams['legend.loc'] = 'upper left'

  ticks = [1] + range(10, maxp+1, 10)

  plt.xlabel('Processors', fontsize=fontSize)
  plt.ylabel('Speedup', fontsize=fontSize)
  plt.yticks(ticks, fontsize=fontSize)
  plt.xticks(ticks, fontsize=fontSize)
  plt.xlim(0, 74)
  plt.ylim(0, 74)
  plt.gca().grid(axis='both', linestyle='dotted')
  plt.gca().set_axisbelow(True)
  # plt.margins(y=10)

  if True:
    plt.legend(
      [b[0] for b in lines],
      map(displayTag, tagsSortedByName),
      # bbox_to_anchor=(-0.16,1.05),
      # loc='upper right',
      bbox_to_anchor=(-0.1,1.01),
      loc='lower left',
      ncol=3
    )

  ensureFigDir()
  # outputName = 'figures/mpl-speedups.pdf'
  plt.savefig(outputName, bbox_inches='tight')
  sys.stdout.write("[INFO] output written to {}\n".format(outputName))
  plt.close()



speedupTags = sorted([
  tag for tag in orderedTags
  if (averageTime(D, 'mpl', tag, maxp) is not None)
  and tag not in sandmarkTags
], key=displayTag)

def getspeedup(tag, p):
  baseline = averageTime(D, 'mlton', tag, 1)
  try:
    return baseline / averageTime(D, 'mpl', tag, p)
  except Exception as e:
    sys.stderr.write('[WARN] error while plotting speedup for {} at P={}: {}\n'.format(tag, p, e))
    return None

sortedBySpeedups = sorted(speedupTags, key=(lambda tag: 1.0/getspeedup(tag, maxp)))

groupA = sortedBySpeedups[::2]
groupB = sortedBySpeedups[1::2]

groupATags = [t for t in speedupTags if t in groupA]
groupBTags = [t for t in speedupTags if t in groupB]

speedupPlot("figures/mpl-speedups.pdf", sortedBySpeedups, speedupTags)
speedupPlot("figures/mpl-speedups-1.pdf", groupA, groupATags)
speedupPlot("figures/mpl-speedups-2.pdf", groupB, groupBTags)

# ============================================================================
# ============================================================================
# ============================================================================
print("[INFO] done reporting {}".format(timingsFile))