Update benchmarks for 0.4.0

.gitignore

@@ -1,5 +1,7 @@
+binaries/
+target/
 bin/.coursier
 bin/.scalafmt*
 results/
 *.iprof
-.idea
+.idea

confs/native-image-pgo/build.properties

@@ -0,0 +1 @@
+sbt.version = 0.13.17

confs/native-image/build.properties

@@ -0,0 +1 @@
+sbt.version = 0.13.17

confs/scala-native-0.3.7/build.properties

@@ -0,0 +1 @@
+sbt.version = 0.13.17

confs/scala-native-0.3.7/run

@@ -1 +1 @@
-target/scala-2.11/benchs-out
+target/scala-2.11/scala-native-benchmarks-out

confs/scala-native-0.3.8/build.properties

@@ -0,0 +1 @@
+sbt.version = 0.13.17

confs/scala-native-0.3.8/run

@@ -1 +1 @@
-target/scala-2.11/benchs-out
+target/scala-2.11/scala-native-benchmarks-out

confs/scala-native-0.3.9/build.properties

@@ -0,0 +1 @@
+sbt.version = 0.13.17

build.sbt → confs/scala-native-0.3.9/build.sbt

@@ -3,4 +3,3 @@ enablePlugins(ScalaNativePlugin)
 nativeLinkStubs := true
 nativeGC := "immix"
 nativeMode := "release"
-nativeLTO := "thin"

confs/scala-native-0.3.9/plugins.sbt

@@ -0,0 +1 @@
+addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.3.9")

confs/scala-native-0.3.9/run

@@ -0,0 +1 @@
+target/scala-2.11/scala-native-benchmarks-out

confs/scala-native-0.4.0-2.11/build.properties

@@ -0,0 +1 @@
+sbt.version = 1.4.1

confs/scala-native-0.4.0-2.11/build.sbt

@@ -0,0 +1,8 @@
+scalaVersion := "2.11.12"
+enablePlugins(ScalaNativePlugin)
+import scala.scalanative.build
+nativeConfig ~= {
+  _.withGC(build.GC.immix)
+   .withMode(build.Mode.releaseFull)
+   .withLTO(build.LTO.thin)
+}

confs/scala-native-0.4.0-2.11/plugins.sbt

@@ -0,0 +1 @@
+addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.4.0")

confs/scala-native-0.4.0-2.11/run

@@ -0,0 +1 @@
+target/scala-2.11/scala-native-benchmarks-out

confs/scala-native-0.4.0-2.12/build.properties

@@ -0,0 +1 @@
+sbt.version = 1.4.1

confs/scala-native-0.4.0-2.12/build.sbt

@@ -0,0 +1,8 @@
+scalaVersion := "2.12.13"
+enablePlugins(ScalaNativePlugin)
+import scala.scalanative.build
+nativeConfig ~= {
+  _.withGC(build.GC.immix)
+   .withMode(build.Mode.releaseFull)
+   .withLTO(build.LTO.thin)
+}

confs/scala-native-0.4.0-2.12/compile

@@ -0,0 +1 @@
+nativeLink

confs/scala-native-0.4.0-2.12/plugins.sbt

@@ -0,0 +1 @@
+addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.4.0")

confs/scala-native-0.4.0-2.12/run

@@ -0,0 +1 @@
+target/scala-2.12/scala-native-benchmarks-out

confs/scala-native-0.4.0-M2/build.properties

@@ -0,0 +1 @@
+sbt.version = 0.13.17

confs/baseline/build.sbt → confs/scala-native-0.4.0-M2/build.sbt

@@ -2,5 +2,5 @@ scalaVersion := "2.11.12"
 enablePlugins(ScalaNativePlugin)
 nativeLinkStubs := true
 nativeGC := "immix"
-nativeMode := "release"
+nativeMode := "release-full"
 nativeLTO := "thin"

confs/scala-native-0.4.0-M2/compile

@@ -0,0 +1 @@
+nativeLink

confs/scala-native-0.4.0-M2/plugins.sbt

@@ -0,0 +1 @@
+addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.4.0-M2")

confs/scala-native-0.4.0-M2/run

@@ -0,0 +1 @@
+target/scala-2.11/scala-native-benchmarks-out

scripts/compiletime-summary.py

@@ -0,0 +1,31 @@
+from run import benchmarks, runs, configurations
+
+import numpy as np
+
+def config_data(bench, conf):
+    try:
+        with open('results/{}/{}/compile_time'.format(conf, bench)) as data:
+            return float(data.read())
+    except IOError:
+        return 0
+
+def compile_time():
+    out = []
+    for bench in benchmarks:
+        res = []
+        for conf in configurations:
+            try:
+                res.append(np.percentile(config_data(bench, conf), 10))
+            except IndexError:
+                res.append(0)
+        out.append(res)
+    return out
+
+if __name__ == '__main__':
+    leading = ['name']
+    for conf in configurations:
+        leading.append(conf)
+    print(','.join(leading))
+    for bench, res in zip(benchmarks, compile_time()):
+        print(','.join([bench] + list(map(str, res))))
+

scripts/run.py

@@ -4,6 +4,7 @@
 import errno
 import subprocess as subp
 import shutil as sh
+import time
 
 def mkdir(path):
     try:
@@ -35,10 +36,13 @@ def run(cmd):
     return subp.check_output(cmd)
 
 def compile(bench, compilecmd):
+    start = time.time_ns()
     cmd = [sbt, '-J-Xmx6G', 'clean']
     cmd.append('set mainClass in Compile := Some("{}")'.format(bench))
     cmd.append(compilecmd)
-    return run(cmd)
+    run(cmd)
+    end = time.time_ns()
+    return end - start
 
 sbt = where('sbt')
 
@@ -62,10 +66,10 @@ def compile(bench, compilecmd):
 ]
 
 configurations = [
-        # 'jvm',
-        # 'scala-native-0.3.8',
-        # 'baseline',
-        'current'
+        'scala-native-0.3.9',
+        'scala-native-0.4.0-M2',
+        'scala-native-0.4.0-2.11',
+        'scala-native-0.4.0-2.12',
 ]
 
 if 'GRAALVM_HOME' in os.environ:
@@ -75,40 +79,57 @@ def compile(bench, compilecmd):
     ]
 
 runs = 20
-batches = 4000
+batches = 2000
 batch_size = 1
 
 if __name__ == "__main__":
     for conf in configurations:
         for bench in benchmarks:
             print('--- conf: {}, bench: {}'.format(conf, bench))
 
+            resultsdir = os.path.join('results', conf, bench)
+            mkdir(resultsdir)
+
+            bindir = os.path.join("binaries", conf, bench)
+            mkdir(bindir)
+
+            binfile = os.path.join(bindir, bench)
+
             input = slurp(os.path.join('input', bench))
             output = slurp(os.path.join('output', bench))
             compilecmd = slurp(os.path.join('confs', conf, 'compile'))
             runcmd = slurp(os.path.join('confs', conf, 'run')).replace('$BENCH', bench).replace('$HOME', os.environ['HOME']).split(' ')
 
-            if os.path.exists(os.path.join('confs', conf, 'build.sbt')):
-                sh.copyfile(os.path.join('confs', conf, 'build.sbt'), 'build.sbt')
-            else:
-                os.remove('build.sbt')
+            if not os.path.exists(binfile):
+                if os.path.exists(os.path.join('confs', conf, 'build.sbt')):
+                    sh.copyfile(os.path.join('confs', conf, 'build.sbt'), 'build.sbt')
+                else:
+                    os.remove('build.sbt')
 
-            if os.path.exists(os.path.join('confs', conf, 'plugins.sbt')):
-                sh.copyfile(os.path.join('confs', conf, 'plugins.sbt'), 'project/plugins.sbt')
-            else:
-                os.remove('project/plugins.sbt')
+                if os.path.exists(os.path.join('confs', conf, 'build.properties')):
+                    sh.copyfile(os.path.join('confs', conf, 'build.properties'), 'project/build.properties')
+                else:
+                    os.remove('project/build.properties')
 
-            compile(bench, compilecmd)
-            resultsdir = os.path.join('results', conf, bench)
-            mkdir(resultsdir)
+                if os.path.exists(os.path.join('confs', conf, 'plugins.sbt')):
+                    sh.copyfile(os.path.join('confs', conf, 'plugins.sbt'), 'project/plugins.sbt')
+                else:
+                    os.remove('project/plugins.sbt')
+
+                compile_time = compile(bench, compilecmd)
+
+                with open(os.path.join(resultsdir, "compile_time"), 'w') as f:
+                    f.write(str(compile_time))
+
+                sh.move(runcmd[0], binfile)
 
-            for n in xrange(runs):
+            for n in range(runs):
                 print('--- run {}/{}'.format(n, runs))
 
-                cmd = []
-                cmd.extend(runcmd)
+                cmd = [binfile]
+                cmd.extend(runcmd[1:])
                 cmd.extend([str(batches), str(batch_size), input, output])
                 out = run(cmd)
-                with open(os.path.join(resultsdir, str(n)), 'w+') as resultfile:
+                with open(os.path.join(resultsdir, str(n)), 'wb') as resultfile:
                     resultfile.write(out)
 

scripts/size-summary.py

@@ -0,0 +1,30 @@
+from run import benchmarks, runs, configurations
+import os
+import numpy as np
+
+def config_data(bench, conf):
+    try:
+        return len(open('binaries/{}/{}/{}'.format(conf, bench, bench), 'rb').read())
+    except IOError:
+        return 0
+
+def binary_size():
+    out = []
+    for bench in benchmarks:
+        res = []
+        for conf in configurations:
+            try:
+                res.append(np.percentile(config_data(bench, conf), 10))
+            except IndexError:
+                res.append(0)
+        out.append(res)
+    return out
+
+if __name__ == '__main__':
+    leading = ['name']
+    for conf in configurations:
+        leading.append(conf)
+    print(','.join(leading))
+    for bench, res in zip(benchmarks, binary_size()):
+        print(','.join([bench] + list(map(str, res))))
+

scripts/summary.py

@@ -4,7 +4,7 @@
 
 def config_data(bench, conf):
     out = []
-    for run in xrange(runs):
+    for run in range(runs):
         try:
             points = []
             with open('results/{}/{}/{}'.format(conf, bench, run)) as data:
@@ -27,7 +27,7 @@ def peak_performance():
         res = []
         for conf in configurations:
             try:
-                res.append(np.percentile(config_data(bench, conf), 50))
+                res.append(np.percentile(config_data(bench, conf), 10))
             except IndexError:
                 res.append(0)
         out.append(res)
@@ -37,7 +37,7 @@ def peak_performance():
     leading = ['name']
     for conf in configurations:
         leading.append(conf)
-    print ','.join(leading)
+    print(','.join(leading))
     for bench, res in zip(benchmarks, peak_performance()):
-        print ','.join([bench] + list(map(str, res)))
+        print(','.join([bench] + list(map(str, res))))