From 24cb5742980a5ff50518cfabd651d59dca1b734d Mon Sep 17 00:00:00 2001 From: Denys Shabalin <320966+densh@users.noreply.github.com> Date: Sun, 7 Feb 2021 12:21:48 +0100 Subject: [PATCH] Update benchmarks for 0.4.0 --- .gitignore | 4 +- confs/baseline/plugins.sbt | 1 - confs/baseline/run | 1 - confs/current/build.sbt | 6 -- confs/current/plugins.sbt | 1 - confs/current/run | 1 - {project => confs/jvm}/build.properties | 0 confs/native-image-pgo/build.properties | 1 + confs/native-image/build.properties | 1 + confs/scala-native-0.3.7/build.properties | 1 + confs/scala-native-0.3.7/run | 2 +- confs/scala-native-0.3.8/build.properties | 1 + confs/scala-native-0.3.8/run | 2 +- confs/scala-native-0.3.9/build.properties | 1 + .../scala-native-0.3.9/build.sbt | 1 - .../{baseline => scala-native-0.3.9}/compile | 0 confs/scala-native-0.3.9/plugins.sbt | 1 + confs/scala-native-0.3.9/run | 1 + .../scala-native-0.4.0-2.11/build.properties | 1 + confs/scala-native-0.4.0-2.11/build.sbt | 8 +++ .../compile | 0 confs/scala-native-0.4.0-2.11/plugins.sbt | 1 + confs/scala-native-0.4.0-2.11/run | 1 + .../scala-native-0.4.0-2.12/build.properties | 1 + confs/scala-native-0.4.0-2.12/build.sbt | 8 +++ confs/scala-native-0.4.0-2.12/compile | 1 + confs/scala-native-0.4.0-2.12/plugins.sbt | 1 + confs/scala-native-0.4.0-2.12/run | 1 + confs/scala-native-0.4.0-M2/build.propert | 0 confs/scala-native-0.4.0-M2/build.properties | 1 + .../build.sbt | 2 +- confs/scala-native-0.4.0-M2/compile | 1 + confs/scala-native-0.4.0-M2/plugins.sbt | 1 + confs/scala-native-0.4.0-M2/run | 1 + project/build.sbt | 1 - project/plugins.sbt | 1 - scripts/compiletime-summary.py | 31 +++++++++ scripts/run.py | 63 ++++++++++++------- scripts/size-summary.py | 30 +++++++++ scripts/summary.py | 8 +-- 40 files changed, 147 insertions(+), 42 deletions(-) delete mode 100644 confs/baseline/plugins.sbt delete mode 100644 confs/baseline/run delete mode 100644 confs/current/build.sbt delete mode 100644 confs/current/plugins.sbt delete mode 100644 confs/current/run rename {project => confs/jvm}/build.properties (100%) create mode 100644 confs/native-image-pgo/build.properties create mode 100644 confs/native-image/build.properties create mode 100644 confs/scala-native-0.3.7/build.properties create mode 100644 confs/scala-native-0.3.8/build.properties create mode 100644 confs/scala-native-0.3.9/build.properties rename build.sbt => confs/scala-native-0.3.9/build.sbt (86%) rename confs/{baseline => scala-native-0.3.9}/compile (100%) create mode 100644 confs/scala-native-0.3.9/plugins.sbt create mode 100644 confs/scala-native-0.3.9/run create mode 100644 confs/scala-native-0.4.0-2.11/build.properties create mode 100644 confs/scala-native-0.4.0-2.11/build.sbt rename confs/{current => scala-native-0.4.0-2.11}/compile (100%) create mode 100644 confs/scala-native-0.4.0-2.11/plugins.sbt create mode 100644 confs/scala-native-0.4.0-2.11/run create mode 100644 confs/scala-native-0.4.0-2.12/build.properties create mode 100644 confs/scala-native-0.4.0-2.12/build.sbt create mode 100644 confs/scala-native-0.4.0-2.12/compile create mode 100644 confs/scala-native-0.4.0-2.12/plugins.sbt create mode 100644 confs/scala-native-0.4.0-2.12/run create mode 100644 confs/scala-native-0.4.0-M2/build.propert create mode 100644 confs/scala-native-0.4.0-M2/build.properties rename confs/{baseline => scala-native-0.4.0-M2}/build.sbt (80%) create mode 100644 confs/scala-native-0.4.0-M2/compile create mode 100644 confs/scala-native-0.4.0-M2/plugins.sbt create mode 100644 confs/scala-native-0.4.0-M2/run delete mode 100644 project/build.sbt delete mode 100644 project/plugins.sbt create mode 100644 scripts/compiletime-summary.py create mode 100644 scripts/size-summary.py diff --git a/.gitignore b/.gitignore index c37ddf5..528a077 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ +binaries/ +target/ bin/.coursier bin/.scalafmt* results/ *.iprof -.idea \ No newline at end of file +.idea diff --git a/confs/baseline/plugins.sbt b/confs/baseline/plugins.sbt deleted file mode 100644 index 2a63bf0..0000000 --- a/confs/baseline/plugins.sbt +++ /dev/null @@ -1 +0,0 @@ -addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.4.0-SNAPSHOT") diff --git a/confs/baseline/run b/confs/baseline/run deleted file mode 100644 index 0ea88ec..0000000 --- a/confs/baseline/run +++ /dev/null @@ -1 +0,0 @@ -target/scala-2.11/benchs-out diff --git a/confs/current/build.sbt b/confs/current/build.sbt deleted file mode 100644 index ae87f31..0000000 --- a/confs/current/build.sbt +++ /dev/null @@ -1,6 +0,0 @@ -scalaVersion := "2.11.12" -enablePlugins(ScalaNativePlugin) -nativeLinkStubs := true -nativeGC := "immix" -nativeMode := "release" -nativeLTO := "thin" diff --git a/confs/current/plugins.sbt b/confs/current/plugins.sbt deleted file mode 100644 index 2a63bf0..0000000 --- a/confs/current/plugins.sbt +++ /dev/null @@ -1 +0,0 @@ -addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.4.0-SNAPSHOT") diff --git a/confs/current/run b/confs/current/run deleted file mode 100644 index 0ea88ec..0000000 --- a/confs/current/run +++ /dev/null @@ -1 +0,0 @@ -target/scala-2.11/benchs-out diff --git a/project/build.properties b/confs/jvm/build.properties similarity index 100% rename from project/build.properties rename to confs/jvm/build.properties diff --git a/confs/native-image-pgo/build.properties b/confs/native-image-pgo/build.properties new file mode 100644 index 0000000..0db266d --- /dev/null +++ b/confs/native-image-pgo/build.properties @@ -0,0 +1 @@ +sbt.version = 0.13.17 diff --git a/confs/native-image/build.properties b/confs/native-image/build.properties new file mode 100644 index 0000000..0db266d --- /dev/null +++ b/confs/native-image/build.properties @@ -0,0 +1 @@ +sbt.version = 0.13.17 diff --git a/confs/scala-native-0.3.7/build.properties b/confs/scala-native-0.3.7/build.properties new file mode 100644 index 0000000..0db266d --- /dev/null +++ b/confs/scala-native-0.3.7/build.properties @@ -0,0 +1 @@ +sbt.version = 0.13.17 diff --git a/confs/scala-native-0.3.7/run b/confs/scala-native-0.3.7/run index 0ea88ec..ae89e34 100644 --- a/confs/scala-native-0.3.7/run +++ b/confs/scala-native-0.3.7/run @@ -1 +1 @@ -target/scala-2.11/benchs-out +target/scala-2.11/scala-native-benchmarks-out diff --git a/confs/scala-native-0.3.8/build.properties b/confs/scala-native-0.3.8/build.properties new file mode 100644 index 0000000..0db266d --- /dev/null +++ b/confs/scala-native-0.3.8/build.properties @@ -0,0 +1 @@ +sbt.version = 0.13.17 diff --git a/confs/scala-native-0.3.8/run b/confs/scala-native-0.3.8/run index 0ea88ec..ae89e34 100644 --- a/confs/scala-native-0.3.8/run +++ b/confs/scala-native-0.3.8/run @@ -1 +1 @@ -target/scala-2.11/benchs-out +target/scala-2.11/scala-native-benchmarks-out diff --git a/confs/scala-native-0.3.9/build.properties b/confs/scala-native-0.3.9/build.properties new file mode 100644 index 0000000..0db266d --- /dev/null +++ b/confs/scala-native-0.3.9/build.properties @@ -0,0 +1 @@ +sbt.version = 0.13.17 diff --git a/build.sbt b/confs/scala-native-0.3.9/build.sbt similarity index 86% rename from build.sbt rename to confs/scala-native-0.3.9/build.sbt index ae87f31..b4a5690 100644 --- a/build.sbt +++ b/confs/scala-native-0.3.9/build.sbt @@ -3,4 +3,3 @@ enablePlugins(ScalaNativePlugin) nativeLinkStubs := true nativeGC := "immix" nativeMode := "release" -nativeLTO := "thin" diff --git a/confs/baseline/compile b/confs/scala-native-0.3.9/compile similarity index 100% rename from confs/baseline/compile rename to confs/scala-native-0.3.9/compile diff --git a/confs/scala-native-0.3.9/plugins.sbt b/confs/scala-native-0.3.9/plugins.sbt new file mode 100644 index 0000000..81f6e69 --- /dev/null +++ b/confs/scala-native-0.3.9/plugins.sbt @@ -0,0 +1 @@ +addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.3.9") diff --git a/confs/scala-native-0.3.9/run b/confs/scala-native-0.3.9/run new file mode 100644 index 0000000..ae89e34 --- /dev/null +++ b/confs/scala-native-0.3.9/run @@ -0,0 +1 @@ +target/scala-2.11/scala-native-benchmarks-out diff --git a/confs/scala-native-0.4.0-2.11/build.properties b/confs/scala-native-0.4.0-2.11/build.properties new file mode 100644 index 0000000..4c69e16 --- /dev/null +++ b/confs/scala-native-0.4.0-2.11/build.properties @@ -0,0 +1 @@ +sbt.version = 1.4.1 diff --git a/confs/scala-native-0.4.0-2.11/build.sbt b/confs/scala-native-0.4.0-2.11/build.sbt new file mode 100644 index 0000000..80c509f --- /dev/null +++ b/confs/scala-native-0.4.0-2.11/build.sbt @@ -0,0 +1,8 @@ +scalaVersion := "2.11.12" +enablePlugins(ScalaNativePlugin) +import scala.scalanative.build +nativeConfig ~= { + _.withGC(build.GC.immix) + .withMode(build.Mode.releaseFull) + .withLTO(build.LTO.thin) +} diff --git a/confs/current/compile b/confs/scala-native-0.4.0-2.11/compile similarity index 100% rename from confs/current/compile rename to confs/scala-native-0.4.0-2.11/compile diff --git a/confs/scala-native-0.4.0-2.11/plugins.sbt b/confs/scala-native-0.4.0-2.11/plugins.sbt new file mode 100644 index 0000000..2f4e38a --- /dev/null +++ b/confs/scala-native-0.4.0-2.11/plugins.sbt @@ -0,0 +1 @@ +addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.4.0") diff --git a/confs/scala-native-0.4.0-2.11/run b/confs/scala-native-0.4.0-2.11/run new file mode 100644 index 0000000..ae89e34 --- /dev/null +++ b/confs/scala-native-0.4.0-2.11/run @@ -0,0 +1 @@ +target/scala-2.11/scala-native-benchmarks-out diff --git a/confs/scala-native-0.4.0-2.12/build.properties b/confs/scala-native-0.4.0-2.12/build.properties new file mode 100644 index 0000000..4c69e16 --- /dev/null +++ b/confs/scala-native-0.4.0-2.12/build.properties @@ -0,0 +1 @@ +sbt.version = 1.4.1 diff --git a/confs/scala-native-0.4.0-2.12/build.sbt b/confs/scala-native-0.4.0-2.12/build.sbt new file mode 100644 index 0000000..fca72b4 --- /dev/null +++ b/confs/scala-native-0.4.0-2.12/build.sbt @@ -0,0 +1,8 @@ +scalaVersion := "2.12.13" +enablePlugins(ScalaNativePlugin) +import scala.scalanative.build +nativeConfig ~= { + _.withGC(build.GC.immix) + .withMode(build.Mode.releaseFull) + .withLTO(build.LTO.thin) +} diff --git a/confs/scala-native-0.4.0-2.12/compile b/confs/scala-native-0.4.0-2.12/compile new file mode 100644 index 0000000..2f3f09f --- /dev/null +++ b/confs/scala-native-0.4.0-2.12/compile @@ -0,0 +1 @@ +nativeLink diff --git a/confs/scala-native-0.4.0-2.12/plugins.sbt b/confs/scala-native-0.4.0-2.12/plugins.sbt new file mode 100644 index 0000000..2f4e38a --- /dev/null +++ b/confs/scala-native-0.4.0-2.12/plugins.sbt @@ -0,0 +1 @@ +addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.4.0") diff --git a/confs/scala-native-0.4.0-2.12/run b/confs/scala-native-0.4.0-2.12/run new file mode 100644 index 0000000..4c1cad0 --- /dev/null +++ b/confs/scala-native-0.4.0-2.12/run @@ -0,0 +1 @@ +target/scala-2.12/scala-native-benchmarks-out diff --git a/confs/scala-native-0.4.0-M2/build.propert b/confs/scala-native-0.4.0-M2/build.propert new file mode 100644 index 0000000..e69de29 diff --git a/confs/scala-native-0.4.0-M2/build.properties b/confs/scala-native-0.4.0-M2/build.properties new file mode 100644 index 0000000..0db266d --- /dev/null +++ b/confs/scala-native-0.4.0-M2/build.properties @@ -0,0 +1 @@ +sbt.version = 0.13.17 diff --git a/confs/baseline/build.sbt b/confs/scala-native-0.4.0-M2/build.sbt similarity index 80% rename from confs/baseline/build.sbt rename to confs/scala-native-0.4.0-M2/build.sbt index ae87f31..5df1bb4 100644 --- a/confs/baseline/build.sbt +++ b/confs/scala-native-0.4.0-M2/build.sbt @@ -2,5 +2,5 @@ scalaVersion := "2.11.12" enablePlugins(ScalaNativePlugin) nativeLinkStubs := true nativeGC := "immix" -nativeMode := "release" +nativeMode := "release-full" nativeLTO := "thin" diff --git a/confs/scala-native-0.4.0-M2/compile b/confs/scala-native-0.4.0-M2/compile new file mode 100644 index 0000000..2f3f09f --- /dev/null +++ b/confs/scala-native-0.4.0-M2/compile @@ -0,0 +1 @@ +nativeLink diff --git a/confs/scala-native-0.4.0-M2/plugins.sbt b/confs/scala-native-0.4.0-M2/plugins.sbt new file mode 100644 index 0000000..9663d2e --- /dev/null +++ b/confs/scala-native-0.4.0-M2/plugins.sbt @@ -0,0 +1 @@ +addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.4.0-M2") diff --git a/confs/scala-native-0.4.0-M2/run b/confs/scala-native-0.4.0-M2/run new file mode 100644 index 0000000..ae89e34 --- /dev/null +++ b/confs/scala-native-0.4.0-M2/run @@ -0,0 +1 @@ +target/scala-2.11/scala-native-benchmarks-out diff --git a/project/build.sbt b/project/build.sbt deleted file mode 100644 index 8b13789..0000000 --- a/project/build.sbt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/project/plugins.sbt b/project/plugins.sbt deleted file mode 100644 index 2a63bf0..0000000 --- a/project/plugins.sbt +++ /dev/null @@ -1 +0,0 @@ -addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.4.0-SNAPSHOT") diff --git a/scripts/compiletime-summary.py b/scripts/compiletime-summary.py new file mode 100644 index 0000000..295d457 --- /dev/null +++ b/scripts/compiletime-summary.py @@ -0,0 +1,31 @@ +from run import benchmarks, runs, configurations + +import numpy as np + +def config_data(bench, conf): + try: + with open('results/{}/{}/compile_time'.format(conf, bench)) as data: + return float(data.read()) + except IOError: + return 0 + +def compile_time(): + out = [] + for bench in benchmarks: + res = [] + for conf in configurations: + try: + res.append(np.percentile(config_data(bench, conf), 10)) + except IndexError: + res.append(0) + out.append(res) + return out + +if __name__ == '__main__': + leading = ['name'] + for conf in configurations: + leading.append(conf) + print(','.join(leading)) + for bench, res in zip(benchmarks, compile_time()): + print(','.join([bench] + list(map(str, res)))) + diff --git a/scripts/run.py b/scripts/run.py index 7a02658..f4de414 100755 --- a/scripts/run.py +++ b/scripts/run.py @@ -4,6 +4,7 @@ import errno import subprocess as subp import shutil as sh +import time def mkdir(path): try: @@ -35,10 +36,13 @@ def run(cmd): return subp.check_output(cmd) def compile(bench, compilecmd): + start = time.time_ns() cmd = [sbt, '-J-Xmx6G', 'clean'] cmd.append('set mainClass in Compile := Some("{}")'.format(bench)) cmd.append(compilecmd) - return run(cmd) + run(cmd) + end = time.time_ns() + return end - start sbt = where('sbt') @@ -62,10 +66,10 @@ def compile(bench, compilecmd): ] configurations = [ - # 'jvm', - # 'scala-native-0.3.8', - # 'baseline', - 'current' + 'scala-native-0.3.9', + 'scala-native-0.4.0-M2', + 'scala-native-0.4.0-2.11', + 'scala-native-0.4.0-2.12', ] if 'GRAALVM_HOME' in os.environ: @@ -75,7 +79,7 @@ def compile(bench, compilecmd): ] runs = 20 -batches = 4000 +batches = 2000 batch_size = 1 if __name__ == "__main__": @@ -83,32 +87,49 @@ def compile(bench, compilecmd): for bench in benchmarks: print('--- conf: {}, bench: {}'.format(conf, bench)) + resultsdir = os.path.join('results', conf, bench) + mkdir(resultsdir) + + bindir = os.path.join("binaries", conf, bench) + mkdir(bindir) + + binfile = os.path.join(bindir, bench) + input = slurp(os.path.join('input', bench)) output = slurp(os.path.join('output', bench)) compilecmd = slurp(os.path.join('confs', conf, 'compile')) runcmd = slurp(os.path.join('confs', conf, 'run')).replace('$BENCH', bench).replace('$HOME', os.environ['HOME']).split(' ') - if os.path.exists(os.path.join('confs', conf, 'build.sbt')): - sh.copyfile(os.path.join('confs', conf, 'build.sbt'), 'build.sbt') - else: - os.remove('build.sbt') + if not os.path.exists(binfile): + if os.path.exists(os.path.join('confs', conf, 'build.sbt')): + sh.copyfile(os.path.join('confs', conf, 'build.sbt'), 'build.sbt') + else: + os.remove('build.sbt') - if os.path.exists(os.path.join('confs', conf, 'plugins.sbt')): - sh.copyfile(os.path.join('confs', conf, 'plugins.sbt'), 'project/plugins.sbt') - else: - os.remove('project/plugins.sbt') + if os.path.exists(os.path.join('confs', conf, 'build.properties')): + sh.copyfile(os.path.join('confs', conf, 'build.properties'), 'project/build.properties') + else: + os.remove('project/build.properties') - compile(bench, compilecmd) - resultsdir = os.path.join('results', conf, bench) - mkdir(resultsdir) + if os.path.exists(os.path.join('confs', conf, 'plugins.sbt')): + sh.copyfile(os.path.join('confs', conf, 'plugins.sbt'), 'project/plugins.sbt') + else: + os.remove('project/plugins.sbt') + + compile_time = compile(bench, compilecmd) + + with open(os.path.join(resultsdir, "compile_time"), 'w') as f: + f.write(str(compile_time)) + + sh.move(runcmd[0], binfile) - for n in xrange(runs): + for n in range(runs): print('--- run {}/{}'.format(n, runs)) - cmd = [] - cmd.extend(runcmd) + cmd = [binfile] + cmd.extend(runcmd[1:]) cmd.extend([str(batches), str(batch_size), input, output]) out = run(cmd) - with open(os.path.join(resultsdir, str(n)), 'w+') as resultfile: + with open(os.path.join(resultsdir, str(n)), 'wb') as resultfile: resultfile.write(out) diff --git a/scripts/size-summary.py b/scripts/size-summary.py new file mode 100644 index 0000000..65783c9 --- /dev/null +++ b/scripts/size-summary.py @@ -0,0 +1,30 @@ +from run import benchmarks, runs, configurations +import os +import numpy as np + +def config_data(bench, conf): + try: + return len(open('binaries/{}/{}/{}'.format(conf, bench, bench), 'rb').read()) + except IOError: + return 0 + +def binary_size(): + out = [] + for bench in benchmarks: + res = [] + for conf in configurations: + try: + res.append(np.percentile(config_data(bench, conf), 10)) + except IndexError: + res.append(0) + out.append(res) + return out + +if __name__ == '__main__': + leading = ['name'] + for conf in configurations: + leading.append(conf) + print(','.join(leading)) + for bench, res in zip(benchmarks, binary_size()): + print(','.join([bench] + list(map(str, res)))) + diff --git a/scripts/summary.py b/scripts/summary.py index e505dbe..ede68c4 100644 --- a/scripts/summary.py +++ b/scripts/summary.py @@ -4,7 +4,7 @@ def config_data(bench, conf): out = [] - for run in xrange(runs): + for run in range(runs): try: points = [] with open('results/{}/{}/{}'.format(conf, bench, run)) as data: @@ -27,7 +27,7 @@ def peak_performance(): res = [] for conf in configurations: try: - res.append(np.percentile(config_data(bench, conf), 50)) + res.append(np.percentile(config_data(bench, conf), 10)) except IndexError: res.append(0) out.append(res) @@ -37,7 +37,7 @@ def peak_performance(): leading = ['name'] for conf in configurations: leading.append(conf) - print ','.join(leading) + print(','.join(leading)) for bench, res in zip(benchmarks, peak_performance()): - print ','.join([bench] + list(map(str, res))) + print(','.join([bench] + list(map(str, res))))