diff --git a/.gitignore b/.gitignore index 528a077..97d29fc 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,13 @@ target/ bin/.coursier bin/.scalafmt* results/ +reports/ *.iprof .idea +build.sbt +project/plugins.sbt +/scripts/.ipynb_checkpoints/ +/.bloop/ +/.bsp/ +/.metals/ +/.vscode/ diff --git a/confs/jvm/build.sbt b/confs/jvm/build.sbt index 9874e27..ef3680d 100644 --- a/confs/jvm/build.sbt +++ b/confs/jvm/build.sbt @@ -1 +1,2 @@ +name := "scala-native-benchmarks" scalaVersion := "2.11.12" diff --git a/confs/jvm/plugins.sbt b/confs/jvm/plugins.sbt deleted file mode 100644 index 8b13789..0000000 --- a/confs/jvm/plugins.sbt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/confs/native-image-pgo/build.sbt b/confs/native-image-pgo/build.sbt index 65e12d6..c622b0a 100644 --- a/confs/native-image-pgo/build.sbt +++ b/confs/native-image-pgo/build.sbt @@ -1,3 +1,4 @@ +name := "scala-native-benchmarks" scalaVersion := "2.11.12" val instrumentedRuns = settingKey[Int]("The number of instrumented runs.") diff --git a/confs/native-image/build.sbt b/confs/native-image/build.sbt index 065cf09..4073552 100644 --- a/confs/native-image/build.sbt +++ b/confs/native-image/build.sbt @@ -1,3 +1,4 @@ +name := "scala-native-benchmarks" scalaVersion := "2.11.12" val nativeImage = taskKey[Unit]("Builds a native image of a benchmark.") diff --git a/confs/scala-native-0.3.7/build.sbt b/confs/scala-native-0.3.7/build.sbt index b4a5690..702aa9d 100644 --- a/confs/scala-native-0.3.7/build.sbt +++ b/confs/scala-native-0.3.7/build.sbt @@ -1,3 +1,4 @@ +name := "scala-native-benchmarks" scalaVersion := "2.11.12" enablePlugins(ScalaNativePlugin) nativeLinkStubs := true diff --git a/confs/scala-native-0.3.8/build.sbt b/confs/scala-native-0.3.8/build.sbt index b4a5690..702aa9d 100644 --- a/confs/scala-native-0.3.8/build.sbt +++ b/confs/scala-native-0.3.8/build.sbt @@ -1,3 +1,4 @@ +name := "scala-native-benchmarks" scalaVersion := "2.11.12" enablePlugins(ScalaNativePlugin) nativeLinkStubs := true diff --git a/confs/scala-native-0.3.9/build.sbt b/confs/scala-native-0.3.9/build.sbt index b4a5690..702aa9d 100644 --- a/confs/scala-native-0.3.9/build.sbt +++ b/confs/scala-native-0.3.9/build.sbt @@ -1,3 +1,4 @@ +name := "scala-native-benchmarks" scalaVersion := "2.11.12" enablePlugins(ScalaNativePlugin) nativeLinkStubs := true diff --git a/confs/scala-native-0.3.9/run b/confs/scala-native-0.3.9/run index 8fedce3..ae89e34 100644 --- a/confs/scala-native-0.3.9/run +++ b/confs/scala-native-0.3.9/run @@ -1 +1 @@ -target/scala-2.11/benchmarks-out +target/scala-2.11/scala-native-benchmarks-out diff --git a/confs/scala-native-0.4.0-2.11/build.sbt b/confs/scala-native-0.4.0-2.11/build.sbt index 80c509f..a5ecbdd 100644 --- a/confs/scala-native-0.4.0-2.11/build.sbt +++ b/confs/scala-native-0.4.0-2.11/build.sbt @@ -1,8 +1,9 @@ +name := "scala-native-benchmarks" scalaVersion := "2.11.12" enablePlugins(ScalaNativePlugin) import scala.scalanative.build nativeConfig ~= { _.withGC(build.GC.immix) - .withMode(build.Mode.releaseFull) - .withLTO(build.LTO.thin) + .withMode(build.Mode.releaseFull) + .withLTO(build.LTO.thin) } diff --git a/confs/scala-native-0.4.0-2.11/run b/confs/scala-native-0.4.0-2.11/run index 8fedce3..ae89e34 100644 --- a/confs/scala-native-0.4.0-2.11/run +++ b/confs/scala-native-0.4.0-2.11/run @@ -1 +1 @@ -target/scala-2.11/benchmarks-out +target/scala-2.11/scala-native-benchmarks-out diff --git a/confs/scala-native-0.4.0-2.12/build.sbt b/confs/scala-native-0.4.0-2.12/build.sbt index fca72b4..6f30aa6 100644 --- a/confs/scala-native-0.4.0-2.12/build.sbt +++ b/confs/scala-native-0.4.0-2.12/build.sbt @@ -1,8 +1,9 @@ +name := "scala-native-benchmarks" scalaVersion := "2.12.13" enablePlugins(ScalaNativePlugin) import scala.scalanative.build nativeConfig ~= { _.withGC(build.GC.immix) - .withMode(build.Mode.releaseFull) - .withLTO(build.LTO.thin) + .withMode(build.Mode.releaseFull) + .withLTO(build.LTO.thin) } diff --git a/confs/scala-native-0.4.0-2.12/run b/confs/scala-native-0.4.0-2.12/run index f21e891..4c1cad0 100644 --- a/confs/scala-native-0.4.0-2.12/run +++ b/confs/scala-native-0.4.0-2.12/run @@ -1 +1 @@ -target/scala-2.12/benchmarks-out +target/scala-2.12/scala-native-benchmarks-out diff --git a/confs/scala-native-0.4.0-M2/build.sbt b/confs/scala-native-0.4.0-M2/build.sbt index 5df1bb4..ffde723 100644 --- a/confs/scala-native-0.4.0-M2/build.sbt +++ b/confs/scala-native-0.4.0-M2/build.sbt @@ -1,3 +1,4 @@ +name := "scala-native-benchmarks" scalaVersion := "2.11.12" enablePlugins(ScalaNativePlugin) nativeLinkStubs := true diff --git a/confs/scala-native-0.4.0-M2/run b/confs/scala-native-0.4.0-M2/run index 8fedce3..ae89e34 100644 --- a/confs/scala-native-0.4.0-M2/run +++ b/confs/scala-native-0.4.0-M2/run @@ -1 +1 @@ -target/scala-2.11/benchmarks-out +target/scala-2.11/scala-native-benchmarks-out diff --git a/confs/scala-native-0.4.1-SNAPSHOT-2.12/build.properties b/confs/scala-native-0.4.1-SNAPSHOT-2.12/build.properties new file mode 100644 index 0000000..d18a12e --- /dev/null +++ b/confs/scala-native-0.4.1-SNAPSHOT-2.12/build.properties @@ -0,0 +1 @@ +sbt.version = 1.4.7 diff --git a/confs/scala-native-0.4.1-SNAPSHOT-2.12/build.sbt b/confs/scala-native-0.4.1-SNAPSHOT-2.12/build.sbt new file mode 100644 index 0000000..6f30aa6 --- /dev/null +++ b/confs/scala-native-0.4.1-SNAPSHOT-2.12/build.sbt @@ -0,0 +1,9 @@ +name := "scala-native-benchmarks" +scalaVersion := "2.12.13" +enablePlugins(ScalaNativePlugin) +import scala.scalanative.build +nativeConfig ~= { + _.withGC(build.GC.immix) + .withMode(build.Mode.releaseFull) + .withLTO(build.LTO.thin) +} diff --git a/confs/scala-native-0.4.1-SNAPSHOT-2.12/compile b/confs/scala-native-0.4.1-SNAPSHOT-2.12/compile new file mode 100644 index 0000000..2f3f09f --- /dev/null +++ b/confs/scala-native-0.4.1-SNAPSHOT-2.12/compile @@ -0,0 +1 @@ +nativeLink diff --git a/confs/scala-native-0.4.1-SNAPSHOT-2.12/plugins.sbt b/confs/scala-native-0.4.1-SNAPSHOT-2.12/plugins.sbt new file mode 100644 index 0000000..05fa43c --- /dev/null +++ b/confs/scala-native-0.4.1-SNAPSHOT-2.12/plugins.sbt @@ -0,0 +1 @@ +addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.4.1-SNAPSHOT") diff --git a/confs/scala-native-0.4.1-SNAPSHOT-2.12/run b/confs/scala-native-0.4.1-SNAPSHOT-2.12/run new file mode 100644 index 0000000..4c1cad0 --- /dev/null +++ b/confs/scala-native-0.4.1-SNAPSHOT-2.12/run @@ -0,0 +1 @@ +target/scala-2.12/scala-native-benchmarks-out diff --git a/scripts/.ipynb_checkpoints/notebook-checkpoint.ipynb b/scripts/.ipynb_checkpoints/notebook-checkpoint.ipynb deleted file mode 100644 index cd185bf..0000000 --- a/scripts/.ipynb_checkpoints/notebook-checkpoint.ipynb +++ /dev/null @@ -1,195 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "ename": "ModuleNotFoundError", - "evalue": "No module named 'matplotlib'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mget_ipython\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmagic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'matplotlib inline'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m/usr/lib/python3/dist-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36mmagic\u001b[0;34m(self, arg_s)\u001b[0m\n\u001b[1;32m 2156\u001b[0m \u001b[0mmagic_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmagic_arg_s\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marg_s\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpartition\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m' '\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2157\u001b[0m \u001b[0mmagic_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmagic_name\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlstrip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprefilter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mESC_MAGIC\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2158\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_line_magic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmagic_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmagic_arg_s\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2159\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2160\u001b[0m \u001b[0;31m#-------------------------------------------------------------------------\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/lib/python3/dist-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36mrun_line_magic\u001b[0;34m(self, magic_name, line)\u001b[0m\n\u001b[1;32m 2077\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'local_ns'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getframe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstack_depth\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf_locals\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2078\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuiltin_trap\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2079\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2080\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2081\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m\u001b[0m in \u001b[0;36mmatplotlib\u001b[0;34m(self, line)\u001b[0m\n", - "\u001b[0;32m/usr/lib/python3/dist-packages/IPython/core/magic.py\u001b[0m in \u001b[0;36m\u001b[0;34m(f, *a, **k)\u001b[0m\n\u001b[1;32m 186\u001b[0m \u001b[0;31m# but it's overkill for just that one bit of state.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 187\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mmagic_deco\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 188\u001b[0;31m \u001b[0mcall\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 189\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 190\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcallable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/lib/python3/dist-packages/IPython/core/magics/pylab.py\u001b[0m in \u001b[0;36mmatplotlib\u001b[0;34m(self, line)\u001b[0m\n\u001b[1;32m 98\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Available matplotlib backends: %s\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mbackends_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 99\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 100\u001b[0;31m \u001b[0mgui\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbackend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshell\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable_matplotlib\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgui\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 101\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_show_matplotlib_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgui\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbackend\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 102\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/lib/python3/dist-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36menable_matplotlib\u001b[0;34m(self, gui)\u001b[0m\n\u001b[1;32m 2933\u001b[0m \"\"\"\n\u001b[1;32m 2934\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mIPython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcore\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpylabtools\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mpt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2935\u001b[0;31m \u001b[0mgui\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbackend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind_gui_and_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgui\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpylab_gui_select\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2936\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2937\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mgui\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m'inline'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/lib/python3/dist-packages/IPython/core/pylabtools.py\u001b[0m in \u001b[0;36mfind_gui_and_backend\u001b[0;34m(gui, gui_select)\u001b[0m\n\u001b[1;32m 257\u001b[0m \"\"\"\n\u001b[1;32m 258\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 259\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 260\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 261\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mgui\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mgui\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m'auto'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'matplotlib'" - ] - } - ], - "source": [ - "%matplotlib inline" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "import numpy as np\n", - "import matplotlib\n", - "import matplotlib.pyplot as plt\n", - "matplotlib.rcParams['figure.figsize'] = (20.0, 15.0)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "rawpoints = []\n", - "with open(\"results/bounce.BounceBenchmark/1\") as data:\n", - " for line in data.readlines():\n", - " rawpoints.append(float(line)/1000000)\n", - "raw = np.array(rawpoints)\n", - "filtered = []\n", - "pmax = np.percentile(raw[500:], 99)\n", - "for point in rawpoints:\n", - " if point < pmax:\n", - " filtered.append(point)\n", - "y = np.array(filtered)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "len(y)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false, - "scrolled": false - }, - "outputs": [], - "source": [ - "#plt.axis([0, len(y), 0.5, 5])\n", - "plt.plot(y)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false, - "scrolled": false - }, - "outputs": [], - "source": [ - "plt.hist(y[:200], bins = 10)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "percentiles = np.arange(0, 100)\n", - "percvalue = np.array([np.percentile(y[:500], perc) for perc in percentiles])\n", - "plt.plot(percentiles, percvalue)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "np.percentile(y, 5)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [], - "source": [ - "np.percentile(y, 50)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "np.percentile(y, 95)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "np.mean(y)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "y.std()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/scripts/interactive.py b/scripts/interactive.py new file mode 100755 index 0000000..cfdd798 --- /dev/null +++ b/scripts/interactive.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python + +# import for interactive mode +# noinspection PyUnresolvedReferences +from shared.benchmarks import * +# noinspection PyUnresolvedReferences +from shared.configurations import * +# noinspection PyUnresolvedReferences +from shared.comparison import * +# noinspection PyUnresolvedReferences +from shared.cmdline import stable,latest + +if __name__ == '__main__': + from IPython import embed + embed() \ No newline at end of file diff --git a/scripts/notebook.ipynb b/scripts/notebook.ipynb index 75c9347..472285a 100644 --- a/scripts/notebook.ipynb +++ b/scripts/notebook.ipynb @@ -2,31 +2,11 @@ "cells": [ { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": { - "collapsed": false, "scrolled": true }, - "outputs": [ - { - "ename": "ModuleNotFoundError", - "evalue": "No module named 'matplotlib'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mget_ipython\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmagic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'matplotlib inline'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m/usr/lib/python3/dist-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36mmagic\u001b[0;34m(self, arg_s)\u001b[0m\n\u001b[1;32m 2156\u001b[0m \u001b[0mmagic_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmagic_arg_s\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marg_s\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpartition\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m' '\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2157\u001b[0m \u001b[0mmagic_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmagic_name\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlstrip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprefilter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mESC_MAGIC\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2158\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_line_magic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmagic_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmagic_arg_s\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2159\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2160\u001b[0m \u001b[0;31m#-------------------------------------------------------------------------\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/lib/python3/dist-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36mrun_line_magic\u001b[0;34m(self, magic_name, line)\u001b[0m\n\u001b[1;32m 2077\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'local_ns'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getframe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstack_depth\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf_locals\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2078\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuiltin_trap\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2079\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2080\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2081\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m\u001b[0m in \u001b[0;36mmatplotlib\u001b[0;34m(self, line)\u001b[0m\n", - "\u001b[0;32m/usr/lib/python3/dist-packages/IPython/core/magic.py\u001b[0m in \u001b[0;36m\u001b[0;34m(f, *a, **k)\u001b[0m\n\u001b[1;32m 186\u001b[0m \u001b[0;31m# but it's overkill for just that one bit of state.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 187\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mmagic_deco\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 188\u001b[0;31m \u001b[0mcall\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 189\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 190\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcallable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/lib/python3/dist-packages/IPython/core/magics/pylab.py\u001b[0m in \u001b[0;36mmatplotlib\u001b[0;34m(self, line)\u001b[0m\n\u001b[1;32m 98\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Available matplotlib backends: %s\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mbackends_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 99\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 100\u001b[0;31m \u001b[0mgui\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbackend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshell\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable_matplotlib\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgui\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 101\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_show_matplotlib_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgui\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbackend\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 102\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/lib/python3/dist-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36menable_matplotlib\u001b[0;34m(self, gui)\u001b[0m\n\u001b[1;32m 2933\u001b[0m \"\"\"\n\u001b[1;32m 2934\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mIPython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcore\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpylabtools\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mpt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2935\u001b[0;31m \u001b[0mgui\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbackend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind_gui_and_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgui\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpylab_gui_select\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2936\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2937\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mgui\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m'inline'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/lib/python3/dist-packages/IPython/core/pylabtools.py\u001b[0m in \u001b[0;36mfind_gui_and_backend\u001b[0;34m(gui, gui_select)\u001b[0m\n\u001b[1;32m 257\u001b[0m \"\"\"\n\u001b[1;32m 258\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 259\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 260\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 261\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mgui\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mgui\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m'auto'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'matplotlib'" - ] - } - ], + "outputs": [], "source": [ "%matplotlib inline" ] @@ -34,9 +14,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", @@ -48,13 +26,14 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "rawpoints = []\n", - "with open(\"results/bounce.BounceBenchmark/1\") as data:\n", + "conf = \"scala-native-0.4.0-2.12\"\n", + "bench = \"bounce.BounceBenchmark\"\n", + "run = 1\n", + "with open(\"../results/{}/{}/{}\".format(conf, bench, run)) as data:\n", " for line in data.readlines():\n", " rawpoints.append(float(line)/1000000)\n", "raw = np.array(rawpoints)\n", @@ -69,9 +48,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "len(y)" @@ -81,7 +58,6 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": false, "scrolled": false }, "outputs": [], @@ -94,7 +70,6 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": false, "scrolled": false }, "outputs": [], @@ -105,9 +80,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "percentiles = np.arange(0, 100)\n", @@ -118,9 +91,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "np.percentile(y, 5)" @@ -130,7 +101,6 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": false, "scrolled": true }, "outputs": [], @@ -141,9 +111,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "np.percentile(y, 95)" @@ -152,9 +120,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "np.mean(y)" @@ -163,13 +129,25 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "y.std()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -188,7 +166,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.3" + "version": "3.8.5" } }, "nbformat": 4, diff --git a/scripts/run.py b/scripts/run.py index 3dc2eda..bd30754 100755 --- a/scripts/run.py +++ b/scripts/run.py @@ -1,136 +1,20 @@ #!/usr/bin/env python -import sys -import os -import errno -import subprocess as subp -import shutil as sh -import time +import argparse -def mkdir(path): - try: - os.makedirs(path) - except OSError as exc: # Python >2.5 - if exc.errno == errno.EEXIST and os.path.isdir(path): - pass - else: - raise - -def slurp(path): - with open(path) as f: - return f.read().strip() - -def where(cmd): - if os.path.isfile(cmd): - return cmd - else: - paths = os.environ['PATH'].split(os.pathsep) - for p in paths: - f = os.path.join(p, cmd) - if os.path.isfile(f): - return f - else: - return None - -def run(cmd): - print(">>> " + str(cmd)) - return subp.check_output(cmd) - -def compile(bench, compilecmd): - start = time.time_ns() - cmd = [sbt, '-J-Xmx6G', 'clean'] - cmd.append('set mainClass in Compile := Some("{}")'.format(bench)) - cmd.append(compilecmd) - run(cmd) - end = time.time_ns() - return end - start - -sbt = where('sbt') - -benchmarks = [ - 'bounce.BounceBenchmark', - 'list.ListBenchmark', - 'queens.QueensBenchmark', - 'richards.RichardsBenchmark', - 'permute.PermuteBenchmark', - 'deltablue.DeltaBlueBenchmark', - 'tracer.TracerBenchmark', - 'json.JsonBenchmark', - 'sudoku.SudokuBenchmark', - 'brainfuck.BrainfuckBenchmark', - 'cd.CDBenchmark', - 'kmeans.KmeansBenchmark', - 'nbody.NbodyBenchmark', - 'rsc.RscBenchmark', - 'gcbench.GCBenchBenchmark', - 'mandelbrot.MandelbrotBenchmark', - 'histogram.Histogram', -] - -configurations = [ - 'scala-native-0.3.9', - 'scala-native-0.4.0-M2', - 'scala-native-0.4.0-2.11', - 'scala-native-0.4.0-2.12', -] - -if 'GRAALVM_HOME' in os.environ: - configurations += [ - 'native-image', - 'native-image-pgo', - ] - -runs = 20 -batches = 2000 -batch_size = 1 +from shared.cmdline import expand_all +from shared.benchmarks import benchmarks +from shared.configurations import Configuration, default_runs, default_batches if __name__ == "__main__": - for conf in configurations: - for bench in benchmarks: - print('--- conf: {}, bench: {}'.format(conf, bench)) - - resultsdir = os.path.join('results', conf, bench) - mkdir(resultsdir) - - bindir = os.path.join("binaries", conf, bench) - mkdir(bindir) - - binfile = os.path.join(bindir, bench) - - input = slurp(os.path.join('input', bench)) - output = slurp(os.path.join('output', bench)) - compilecmd = slurp(os.path.join('confs', conf, 'compile')) - runcmd = slurp(os.path.join('confs', conf, 'run')).replace('$BENCH', bench).replace('$HOME', os.environ['HOME']).split(' ') - - if not os.path.exists(binfile): - if os.path.exists(os.path.join('confs', conf, 'build.sbt')): - sh.copyfile(os.path.join('confs', conf, 'build.sbt'), 'build.sbt') - else: - os.remove('build.sbt') - - if os.path.exists(os.path.join('confs', conf, 'build.properties')): - sh.copyfile(os.path.join('confs', conf, 'build.properties'), 'project/build.properties') - else: - os.remove('project/build.properties') - - if os.path.exists(os.path.join('confs', conf, 'plugins.sbt')): - sh.copyfile(os.path.join('confs', conf, 'plugins.sbt'), 'project/plugins.sbt') - else: - os.remove('project/plugins.sbt') - - compile_time = compile(bench, compilecmd) - - with open(os.path.join(resultsdir, "compile_time"), 'w') as f: - f.write(str(compile_time)) - - sh.move(runcmd[0], binfile) - for n in range(runs): - print('--- run {}/{}'.format(n, runs)) + parser = argparse.ArgumentParser() + parser.add_argument("--runs", help="number of runs", type=int, default=default_runs) + parser.add_argument("--batches", help="number of batches per run", type=int, default=default_batches) + parser.add_argument("configurations", nargs='*', default=[None]) + args = parser.parse_args() - cmd = [binfile] - cmd.extend(runcmd[1:]) - cmd.extend([str(batches), str(batch_size), input, output]) - out = run(cmd) - with open(os.path.join(resultsdir, str(n)), 'wb') as resultfile: - resultfile.write(out) + configuration_names = expand_all(args.configurations) + print("configurations:", configuration_names) + for conf_name in configuration_names: + Configuration(conf_name, batches=args.batches, runs=args.runs).run_benchmarks(benchmarks) diff --git a/confs/scala-native-0.4.0-M2/build.propert b/scripts/shared/__init__.py similarity index 100% rename from confs/scala-native-0.4.0-M2/build.propert rename to scripts/shared/__init__.py diff --git a/scripts/shared/benchmarks.py b/scripts/shared/benchmarks.py new file mode 100644 index 0000000..97635cb --- /dev/null +++ b/scripts/shared/benchmarks.py @@ -0,0 +1,77 @@ +import os + +from shared.file_utils import sbt, run, slurp, mkdir +import time +import subprocess +import sys + +benchmarks = [ + 'bounce.BounceBenchmark', + 'list.ListBenchmark', + 'queens.QueensBenchmark', + 'richards.RichardsBenchmark', + 'permute.PermuteBenchmark', + 'deltablue.DeltaBlueBenchmark', + 'tracer.TracerBenchmark', + 'json.JsonBenchmark', + 'sudoku.SudokuBenchmark', + 'brainfuck.BrainfuckBenchmark', + 'cd.CDBenchmark', + 'kmeans.KmeansBenchmark', + 'nbody.NbodyBenchmark', + 'rsc.RscBenchmark', + 'gcbench.GCBenchBenchmark', + 'mandelbrot.MandelbrotBenchmark', + 'histogram.Histogram' +] + + +class Benchmark: + + def __init__(self, name): + self.name = name + self.short_name = name.split(".")[0] + + def compile(self, conf): + cmd = [sbt, '-J-Xmx6G', 'clean', + 'set mainClass in Compile := Some("{}")'.format(self.name), + conf.compile_cmd()] + start = time.time_ns() + subprocess.run(cmd, stdout=sys.stdout, stderr=subprocess.STDOUT) + end = time.time_ns() + return end - start + + def run(self, conf): + run_cmd = conf.run_cmd(self) + input = slurp(os.path.join('input', self.name)) + output = slurp(os.path.join('output', self.name)) + cmd = [] + cmd.extend(run_cmd) + cmd.extend([str(conf.batches), str(conf.batch_size), input, output]) + return run(cmd) + + def ensure_results_dir(self, conf): + dir = os.path.join(conf.ensure_results_dir(), self.name) + mkdir(dir) + return dir + + def results_dir(self, conf): + return os.path.join(conf.results_dir, self.name) + + def __eq__(self, other): + if isinstance(other, Benchmark): + return self.name == other.name + else: + return False + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash(self.name) + + def __str__(self): + return self.name + + def __repr__(self): + return 'Benchmark(\'{}\')'.format(self.name) diff --git a/scripts/shared/cmdline.py b/scripts/shared/cmdline.py new file mode 100644 index 0000000..9a0494c --- /dev/null +++ b/scripts/shared/cmdline.py @@ -0,0 +1,22 @@ +stable = 'scala-native-0.4.0-2.12' +latest = 'scala-native-0.4.1-SNAPSHOT-2.12' + +def expand_all(arg): + confs = [] + for choice in arg: + expanded = expand_wild_cards(choice) + if expanded is None: + confs = [stable, latest] + else: + confs += [expanded] + return confs + +def expand_wild_cards(arg): + if arg is None: + return arg + elif arg.startswith("latest"): + return latest + arg[len("latest"):] + elif arg.startswith("stable"): + return stable + arg[len("stable"):] + else: + return arg \ No newline at end of file diff --git a/scripts/shared/comparison.py b/scripts/shared/comparison.py new file mode 100644 index 0000000..036148a --- /dev/null +++ b/scripts/shared/comparison.py @@ -0,0 +1,68 @@ +import numpy as np +from shared.configurations import Configuration +from shared.parser import config_data +from shared.reports import Report +from shared.misc_utils import dict_write_arr, dict_get_arr + +default_warmup = 500 + +class Comparison: + def __init__(self, confs, warmup=default_warmup): + assert len(confs) > 0 + + configurations = [] + for item in confs: + if isinstance(item, str): + configurations.append(Configuration.from_results(item)) + else: + configurations.append(item) + self.configurations = configurations + + common_benchmarks = configurations[0].finished_benchmarks() + for other_conf in configurations[1:]: + present = set(other_conf.finished_benchmarks()) + common_benchmarks = list(filter(lambda b: b in present, common_benchmarks)) + self.common_benchmarks = common_benchmarks + self.warmup = warmup + + def percentile(self, p): + out = [] + for bench in self.common_benchmarks: + out.append(self.percentile_bench(bench, p)) + return out + + def percentile_bench(self, bench, p): + res = [] + for conf in self.configurations: + try: + res.append(np.percentile(config_data(bench, conf, self.warmup), p)) + except (IndexError, FileNotFoundError): + res.append(0) + return res + + def csv_file(self, data, path): + with open(path, 'w+') as resultfile: + leading = ['name'] + for conf in self.configurations: + leading.append(conf.full_name) + resultfile.write(','.join(leading) + '\n') + for bench, res in zip(self.common_benchmarks, data): + resultfile.write(','.join([str(bench)] + list(map(str, res))) + '\n') + + def simple_report(self, comment=None): + report = Report(self, comment=comment) + report.generate() + return report + + @classmethod + def from_dict(cls, kv): + warmup = int(kv.get('warmup', default_warmup)) + confs = map(Configuration.from_dict, dict_get_arr(kv, 'confs')) + return cls(confs, warmup) + + def to_dict(self): + kv = {'warmup': self.warmup} + conf_kvs = map(Configuration.to_dict, self.configurations) + dict_write_arr(kv, 'confs', conf_kvs) + + return kv diff --git a/scripts/shared/configurations.py b/scripts/shared/configurations.py new file mode 100644 index 0000000..361c46c --- /dev/null +++ b/scripts/shared/configurations.py @@ -0,0 +1,179 @@ +import os +import shutil as sh + +from shared.benchmarks import Benchmark +from shared.file_utils import slurp, mkdir, dict_from_file, dict_to_file + +base_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) +confs_path = base_path + '/confs' +binaries_path = base_path + '/binaries' +results_path = base_path + '/results' + +default_batches = 2000 +default_runs = 20 + +class Configuration: + + def __init__(self, name, batches=default_batches, runs=default_runs): + self.name = name + self.conf_dir = os.path.join(confs_path, self.name) + self.bin_dir = os.path.join(binaries_path, self.name) + self.batches = batches + self.batch_size = 1 + self.runs = runs + self.isJVM = self.name.lower().startswith("jvm") + + suffix = "" + if runs != default_runs: + suffix += "-r" + str(runs) + if batches != default_batches: + suffix += "-b" + str(batches) + + self.full_name = self.name + suffix + self.results_dir = os.path.join(results_path, self.full_name) + self.settings_file = os.path.join(self.results_dir, 'settings.properties') + + @classmethod + def from_results(cls, full_name): + return cls.from_results_dir(os.path.join(results_path, full_name)) + + @classmethod + def from_results_dir(cls, full_path): + settings_file = os.path.join(full_path, 'settings.properties') + try: + kv = dict_from_file(settings_file) + return cls.from_dict(kv) + except FileNotFoundError: + kv = {} + kv['name'] = os.path.basename(full_path) + return cls.from_dict(kv) + + @classmethod + def from_dict(cls, kv): + batches = int(kv.get('batches', default_batches)) + runs = int(kv.get('runs', default_runs)) + name = kv.get('name') + return cls(name, batches=batches, runs=runs) + + def make_active(self): + required = True + optional = False + + files = [ + ('build.sbt', 'build.sbt', required), + ('plugins.sbt', 'project/plugins.sbt', optional), + ('build.properties', 'project/build.properties', optional) + ] + for filename, destName, isRequired in files: + src_path = os.path.join(self.conf_dir, filename) + if os.path.exists(src_path): + sh.copyfile(src_path, destName) + elif isRequired: + raise Exception("Required file {} missing for config {}".format(filename, self.name)) + elif os.path.exists(destName): + os.remove(destName) + + + def run_benchmarks(self, benchmarks): + + for item in benchmarks: + if isinstance(item, str): + bench = Benchmark(item) + else: + bench = item + print('--- conf: {}, bench: {}'.format(self.name, bench)) + results_dir = bench.ensure_results_dir(self) + if self.isJVM or not os.path.exists(self.__bin_file(bench)): + self.make_active() + compile_time = bench.compile(self) + with open(os.path.join(results_dir, "compile_time"), 'w') as f: + f.write(str(compile_time)) + + if not self.isJVM: + if not os.path.exists(self.bin_dir): + mkdir(self.bin_dir) + bin_file = self.__bin_file(bench) + sh.move(self.__native_binary_path(), bin_file) + + runs = self.runs + for n in range(runs): + print('--- run {}/{}'.format(n, runs)) + out = bench.run(self) + with open(os.path.join(results_dir, str(n)), 'wb') as resultfile: + resultfile.write(out) + + def __bin_file(self, bench): + assert not self.isJVM + return os.path.join(self.bin_dir, bench.name) + + def __native_binary_path(self): + assert not self.isJVM + return slurp(os.path.join(self.conf_dir, 'run')).split(' ')[0] + + def run_cmd(self, bench): + cmd = slurp(os.path.join(self.conf_dir, 'run')) \ + .replace('$BENCH', bench.name) \ + .replace('$HOME', os.environ['HOME']) \ + .split(' ') + + if self.isJVM: + return cmd + else: + return [ self.__bin_file(bench) ] + cmd[1:] + + def compile_cmd(self): + return slurp(os.path.join(self.conf_dir, 'compile')) + + def to_dict(self): + return {'name': self.name, 'batches': self.batches, 'runs': self.runs} + + def write_settings(self): + settings_file = self.settings_file + kv = self.to_dict() + dict_to_file(settings_file, kv) + + def ensure_results_dir(self): + results_dir = self.results_dir + mkdir(results_dir) + self.write_settings() + return results_dir + + def finished_benchmarks(self): + benchmarks = [] + results_dir = self.results_dir + all_subdirs = next(os.walk(results_dir), [results_dir, []])[1] + for subdir in all_subdirs: + all_runs_present = True + for run in range(self.runs): + if not os.path.isfile(os.path.join(results_dir, subdir, str(run))): + all_runs_present = False + break + if all_runs_present: + benchmarks.append(Benchmark(subdir)) + + return benchmarks + + def benchmark_result_files(self, benchmark): + benchmark_dir = benchmark.results_dir(self) + for run in range(self.runs): + yield os.path.join(benchmark_dir, str(run)) + + def __str__(self): + s = '{}('.format(self.name) + s += 'runs={}'.format(str(self.runs)) + if self.runs == default_runs: + s += '[default]' + s += ', batches={}'.format(str(self.batches)) + if self.batches == default_batches: + s += '[default]' + s += ') = {}'.format(self.full_name) + return s + + +configs = next(os.walk(confs_path))[1] + +if 'GRAALVM_HOME' in os.environ: + configs += [ + 'native-image', + 'native-image-pgo', + ] diff --git a/scripts/shared/file_utils.py b/scripts/shared/file_utils.py new file mode 100644 index 0000000..e4146f0 --- /dev/null +++ b/scripts/shared/file_utils.py @@ -0,0 +1,58 @@ +import os +import errno +import subprocess as subp + + +def mkdir(path): + try: + os.makedirs(path) + except OSError as exc: # Python >2.5 + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + + +def touch(path): + open(path, 'w+').close() + +def slurp(path): + with open(path) as f: + return f.read().strip() + + +def where(cmd): + if os.path.isfile(cmd): + return cmd + else: + paths = os.environ['PATH'].split(os.pathsep) + for p in paths: + f = os.path.join(p, cmd) + if os.path.isfile(f): + return f + else: + return None + + +def run(cmd): + print(">>> " + str(cmd)) + return subp.check_output(cmd) + + +def dict_from_file(settings_file): + kv = {} + with open(settings_file) as settings: + for line in settings.readlines(): + key, raw_value = line.split('=') + value = raw_value.strip() + kv[key] = value + return kv + + +def dict_to_file(settings_file, kv): + with open(settings_file, 'w+') as settings: + for k, v in kv.items(): + settings.write('{}={}\n'.format(k, v)) + + +sbt = where('sbt') diff --git a/scripts/shared/misc_utils.py b/scripts/shared/misc_utils.py new file mode 100644 index 0000000..3a95ab2 --- /dev/null +++ b/scripts/shared/misc_utils.py @@ -0,0 +1,20 @@ +def dict_write_arr(kv, arr_key, conf_kvs): + for index, inner_kv in enumerate(conf_kvs): + for k, v in inner_kv.items(): + kv['{}.{}.{}'.format(arr_key, index, k)] = v + + +def dict_get_arr(kv, arr_key): + arr = [] + arr_kv = {} + max_index = -1 + for k, v in kv.items(): + if k.startswith(arr_key + '.'): + _, raw_index, key = k.split('.') + index = int(raw_index) + max_index = max(index, max_index) + arr_kv.get(index, {})[key] = v + for index in range(max_index + 1): + arr.append(arr_kv.get(index)) + + return arr \ No newline at end of file diff --git a/scripts/shared/parser.py b/scripts/shared/parser.py new file mode 100644 index 0000000..e72e9f0 --- /dev/null +++ b/scripts/shared/parser.py @@ -0,0 +1,19 @@ +import numpy as np + + +def config_data(bench, conf, warmup): + out = [] + for result_file in conf.benchmark_result_files(bench): + try: + raw_points = [] + with open(result_file) as data: + for line in data.readlines(): + try: + # in ms + raw_points.append(float(line) / 1000000) + except Exception as e: + print(e) + out += raw_points[warmup:] + except IOError: + pass + return np.array(out) \ No newline at end of file diff --git a/scripts/shared/reports.py b/scripts/shared/reports.py new file mode 100644 index 0000000..056dcbb --- /dev/null +++ b/scripts/shared/reports.py @@ -0,0 +1,81 @@ +import os +from datetime import datetime + +from shared.file_utils import mkdir, dict_to_file, dict_from_file, touch + +date_format = '%Y%m%d_%H%M%S' + +reports_path = 'reports' + + +class Report: + def __init__(self, comparison, date=None, comment=None): + self.comparison = comparison + if comment is None: + comment = '_vs_'.join(map(lambda c: c.full_name, comparison.configurations)).replace(os.sep, "_") + self.comment = comment + if date is None: + date = datetime.now() + self.date = date + self.full_name = 'summary_' + datetime.strftime(self.date, date_format) + '_' + comment + self.results_dir = os.path.join(reports_path, self.full_name) + self.settings_file = os.path.join(self.results_dir, 'settings.properties') + self.done_file = os.path.join(self.results_dir, '.done') + + def generate(self): + if self.is_done(): + copy = Report(self.comparison, date=None, comment=self.comment) + copy.generate() + return copy + else: + c = self.comparison + results_dir = self.ensure_dir() + for p in [50, 90, 99]: + path = os.path.join(results_dir, 'percentile{}.csv'.format(str(p))) + c.csv_file(c.percentile(p), path) + self.mark_done() + return self + + def mark_done(self): + touch(self.done_file) + + def is_done(self): + return os.path.isfile(self.done_file) + + def write_settings(self): + settings_file = self.settings_file + kv = self.to_dict() + dict_to_file(settings_file, kv) + + @classmethod + def from_full_name(cls, full_name): + return cls.from_dir(os.path.join(reports_path, full_name)) + + @classmethod + def from_dir(cls, full_path): + settings_file = os.path.join(full_path, 'settings.properties') + kv = dict_from_file(settings_file) + return cls.from_dict(kv) + + def ensure_dir(self): + results_dir = self.results_dir + mkdir(results_dir) + self.write_settings() + return results_dir + + @classmethod + def from_dict(cls, kv): + comment = kv.get('comment') + raw_date = kv.get('date') + if raw_date is None: + date = None + else: + date = datetime.strptime(raw_date, date_format) + from comparison import Comparison + return cls(Comparison.from_dict(kv), date, comment) + + def to_dict(self): + kv = self.comparison.to_dict() + kv['comment'] = self.comment + kv['date'] = datetime.strftime(self.date, date_format) + return kv diff --git a/scripts/size-summary.py b/scripts/size-summary.py index 65783c9..161dd94 100644 --- a/scripts/size-summary.py +++ b/scripts/size-summary.py @@ -1,10 +1,11 @@ -from run import benchmarks, runs, configurations +from shared.benchmarks import benchmarks +from shared.configurations import configs, default_runs import os import numpy as np def config_data(bench, conf): try: - return len(open('binaries/{}/{}/{}'.format(conf, bench, bench), 'rb').read()) + return len(open('binaries/{}/{}'.format(conf, bench), 'rb').read()) except IOError: return 0 @@ -12,7 +13,7 @@ def binary_size(): out = [] for bench in benchmarks: res = [] - for conf in configurations: + for conf in configs: try: res.append(np.percentile(config_data(bench, conf), 10)) except IndexError: @@ -22,7 +23,7 @@ def binary_size(): if __name__ == '__main__': leading = ['name'] - for conf in configurations: + for conf in configs: leading.append(conf) print(','.join(leading)) for bench, res in zip(benchmarks, binary_size()): diff --git a/scripts/summary.py b/scripts/summary.py old mode 100644 new mode 100755 index ede68c4..2cf8a08 --- a/scripts/summary.py +++ b/scripts/summary.py @@ -1,43 +1,16 @@ -from run import benchmarks, runs, configurations +#!/usr/bin/env python +import argparse -import numpy as np - -def config_data(bench, conf): - out = [] - for run in range(runs): - try: - points = [] - with open('results/{}/{}/{}'.format(conf, bench, run)) as data: - for line in data.readlines(): - points.append(float(line)) - # take only last 2000 to account for startup - points = points[-2000:] - # filter out 1% worst measurements as outliers - pmax = np.percentile(points, 99) - for point in points: - if point <= pmax: - out.append(point) - except IOError: - pass - return np.array(out) - -def peak_performance(): - out = [] - for bench in benchmarks: - res = [] - for conf in configurations: - try: - res.append(np.percentile(config_data(bench, conf), 10)) - except IndexError: - res.append(0) - out.append(res) - return out +from shared.cmdline import expand_all +from shared.comparison import Comparison, default_warmup if __name__ == '__main__': - leading = ['name'] - for conf in configurations: - leading.append(conf) - print(','.join(leading)) - for bench, res in zip(benchmarks, peak_performance()): - print(','.join([bench] + list(map(str, res)))) + parser = argparse.ArgumentParser() + parser.add_argument("--comment", help="comment at the suffix of the report name", default=None) + parser.add_argument("--warmup", help="number of iterations to skip before calculating percentiles", type=int, default=default_warmup) + parser.add_argument("configurations", nargs='*', default=[None]) + args = parser.parse_args() + configuration_names = expand_all(args.configurations) + comparison = Comparison(configuration_names, warmup=args.warmup) + comparison.simple_report(comment=args.comment)