scala-native · valdisxp1 · Sep 14, 2018 · Sep 21, 2018 · Sep 21, 2018 · Sep 22, 2018
diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,9 @@
 bin/.coursier
 bin/.scalafmt*
 results/
+reports/
 *.iprof
-.idea
+.idea
+*.pid
+jobs.sh
+logs/
diff --git a/README.md b/README.md
@@ -13,8 +13,89 @@ as-is and summarized in a separate post-processing step.
 python scripts/run.py
 ```
 
-## Viewing result summary
+## Creating result summary
 
 ```
 python scripts/summary.py
 ```
+
+The reports can be viewed in the `reports` directory.
+
+## Advanced use
+
+### Comparing specific versions
+
+You can run just the configurations you are interested in
+```bash
+scripts/run.py stable latest
+```
+
+Compare the lastest `stable` relesea vs `latest` snapshot
+```bash
+REPORT=$(scripts/summary.py stable latest)
+```
+
+### Specifying number of runs, batches, tests run in parallel
+```bash
+scripts/run.py --par 4 --runs 50 --batches 3000 stable
+# 50 runs (4 in parallel) each with 3000 batches for the stable release.
+```
+
+These settings will impact accuracy, this is why the names of the results folders will include the settings, in this case `scala-native-0.3.8-r50-p40-b3000`.
+Note that you can also use `stable-r50-p40-b3000` when using the `summary.py`.
+
+### Comparing an experimental feature with latest from master
+1. specify a suffix to identify it
+```bash
+NAME=PR9001-adding-a-kitchen-sink
+SHA1=adcd124eee
+```
+2. run the benchmark and get the summary report
+
+It will automatically compile the project in `../scala-native` if you provide the SHA1 or reference.
+```bash
+SHA1=adcd124eee
+scripts/run.py latest@master
+scripts/run.py --suffix "$NAME" latest@"$SHA1" &&
+REPORT=$(scripts/summary.py --comment "$NAME" latest latest@"$SHA1"_"$NAME")
+```
+
+Alternatively you can build the scala-native project from your branch manually and run
+```bash
+scripts/run.py --suffix "$NAME" latest &&
+REPORT=$(scripts/summary.py --comment "$NAME" latest latest_"$NAME")
+```
+
+## Persisting reports
+The following commands assume that you have a git repository checked out at `gh-pages` under `../scala-native-benchmark-results`.
+
+Also that there is an executable script `just-upload.sh` in the root of that repository.
+```bash
+#just-upload.sh
+
+
+#!/bin/bash
+# move to the directory of the script
+cd $(dirname "$0")
+
+git add . &&
+git commit -m "automated commit" && git push
+```
+
+### saving experiment data
+```bash
+cp -r results/ ../scala-native-benchmark-results &&
+../scala-native-benchmark-results/just-upload.sh
+```
+
+### restoring experiment data
+```bash
+cp -r ../scala-native-benchmark-results results/
+```
+
+### uploading a report
+```bash
+mkdir -p ../scala-native-benchmark-results/reports
+cp -r "$REPORT" ../scala-native-benchmark-results/reports &&
+../scala-native-benchmark-results/just-upload.sh
+```
diff --git a/build.sbt b/build.sbt
@@ -1 +1,6 @@
 scalaVersion := "2.11.12"
+enablePlugins(ScalaNativePlugin)
+nativeLinkStubs := true
+nativeGC := "immix"
+nativeMode := "release"
+nativeLTO := "thin"
diff --git a/confs/jvm/run b/confs/jvm/run
@@ -1 +1 @@
-java -Xmx1024M -Xms1024M -classpath target/scala-2.11/classes:$HOME/.ivy2/cache/org.scala-lang/scala-library/jars/scala-library-2.11.12.jar $BENCH
+java $JAVA_ARGS -classpath target/scala-2.11/classes:$HOME/.ivy2/cache/org.scala-lang/scala-library/jars/scala-library-2.11.12.jar $BENCH
diff --git a/confs/native-image-pgo/run b/confs/native-image-pgo/run
@@ -1,2 +1,2 @@
-target/native-image-pgo-bench -Xmx1g -Xmx1g
+target/native-image-pgo-bench $JAVA_SIZE_ARGS
 
diff --git a/confs/native-image/run b/confs/native-image/run
@@ -1 +1 @@
-target/native-image-bench -Xmx1g -Xmx1g
+target/native-image-bench $JAVA_SIZE_ARGS
diff --git a/confs/scala-native-0.3.7/plugins.sbt b/confs/scala-native-0.3.7/plugins.sbt
diff --git a/confs/scala-native-0.3.7/build.sbt → confs/scala-native-0.3.8/build.sbt b/confs/scala-native-0.3.7/build.sbt → confs/scala-native-0.3.8/build.sbt
diff --git a/confs/scala-native-0.3.7/compile → confs/scala-native-0.3.8/compile b/confs/scala-native-0.3.7/compile → confs/scala-native-0.3.8/compile
diff --git a/confs/scala-native-0.3.8/plugins.sbt b/confs/scala-native-0.3.8/plugins.sbt
@@ -0,0 +1 @@
+addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.3.8")
diff --git a/confs/scala-native-0.3.7/run → confs/scala-native-0.3.8/run b/confs/scala-native-0.3.7/run → confs/scala-native-0.3.8/run
diff --git a/confs/scala-native-0.3.9-SNAPSHOT-commix/build.sbt b/confs/scala-native-0.3.9-SNAPSHOT-commix/build.sbt
@@ -0,0 +1,6 @@
+scalaVersion := "2.11.12"
+enablePlugins(ScalaNativePlugin)
+nativeLinkStubs := true
+nativeGC := "commix"
+nativeMode := "release"
+nativeLTO := "thin"
diff --git a/confs/scala-native-0.3.9-SNAPSHOT-commix/compile b/confs/scala-native-0.3.9-SNAPSHOT-commix/compile
@@ -0,0 +1 @@
+nativeLink
diff --git a/confs/scala-native-0.3.9-SNAPSHOT-commix/plugins.sbt b/confs/scala-native-0.3.9-SNAPSHOT-commix/plugins.sbt
@@ -0,0 +1 @@
+addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.3.9-SNAPSHOT")
diff --git a/confs/scala-native-0.3.9-SNAPSHOT-commix/run b/confs/scala-native-0.3.9-SNAPSHOT-commix/run
@@ -0,0 +1 @@
+target/scala-2.11/scala-native-benchmarks-out
diff --git a/confs/scala-native-0.3.9-SNAPSHOT-full-lto/build.sbt b/confs/scala-native-0.3.9-SNAPSHOT-full-lto/build.sbt
@@ -0,0 +1,6 @@
+scalaVersion := "2.11.12"
+enablePlugins(ScalaNativePlugin)
+nativeLinkStubs := true
+nativeGC := "immix"
+nativeMode := "release"
+nativeLTO := "full"
diff --git a/confs/scala-native-0.3.9-SNAPSHOT-full-lto/compile b/confs/scala-native-0.3.9-SNAPSHOT-full-lto/compile
@@ -0,0 +1 @@
+nativeLink
diff --git a/confs/scala-native-0.3.9-SNAPSHOT-full-lto/plugins.sbt b/confs/scala-native-0.3.9-SNAPSHOT-full-lto/plugins.sbt
@@ -0,0 +1 @@
+addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.3.9-SNAPSHOT")
diff --git a/confs/scala-native-0.3.9-SNAPSHOT-full-lto/run b/confs/scala-native-0.3.9-SNAPSHOT-full-lto/run
@@ -0,0 +1 @@
+target/scala-2.11/scala-native-benchmarks-out
diff --git a/confs/scala-native-0.3.9-SNAPSHOT-no-lto/build.sbt b/confs/scala-native-0.3.9-SNAPSHOT-no-lto/build.sbt
@@ -0,0 +1,6 @@
+scalaVersion := "2.11.12"
+enablePlugins(ScalaNativePlugin)
+nativeLinkStubs := true
+nativeGC := "immix"
+nativeMode := "release"
+nativeLTO := "none"
diff --git a/confs/scala-native-0.3.9-SNAPSHOT-no-lto/compile b/confs/scala-native-0.3.9-SNAPSHOT-no-lto/compile
@@ -0,0 +1 @@
+nativeLink
diff --git a/confs/scala-native-0.3.9-SNAPSHOT-no-lto/plugins.sbt b/confs/scala-native-0.3.9-SNAPSHOT-no-lto/plugins.sbt
@@ -0,0 +1 @@
+addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.3.9-SNAPSHOT")
diff --git a/confs/scala-native-0.3.9-SNAPSHOT-no-lto/run b/confs/scala-native-0.3.9-SNAPSHOT-no-lto/run
@@ -0,0 +1 @@
+target/scala-2.11/scala-native-benchmarks-out
diff --git a/confs/scala-native-0.3.9-SNAPSHOT/build.sbt b/confs/scala-native-0.3.9-SNAPSHOT/build.sbt
@@ -0,0 +1,6 @@
+scalaVersion := "2.11.12"
+enablePlugins(ScalaNativePlugin)
+nativeLinkStubs := true
+nativeGC := "immix"
+nativeMode := "release"
+nativeLTO := "thin"
diff --git a/confs/scala-native-0.3.9-SNAPSHOT/compile b/confs/scala-native-0.3.9-SNAPSHOT/compile
@@ -0,0 +1 @@
+nativeLink
diff --git a/confs/scala-native-0.3.9-SNAPSHOT/plugins.sbt b/confs/scala-native-0.3.9-SNAPSHOT/plugins.sbt
@@ -0,0 +1 @@
+addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.3.9-SNAPSHOT")
diff --git a/confs/scala-native-0.3.9-SNAPSHOT/run b/confs/scala-native-0.3.9-SNAPSHOT/run
@@ -0,0 +1 @@
+target/scala-2.11/scala-native-benchmarks-out
diff --git a/confs/scala-native-0.4.0-SNAPSHOT-commix/build.sbt b/confs/scala-native-0.4.0-SNAPSHOT-commix/build.sbt
@@ -0,0 +1,6 @@
+scalaVersion := "2.11.12"
+enablePlugins(ScalaNativePlugin)
+nativeLinkStubs := true
+nativeGC := "commix"
+nativeMode := "release"
+nativeLTO := "thin"
diff --git a/confs/scala-native-0.4.0-SNAPSHOT-commix/compile b/confs/scala-native-0.4.0-SNAPSHOT-commix/compile
@@ -0,0 +1 @@
+nativeLink
diff --git a/confs/scala-native-0.4.0-SNAPSHOT-commix/plugins.sbt b/confs/scala-native-0.4.0-SNAPSHOT-commix/plugins.sbt
@@ -0,0 +1 @@
+addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.4.0-SNAPSHOT")
diff --git a/confs/scala-native-0.4.0-SNAPSHOT-commix/run b/confs/scala-native-0.4.0-SNAPSHOT-commix/run
@@ -0,0 +1 @@
+target/scala-2.11/scala-native-benchmarks-out
diff --git a/confs/scala-native-0.4.0-SNAPSHOT/build.sbt b/confs/scala-native-0.4.0-SNAPSHOT/build.sbt
@@ -0,0 +1,6 @@
+scalaVersion := "2.11.12"
+enablePlugins(ScalaNativePlugin)
+nativeLinkStubs := true
+nativeGC := "immix"
+nativeMode := "release"
+nativeLTO := "thin"
diff --git a/confs/scala-native-0.4.0-SNAPSHOT/compile b/confs/scala-native-0.4.0-SNAPSHOT/compile
@@ -0,0 +1 @@
+nativeLink
diff --git a/confs/scala-native-0.4.0-SNAPSHOT/plugins.sbt b/confs/scala-native-0.4.0-SNAPSHOT/plugins.sbt
@@ -0,0 +1 @@
+addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.4.0-SNAPSHOT")
diff --git a/confs/scala-native-0.4.0-SNAPSHOT/run b/confs/scala-native-0.4.0-SNAPSHOT/run
@@ -0,0 +1 @@
+target/scala-2.11/scala-native-benchmarks-out
diff --git a/input/histogram.Histogram b/input/histogram.Histogram
@@ -0,0 +1 @@
+300000,100000
diff --git a/output/histogram.Histogram b/output/histogram.Histogram
@@ -0,0 +1 @@
+true
diff --git a/project/plugins.sbt b/project/plugins.sbt
@@ -1 +1 @@
-addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.3.7")
+addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.4.0-SNAPSHOT")
diff --git a/scripts/install-dependecies.sh b/scripts/install-dependecies.sh
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+sudo pip2 install numpy matplotlib
+sudo apt update && sudo apt install python-tk
diff --git a/scripts/notebook.ipynb b/scripts/notebook.ipynb
@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "metadata": {
     "collapsed": false,
     "scrolled": true
@@ -11,7 +11,6 @@
     {
      "ename": "ModuleNotFoundError",
      "evalue": "No module named 'matplotlib'",
-     "output_type": "error",
      "traceback": [
       "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
       "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
@@ -24,7 +23,8 @@
       "\u001b[0;32m/usr/lib/python3/dist-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36menable_matplotlib\u001b[0;34m(self, gui)\u001b[0m\n\u001b[1;32m   2933\u001b[0m         \"\"\"\n\u001b[1;32m   2934\u001b[0m         \u001b[0;32mfrom\u001b[0m \u001b[0mIPython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcore\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpylabtools\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mpt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2935\u001b[0;31m         \u001b[0mgui\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbackend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind_gui_and_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgui\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpylab_gui_select\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2936\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2937\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mgui\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m'inline'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
       "\u001b[0;32m/usr/lib/python3/dist-packages/IPython/core/pylabtools.py\u001b[0m in \u001b[0;36mfind_gui_and_backend\u001b[0;34m(gui, gui_select)\u001b[0m\n\u001b[1;32m    257\u001b[0m     \"\"\"\n\u001b[1;32m    258\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 259\u001b[0;31m     \u001b[0;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    260\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    261\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mgui\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mgui\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m'auto'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
       "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'matplotlib'"
-     ]
+     ],
+     "output_type": "error"
     }
    ],
    "source": [
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		java -Xmx1024M -Xms1024M -classpath target/scala-2.11/classes:$HOME/.ivy2/cache/org.scala-lang/scala-library/jars/scala-library-2.11.12.jar $BENCH
		java $JAVA_ARGS -classpath target/scala-2.11/classes:$HOME/.ivy2/cache/org.scala-lang/scala-library/jars/scala-library-2.11.12.jar $BENCH
Original file line number	Diff line number	Diff line change
		@@ -1,2 +1,2 @@
		target/native-image-pgo-bench -Xmx1g -Xmx1g
		target/native-image-pgo-bench $JAVA_SIZE_ARGS
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		target/native-image-bench -Xmx1g -Xmx1g
		target/native-image-bench $JAVA_SIZE_ARGS
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.3.8")