Allow passing any JMH option to scala3-bench

dotty-staging · Oct 21, 2021 · 32324fe · 32324fe
1 parent f8b869b
commit 32324fe
Show file tree

Hide file tree

Showing 15 changed files with 86 additions and 72 deletions.
diff --git a/bench/profiles/ci.yml b/bench/profiles/ci.yml
@@ -1,55 +1,55 @@
 scripts:
 
     re2s:
-        - measure 1 1 1 $(find $PROG_HOME/tests/re2s/src -name *.scala)
+        - measure -wi 1 -i 1 -f 1 -- $(find $PROG_HOME/tests/re2s/src -name *.scala)
 
     implicit-cache:
-        - measure 1 1 1 $PROG_HOME/dotty/tests/bench/implicit_cache.scala
+        - measure -wi 1 -i 1 -f 1 -- $PROG_HOME/dotty/tests/bench/implicit_cache.scala
 
     implicit-cache-from-tasty:
         - source $PROG_HOME/dotty/bench/scripts/implicit-cache-from-tasty
 
     implicit-scope-loop:
-        - measure 1 1 1 $PROG_HOME/dotty/tests/bench/implicit-scope-loop.scala
+        - measure -wi 1 -i 1 -f 1 -- $PROG_HOME/dotty/tests/bench/implicit-scope-loop.scala
 
     i1535:
-        - measure 1 1 1 $PROG_HOME/dotty/tests/bench/i1535.scala
+        - measure -wi 1 -i 1 -f 1 -- $PROG_HOME/dotty/tests/bench/i1535.scala
 
     i1687:
-        - measure 1 1 1 $PROG_HOME/dotty/tests/bench/i1687.scala
+        - measure -wi 1 -i 1 -f 1 -- $PROG_HOME/dotty/tests/bench/i1687.scala
 
     empty-class:
-        - measure 1 1 1 $PROG_HOME/dotty/tests/bench/empty-class.scala
+        - measure -wi 1 -i 1 -f 1 -- $PROG_HOME/dotty/tests/bench/empty-class.scala
 
     empty-object:
-        - measure 1 1 1 $PROG_HOME/dotty/tests/bench/empty-object.scala
+        - measure -wi 1 -i 1 -f 1 -- $PROG_HOME/dotty/tests/bench/empty-object.scala
 
     empty-file:
-        - measure 1 1 1 $PROG_HOME/dotty/tests/bench/empty-file.scala
+        - measure -wi 1 -i 1 -f 1 -- $PROG_HOME/dotty/tests/bench/empty-file.scala
 
     patmatexhaust:
-        - measure 1 1 1 $PROG_HOME/dotty/tests/bench/patmatexhaust.scala
+        - measure -wi 1 -i 1 -f 1 -- $PROG_HOME/dotty/tests/bench/patmatexhaust.scala
 
     exhaustivity-I:
-        - measure 1 1 1 $PROG_HOME/dotty/tests/bench/exhaustivity-I.scala
+        - measure -wi 1 -i 1 -f 1 -- $PROG_HOME/dotty/tests/bench/exhaustivity-I.scala
 
     exhaustivity-S:
-        - measure 1 1 1 $PROG_HOME/dotty/tests/bench/exhaustivity-S.scala
+        - measure -wi 1 -i 1 -f 1 -- $PROG_HOME/dotty/tests/bench/exhaustivity-S.scala
 
     exhaustivity-T:
-        - measure 1 1 1 $PROG_HOME/dotty/tests/bench/exhaustivity-T.scala
+        - measure -wi 1 -i 1 -f 1 -- $PROG_HOME/dotty/tests/bench/exhaustivity-T.scala
 
     exhaustivity-V:
-        - measure 1 1 1 $PROG_HOME/dotty/tests/bench/exhaustivity-V.scala
+        - measure -wi 1 -i 1 -f 1 -- $PROG_HOME/dotty/tests/bench/exhaustivity-V.scala
 
     implicitNums:
-        - measure 1 1 1 $PROG_HOME/dotty/tests/bench/implicitNums.scala
+        - measure -wi 1 -i 1 -f 1 -- $PROG_HOME/dotty/tests/bench/implicitNums.scala
 
     implicitNums-from-tasty:
         - source $PROG_HOME/dotty/bench/scripts/implicitNums-from-tasty
 
     inductive-implicits:
-        - measure 1 1 1 $PROG_HOME/dotty/tests/bench/inductive-implicits.scala
+        - measure -wi 1 -i 1 -f 1 -- $PROG_HOME/dotty/tests/bench/inductive-implicits.scala
 
     scalap:
         - source $PROG_HOME/dotty/bench/scripts/scalap

diff --git a/bench/profiles/empty.yml b/bench/profiles/empty.yml
@@ -21,13 +21,13 @@ charts:
 scripts:
 
     empty-class:
-        - measure $PROG_HOME/dotty/tests/bench/empty-class.scala
+        - measure -- $PROG_HOME/dotty/tests/bench/empty-class.scala
 
     empty-object:
-        - measure $PROG_HOME/dotty/tests/bench/empty-object.scala
+        - measure -- $PROG_HOME/dotty/tests/bench/empty-object.scala
 
     empty-file:
-        - measure $PROG_HOME/dotty/tests/bench/empty-file.scala
+        - measure -- $PROG_HOME/dotty/tests/bench/empty-file.scala
 
 config:
     pr_base_url: "https://github.com/lampepfl/dotty/pull/"
diff --git a/bench/profiles/exhaustivity.yml b/bench/profiles/exhaustivity.yml
@@ -50,28 +50,28 @@ charts:
 scripts:
 
     patmatexhaust:
-        - measure 20 40 3 $PROG_HOME/dotty/tests/bench/patmatexhaust.scala
+        - measure -wi 20 -i 40 -f 3 -- $PROG_HOME/dotty/tests/bench/patmatexhaust.scala
 
     exhaustivity-I:
-        - measure 20 40 3 $PROG_HOME/dotty/tests/bench/exhaustivity-I.scala
+        - measure -wi 20 -i 40 -f 3 -- $PROG_HOME/dotty/tests/bench/exhaustivity-I.scala
 
     exhaustivity-S:
-        - measure 20 40 3 $PROG_HOME/dotty/tests/bench/exhaustivity-S.scala
+        - measure -wi 20 -i 40 -f 3 -- $PROG_HOME/dotty/tests/bench/exhaustivity-S.scala
 
     exhaustivity-T:
-        - measure 20 40 3 $PROG_HOME/dotty/tests/bench/exhaustivity-T.scala
+        - measure -wi 20 -i 40 -f 3 -- $PROG_HOME/dotty/tests/bench/exhaustivity-T.scala
 
     exhaustivity-V:
-        - measure 20 40 3 $PROG_HOME/dotty/tests/bench/exhaustivity-V.scala
+        - measure -wi 20 -i 40 -f 3 -- $PROG_HOME/dotty/tests/bench/exhaustivity-V.scala
 
     exhaustivity-mips:
-        - measure 20 40 3 $PROG_HOME/dotty/tests/patmat/i7186.scala
+        - measure -wi 20 -i 40 -f 3 -- $PROG_HOME/dotty/tests/patmat/i7186.scala
 
     exhaustivity-i12241:
-        - measure 20 40 3 $PROG_HOME/dotty/tests/patmat/i12241.scala
+        - measure -wi 20 -i 40 -f 3 -- $PROG_HOME/dotty/tests/patmat/i12241.scala
 
     exhaustivity-i12358:
-        - measure 20 40 3 $PROG_HOME/dotty/tests/patmat/i12358.scala
+        - measure -wi 20 -i 40 -f 3 -- $PROG_HOME/dotty/tests/patmat/i12358.scala
 
 config:
     pr_base_url: "https://github.com/lampepfl/dotty/pull/"
diff --git a/bench/profiles/implicits.yml b/bench/profiles/implicits.yml
@@ -30,19 +30,19 @@ charts:
 scripts:
 
     implicit-cache:
-        - measure $PROG_HOME/dotty/tests/bench/implicit_cache.scala
+        - measure -- $PROG_HOME/dotty/tests/bench/implicit_cache.scala
 
     implicit-cache-from-tasty:
         - source $PROG_HOME/dotty/bench/scripts/implicit-cache-from-tasty
 
     implicit-scope-loop:
-        - measure $PROG_HOME/dotty/tests/bench/implicit-scope-loop.scala
+        - measure -- $PROG_HOME/dotty/tests/bench/implicit-scope-loop.scala
 
     inductive-implicits:
-        - measure $PROG_HOME/dotty/tests/bench/inductive-implicits.scala
+        - measure -- $PROG_HOME/dotty/tests/bench/inductive-implicits.scala
 
     implicitNums:
-        - measure $PROG_HOME/dotty/tests/bench/implicitNums.scala
+        - measure -- $PROG_HOME/dotty/tests/bench/implicitNums.scala
 
     implicitNums-from-tasty:
         - source $PROG_HOME/dotty/bench/scripts/implicitNums-from-tasty

diff --git a/bench/profiles/misc.yml b/bench/profiles/misc.yml
@@ -15,7 +15,7 @@ charts:
 scripts:
 
     i1535:
-        - measure $PROG_HOME/dotty/tests/bench/i1535.scala
+        - measure -- $PROG_HOME/dotty/tests/bench/i1535.scala
 
     i1687:
-        - measure $PROG_HOME/dotty/tests/bench/i1687.scala
+        - measure -- $PROG_HOME/dotty/tests/bench/i1687.scala
diff --git a/bench/profiles/projects.yml b/bench/profiles/projects.yml
@@ -26,10 +26,10 @@ charts:
 
 scripts:
     dotty:
-        - measure -with-compiler $(find $PROG_HOME/dotty/compiler/src/dotty -name *.scala -o -name *.java)
+        - measure --  -with-compiler $ $(find $PROG_HOME/dotty/compiler/src/dotty -name *.scala -o -name *.java)
 
     re2s:
-        - measure $(find $PROG_HOME/tests/re2s/src -name *.scala)
+        - measure -- $(find $PROG_HOME/tests/re2s/src -name *.scala)
 
     # scalapb:
     #    - source $PROG_HOME/dotty/bench/scripts/scalapb

diff --git a/bench/profiles/sbt.yml b/bench/profiles/sbt.yml
@@ -9,7 +9,7 @@ charts:
 
 scripts:
     dotty-sbt:
-        - measure -with-compiler -Yforce-sbt-phases -with-dotty $(find $PROG_HOME/dotty/compiler/src/dotty -name *.scala -o -name *.java)
+        - measure --  -with-compiler $ -Yforce-sbt-phases -with-dotty $(find $PROG_HOME/dotty/compiler/src/dotty -name *.scala -o -name *.java)
 
 config:
     pr_base_url: "https://github.com/lampepfl/dotty/pull/"
diff --git a/bench/profiles/tuples.yml b/bench/profiles/tuples.yml
@@ -44,19 +44,19 @@ charts:
 scripts:
 
     tuple22-creation-apply:
-        - measure $PROG_HOME/dotty/tests/bench/tuple22-creation-apply.scala
+        - measure -- $PROG_HOME/dotty/tests/bench/tuple22-creation-apply.scala
 
     tuple22-creation-cons:
-        - measure $PROG_HOME/dotty/tests/bench/tuple22-creation-cons.scala
+        - measure -- $PROG_HOME/dotty/tests/bench/tuple22-creation-cons.scala
 
     tuple22-tails:
-        - measure $PROG_HOME/dotty/tests/bench/tuple22-tails.scala
+        - measure -- $PROG_HOME/dotty/tests/bench/tuple22-tails.scala
 
     tuple22-apply:
-        - measure $PROG_HOME/dotty/tests/bench/tuple22-apply.scala
+        - measure -- $PROG_HOME/dotty/tests/bench/tuple22-apply.scala
 
     tuple22-size:
-        - measure $PROG_HOME/dotty/tests/bench/tuple22-size.scala
+        - measure -- $PROG_HOME/dotty/tests/bench/tuple22-size.scala
 
     tuple-reverse:
         - measure-run TupleOps.reverse

diff --git a/bench/profiles/typing.yml b/bench/profiles/typing.yml
@@ -8,7 +8,7 @@ charts:
 scripts:
 
     find-ref:
-        - measure $PROG_HOME/dotty/tests/bench/FindRef.scala
+        - measure -- $PROG_HOME/dotty/tests/bench/FindRef.scala
 
 config:
     pr_base_url: "https://github.com/lampepfl/dotty/pull/"
diff --git a/bench/scripts/collection-vector.sh b/bench/scripts/collection-vector.sh
@@ -1,2 +1,2 @@
 #!/usr/bin/env bash
-sbt "dotty-bench-bootstrapped/jmh:run 40 40 3 bench/tests/Vector.scala"
+sbt "dotty-bench-bootstrapped/jmh:run -wi 40 -i 40 -f 3 -- bench/tests/Vector.scala"
diff --git a/bench/scripts/compiler-cold.sh b/bench/scripts/compiler-cold.sh
@@ -1,2 +1,2 @@
 #!/usr/bin/env bash
-find compiler/src/ -type f \( -name "*.scala" -or -name "*.java" \) -exec echo "dotty-bench-bootstrapped/jmh:run 0 1 10" {} + | sbt
+find compiler/src/ -type f \( -name "*.scala" -or -name "*.java" \) -exec echo "dotty-bench-bootstrapped/jmh:run -wi 0 -i 1 -f 10 -- " {} + | sbt
diff --git a/bench/scripts/compiler.sh b/bench/scripts/compiler.sh
@@ -1,2 +1,2 @@
 #!/usr/bin/env bash
-find compiler/src/ -type f \( -name "*.scala" -or -name "*.java" \) -exec echo "dotty-bench-bootstrapped/jmh:run 5 10" {} + | sbt
+find compiler/src/ -type f \( -name "*.scala" -or -name "*.java" \) -exec echo "dotty-bench-bootstrapped/jmh:run -wi 5 -i 10 -- " {} + | sbt
diff --git a/bench/scripts/library-cold.sh b/bench/scripts/library-cold.sh
@@ -1,2 +1,2 @@
 #!/usr/bin/env bash
-find library/src/ -type f \( -name "*.scala" -or -name "*.java" \) -exec echo "dotty-bench-bootstrapped/jmh:run 0 1 10" {} + | sbt
+find library/src/ -type f \( -name "*.scala" -or -name "*.java" \) -exec echo "dotty-bench-bootstrapped/jmh:run -wi 0 -i 1 -f 10 -- " {} + | sbt
diff --git a/bench/scripts/library.sh b/bench/scripts/library.sh
@@ -1,2 +1,2 @@
 #!/usr/bin/env bash
-find library/src/ -type f \( -name "*.scala" -or -name "*.java" \) -exec echo "dotty-bench-bootstrapped/jmh:run 40 30" {} + | sbt
+find library/src/ -type f \( -name "*.scala" -or -name "*.java" \) -exec echo "dotty-bench-bootstrapped/jmh:run -wi 40 -i 30 -- " {} + | sbt
diff --git a/bench/src/main/scala/Benchmarks.scala b/bench/src/main/scala/Benchmarks.scala
@@ -7,7 +7,7 @@ import reporting._
 
 import org.openjdk.jmh.results.RunResult
 import org.openjdk.jmh.runner.Runner
-import org.openjdk.jmh.runner.options.OptionsBuilder
+import org.openjdk.jmh.runner.options.{OptionsBuilder, CommandLineOptions}
 import org.openjdk.jmh.annotations._
 import org.openjdk.jmh.results.format._
 import java.util.concurrent.TimeUnit
@@ -22,37 +22,47 @@ import dotty.tools.io.AbstractFile
 object Bench {
   val COMPILE_OPTS_FILE = "compile.txt"
 
+  def printUsage() =
+    println("Usage (from SBT): scala3-bench/jmh:run <JMH arguments> -- <scalac arguments>")
+    println("Display JMH help: scala3-bench/jmh:run -h")
+    println("Our default JMH options: -wi 30 -i 20 -f 3 -tu ms -bm AverageTime -jvmArgs \"-Xms2G -Xmx2G\"")
+
   def main(args: Array[String]): Unit = {
     if (args.isEmpty) {
-      println("Missing <args>")
+      println("Missing arguments.")
+      printUsage()
       return
     }
-    val (intArgs, args1) = args.span(x => try { x.toInt; true } catch { case _: Throwable => false } )
 
-    val warmup = if (intArgs.length > 0) intArgs(0).toInt else 30
-    val iterations = if (intArgs.length > 1) intArgs(1).toInt else 20
-    val forks = if (intArgs.length > 2) intArgs(2).toInt else 1
+    val (jmhArgs, _scalacArgs) = args.span(_ != "--")
+    val scalacArgs = _scalacArgs.drop(1)
 
+    storeCompileOptions(scalacArgs)
 
-    import File.{ separator => sep }
+    val jmhCliOps = new CommandLineOptions(jmhArgs:_*)
+    val jmhOps = new OptionsBuilder().parent(jmhCliOps)
+
+    // set our own default options
+    if !jmhCliOps.shouldFailOnError().hasValue() then jmhOps.shouldFailOnError(true)
+    if !jmhCliOps.getWarmupIterations().hasValue() then jmhOps.warmupIterations(30)
+    if !jmhCliOps.getMeasurementIterations().hasValue() then jmhOps.measurementIterations(20)
+    if !jmhCliOps.getForkCount().hasValue() then jmhOps.forks(1)
+    if jmhCliOps.getBenchModes().isEmpty() then jmhOps.mode(Mode.AverageTime)
+    if !jmhCliOps.getTimeUnit().hasValue() then jmhOps.timeUnit(TimeUnit.MILLISECONDS)
+    if !jmhCliOps.getJvmArgs().hasValue() then jmhOps.jvmArgs("-Xms2G", "-Xmx2G")
+
+    val runner = new Runner(jmhOps.build())
+
+    if jmhCliOps.shouldHelp() then
+      printUsage()
+      println("Following is the JMH options documentation.")
+      println("-------------------------------------------")
+      return jmhCliOps.showHelp()
+    if jmhCliOps.shouldList() then return runner.list()
+    if jmhCliOps.shouldListWithParams() then return runner.listWithParams(jmhCliOps)
+    if jmhCliOps.shouldListProfilers() then return jmhCliOps.listProfilers()
+    if jmhCliOps.shouldListResultFormats() then return jmhCliOps.listResultFormats()
 
-    val args2 = args1.map { arg =>
-      if ((arg.endsWith(".scala") || arg.endsWith(".java")) && !(new File(arg)).isAbsolute) ".." + sep + arg
-      else arg
-    }
-    storeCompileOptions(args2)
-
-    val opts = new OptionsBuilder()
-               .shouldFailOnError(true)
-               .jvmArgs("-Xms2G", "-Xmx2G")
-               .mode(Mode.AverageTime)
-               .timeUnit(TimeUnit.MILLISECONDS)
-               .warmupIterations(warmup)
-               .measurementIterations(iterations)
-               .forks(forks)
-               .build
-
-    val runner = new Runner(opts) // full access to all JMH features, you can also provide a custom output Format here
     runner.run() // actually run the benchmarks
 
     removeCompileOptions
@@ -61,13 +71,17 @@ object Bench {
   def removeCompileOptions: Unit = new File(COMPILE_OPTS_FILE).delete()
 
   def storeCompileOptions(args: Array[String]): Unit = {
+    import File.{ separator => sep }
+
     val standard_libs = System.getProperty("BENCH_CLASS_PATH")
     val compiler_libs = System.getProperty("BENCH_COMPILER_CLASS_PATH")
 
     val libs = if (args.contains("-with-compiler")) compiler_libs else standard_libs
-    var argsNorm = args.filter(_ != "-with-compiler")
+    var argsNorm = args.filter(_ != "-with-compiler").map { arg =>
+      if ((arg.endsWith(".scala") || arg.endsWith(".java")) && !(new File(arg)).isAbsolute) ".." + sep + arg
+      else arg
+    }
 
-    import File.{ pathSeparator => sep }
     var cpIndex = argsNorm.indexOf("-classpath")
     if (cpIndex == -1) cpIndex = argsNorm.indexOf("-cp")
     if (cpIndex != -1) argsNorm(cpIndex + 1) = argsNorm(cpIndex + 1) + sep + libs