Make Benchmarks more deterministic

benchmarks/src/main/scala/benchmarks/Benchmark.scala

@@ -14,26 +14,25 @@ case class BenchmarkFailed(override val name: String, cause: Throwable)
 case class BenchmarkDisabled(override val name: String)
     extends BenchmarkResult(name, true)
 
-abstract class Benchmark[T] {
+sealed class BenchmarkRunningTime(val iterations: Int)
+
+case object VeryLongRunningTime extends BenchmarkRunningTime(20)
+case object LongRunningTime     extends BenchmarkRunningTime(1000)
+case object MediumRunningTime   extends BenchmarkRunningTime(10000)
+case object ShortRunningTime    extends BenchmarkRunningTime(30000)
+case object UnknownRunningTime  extends BenchmarkRunningTime(1)
+
+abstract class Benchmark[T]() {
   def run(): T
   def check(t: T): Boolean
 
-  def iterations(): Int = {
-    // Run once to estimate how long this benchmark takes
-    val nsPerBenchmark = 3e9.toLong
-    val timeEstimate   = estimateTime()
-    Math.max(20, (nsPerBenchmark / timeEstimate).toInt)
-  }
+  val runningTime: BenchmarkRunningTime
+
+  def iterations(): Int = runningTime.iterations
 
   private class BenchmarkDisabledException extends Exception
   final def disableBenchmark(): Nothing = throw new BenchmarkDisabledException
 
-  final def estimateTime(): Long = {
-    val start = System.nanoTime()
-    val _     = try { run() } catch { case _: Throwable => () }
-    System.nanoTime() - start
-  }
-
   final def loop(iterations: Int): BenchmarkResult =
     try {
       var success: Boolean   = true

benchmarks/src/main/scala/benchmarks/DummyBenchmark.scala

@@ -1,6 +1,9 @@
 package benchmarks
 
 class DummyBenchmark extends Benchmark[Int] {
+
+  override val runningTime: BenchmarkRunningTime = ShortRunningTime
+
   override def run(): Int =
     (1 to 1000).sum
 

benchmarks/src/main/scala/benchmarks/Formats.scala

@@ -39,7 +39,11 @@ object TextFormat extends Format {
             "avg [ms]")
     }
 
-    val rows = results.map {
+    val rows = results.sortBy {
+      case _: BenchmarkCompleted => 0
+      case _: BenchmarkDisabled  => 1
+      case _: BenchmarkFailed    => 2
+    } map {
       case completed: BenchmarkCompleted =>
         CompletedRow(completed)
       case failed: BenchmarkFailed =>

benchmarks/src/main/scala/benchmarks/Main.scala

@@ -4,11 +4,11 @@ import java.lang.System.exit
 
 object Main {
   def main(args: Array[String]): Unit = {
-    val benchmarks = Discover.discovered
+    val benchmarks = Discover.discovered.sortBy(_.getClass.getSimpleName)
 
     val opts = Opts(args)
     val results = benchmarks.map { bench =>
-      val iterations = if (!opts.test) bench.iterations else 1
+      val iterations = if (!opts.test) bench.iterations() else 1
       bench.loop(iterations)
       bench.loop(iterations)
     }

benchmarks/src/main/scala/bounce/BounceBenchmark.scala

@@ -22,6 +22,7 @@
  */
 package bounce
 
+import benchmarks.{BenchmarkRunningTime, ShortRunningTime}
 import som.Random
 
 class BounceBenchmark extends benchmarks.Benchmark[Int] {
@@ -51,6 +52,8 @@ class BounceBenchmark extends benchmarks.Benchmark[Int] {
     }
   }
 
+  override val runningTime: BenchmarkRunningTime = ShortRunningTime
+
   override def run(): Int = {
     val random = new Random()
 

benchmarks/src/main/scala/brainfuck/BrainfuckBenchmark.scala

@@ -26,7 +26,11 @@
 
 package brainfuck
 
+import benchmarks.{BenchmarkRunningTime, LongRunningTime}
+
 class BrainfuckBenchmark extends benchmarks.Benchmark[String] {
+  override val runningTime: BenchmarkRunningTime = LongRunningTime
+
   override def run(): String = {
     new Program(Program.asText).run
   }

benchmarks/src/main/scala/cd/CDBenchmark.scala

@@ -21,13 +21,16 @@
  */
 package cd
 
+import benchmarks.{BenchmarkRunningTime, VeryLongRunningTime}
 import som._
 
 class CDBenchmark extends benchmarks.Benchmark[(Int, Int)] {
 
   private val numAircrafts = Array(1000, 500, 250, 100, 10)
   private var i            = 0
 
+  override val runningTime: BenchmarkRunningTime = VeryLongRunningTime
+
   override def run(): (Int, Int) = {
     val aircrafts = numAircrafts(i % numAircrafts.length)
     i = i + 1

benchmarks/src/main/scala/deltablue/DeltaBlueBenchmark.scala

@@ -44,10 +44,14 @@ package deltablue
  * I've kept it this way to avoid deviating too much from the original
  * implementation.
  */
+import benchmarks.{BenchmarkRunningTime, MediumRunningTime}
+
 import scala.collection.mutable.{ArrayBuffer, ListBuffer, Stack}
 
 class DeltaBlueBenchmark extends benchmarks.Benchmark[Unit] {
 
+  override val runningTime: BenchmarkRunningTime = MediumRunningTime
+
   override def run(): Unit = {
     chainTest(100)
     projectionTest(100)

benchmarks/src/main/scala/gcbench/GCBenchBenchmark.scala

@@ -40,9 +40,13 @@
 
 package gcbench
 
+import benchmarks.{BenchmarkRunningTime, VeryLongRunningTime}
+
 class GCBenchBenchmark extends benchmarks.Benchmark[(Node, Array[Double])] {
+  override val runningTime: BenchmarkRunningTime = VeryLongRunningTime
 
   override def run(): (Node, Array[Double]) = GCBenchBenchmark.start()
+
   override def check(result: (Node, Array[Double])): Boolean =
     result._1 != null && result._2(1000) == 1.0 / 1000
 

benchmarks/src/main/scala/havlak/HavlakBenchmark.scala

@@ -21,8 +21,12 @@
  */
 package havlak
 
+import benchmarks.{BenchmarkRunningTime, VeryLongRunningTime}
+
 class HavlakBenchmark extends benchmarks.Benchmark[Array[Int]] {
 
+  override val runningTime: BenchmarkRunningTime = VeryLongRunningTime
+
   override def run(): Array[Int] =
     new LoopTesterApp().main(15, 50, 10, 10, 5)
 

benchmarks/src/main/scala/json/JsonBenchmark.scala

@@ -20,6 +20,7 @@
  * SOFTWARE.
  ******************************************************************************/
 package json
+import benchmarks.{BenchmarkRunningTime, LongRunningTime}
 
 /**
  * This benchmark uses a variant of the JsonParser that operates only on
@@ -28,6 +29,9 @@ package json
  * @author smarr
  */
 class JsonBenchmark extends benchmarks.Benchmark[JsonValue] {
+
+  override val runningTime: BenchmarkRunningTime = LongRunningTime
+
   override def run(): JsonValue = {
     (new JsonPureStringParser(rapBenchmarkMinified)).parse()
   }

benchmarks/src/main/scala/list/ListBenchmark.scala

@@ -22,6 +22,8 @@
  */
 package list
 
+import benchmarks.{BenchmarkRunningTime, ShortRunningTime}
+
 class ListBenchmark extends benchmarks.Benchmark[Int] {
   final class Element(var value: Any, var next: Element = null) {
     def length(): Int = {
@@ -33,6 +35,8 @@ class ListBenchmark extends benchmarks.Benchmark[Int] {
     }
   }
 
+  override val runningTime: BenchmarkRunningTime = ShortRunningTime
+
   override def run(): Int = {
     val result = tail(makeList(15), makeList(10), makeList(6))
     result.length()

benchmarks/src/main/scala/listperm/ListPermutationsBenchmark.scala

@@ -1,7 +1,12 @@
 package listperm
 
+import benchmarks.{BenchmarkRunningTime, VeryLongRunningTime}
+
 class ListPermutationsBenchmark extends benchmarks.Benchmark[Int] {
   val size = 8
+
+  override val runningTime: BenchmarkRunningTime = VeryLongRunningTime
+
   override def run(): Int = {
     val permIter = (0 until size).toList.permutations
 

benchmarks/src/main/scala/mandelbrot/MandelbrotBenchmark.scala

@@ -42,11 +42,15 @@
 // http://benchmarksgame.alioth.debian.org/u64q/program.php?test=mandelbrot&lang=yarv&id=3
 package mandelbrot
 
+import benchmarks.{BenchmarkRunningTime, VeryLongRunningTime}
+
 class MandelbrotBenchmark extends benchmarks.Benchmark[(Int, Int)] {
 
   private val sizes = List(750, 500, 1)
   private var i     = 0
 
+  override val runningTime: BenchmarkRunningTime = VeryLongRunningTime
+
   override def run(): (Int, Int) = {
     val size = sizes(i % sizes.length)
     i = i + 1

benchmarks/src/main/scala/nbody/NbodyBenchmark.scala

@@ -5,7 +5,12 @@
  */
 package nbody
 
+import benchmarks.{BenchmarkRunningTime, VeryLongRunningTime}
+
 class NbodyBenchmark extends benchmarks.Benchmark[Double] {
+
+  override val runningTime: BenchmarkRunningTime = VeryLongRunningTime
+
   override def run(): Double = {
     val system = new NBodySystem()
 

benchmarks/src/main/scala/permute/PermuteBenchmark.scala

@@ -22,10 +22,14 @@
  */
 package permute
 
+import benchmarks.{BenchmarkRunningTime, ShortRunningTime}
+
 class PermuteBenchmark extends benchmarks.Benchmark[Int] {
   var count: Int    = _
   var v: Array[Int] = _
 
+  override val runningTime: BenchmarkRunningTime = ShortRunningTime
+
   override def run(): Int = {
     count = 0
     v = new Array[Int](6)

benchmarks/src/main/scala/queens/QueensBenchmark.scala

@@ -22,12 +22,16 @@
  */
 package queens
 
+import benchmarks.{BenchmarkRunningTime, MediumRunningTime}
+
 class QueensBenchmark extends benchmarks.Benchmark[Boolean] {
   var freeMaxs: Array[Boolean] = _
   var freeRows: Array[Boolean] = _
   var freeMins: Array[Boolean] = _
   var queenRows: Array[Int]    = _
 
+  override val runningTime: BenchmarkRunningTime = MediumRunningTime
+
   override def run(): Boolean = {
     var result = true
     (0 until 10).foreach { i =>

benchmarks/src/main/scala/richards/RichardsBenchmark.scala

@@ -45,12 +45,16 @@
 
 package richards
 
+import benchmarks.{BenchmarkRunningTime, ShortRunningTime}
+
 /**
  * Richards simulates the task dispatcher of an operating system.
  */
 class RichardsBenchmark extends benchmarks.Benchmark[(Int, Int)] {
   import Richards._
 
+  override val runningTime: BenchmarkRunningTime = ShortRunningTime
+
   override def run(): (Int, Int) = {
     val scheduler = new Scheduler()
     scheduler.addIdleTask(ID_IDLE, 0, null, COUNT)

benchmarks/src/main/scala/sha512/SHA512Benchmark.scala

@@ -36,11 +36,15 @@
  */
 
 package sha512
+import benchmarks.{BenchmarkRunningTime, UnknownRunningTime}
 
 /**
  * SHA-512 hashing.
  */
 class SHA512Benchmark extends benchmarks.Benchmark[Boolean] {
+
+  override val runningTime: BenchmarkRunningTime = UnknownRunningTime
+
   override def run(): Boolean = {
     disableBenchmark()
     // Fails

benchmarks/src/main/scala/sieve/SieveBenchmark.scala

@@ -22,7 +22,11 @@
  */
 package sieve
 
+import benchmarks.{BenchmarkRunningTime, ShortRunningTime}
+
 class SieveBenchmark extends benchmarks.Benchmark[Int] {
+  override val runningTime: BenchmarkRunningTime = ShortRunningTime
+
   override def run(): Int = {
     val flags = Array.fill(5000)(true)
     return sieve(flags, 5000)

benchmarks/src/main/scala/storage/StorageBenchmark.scala

@@ -24,11 +24,14 @@ package storage
 
 import java.util.Arrays
 
+import benchmarks.{BenchmarkRunningTime, MediumRunningTime}
 import som.Random
 
 class StorageBenchmark extends benchmarks.Benchmark[Int] {
   private var count: Int = _
 
+  override val runningTime: BenchmarkRunningTime = MediumRunningTime
+
   override def run(): Int = {
     val random = new Random()
     count = 0

benchmarks/src/main/scala/sudoku/SudokuBenchmark.scala

@@ -10,12 +10,16 @@
 
 package sudoku
 
+import benchmarks.{BenchmarkRunningTime, LongRunningTime}
+
 import scala.language.implicitConversions
 
 class SudokuBenchmark
     extends benchmarks.Benchmark[Option[
       scala.collection.mutable.Map[String, String]]] {
 
+  override val runningTime: BenchmarkRunningTime = LongRunningTime
+
   override def run(): Option[Grid] = {
     solve(grid1)
   }

benchmarks/src/main/scala/towers/TowersBenchmark.scala

@@ -22,6 +22,8 @@
  */
 package towers
 
+import benchmarks.{BenchmarkRunningTime, ShortRunningTime}
+
 class TowersBenchmark extends benchmarks.Benchmark[Int] {
   final class TowersDisk(val size: Int, var next: TowersDisk = null)
 
@@ -74,6 +76,8 @@ class TowersBenchmark extends benchmarks.Benchmark[Int] {
     }
   }
 
+  override val runningTime: BenchmarkRunningTime = ShortRunningTime
+
   override def run(): Int = {
     piles = new Array[TowersDisk](3)
     buildTowerAt(0, 13)

benchmarks/src/main/scala/tracer/TracerBenchmark.scala

@@ -16,6 +16,8 @@
 
 package tracer
 
+import benchmarks.{BenchmarkRunningTime, LongRunningTime}
+
 class TracerBenchmark extends benchmarks.Benchmark[Unit] {
 
   val config = EngineConfiguration(
@@ -30,6 +32,8 @@ class TracerBenchmark extends benchmarks.Benchmark[Unit] {
     renderReflections = true
   )
 
+  override val runningTime: BenchmarkRunningTime = LongRunningTime
+
   override def run(): Unit =
     new RenderScene().renderScene(config, null)