Merge pull request #364 from Duhemm/wip/benchmarks-base

Basic benchmarking infrastructure

.travis.yml

@@ -55,4 +55,4 @@ before_script:
 script:
   - java -version
   - bin/scalafmt --version 0.4.7 --test
-  - sbt 'cleanCache' 'cleanLocal' 'nscplugin/publishLocal' 'nativelib/publishLocal' 'publishLocal' 'sandbox/run' 'demoNative/run' 'tests/run' 'tools/test' 'scripted' 'publishSnapshot'
+  - sbt 'cleanCache' 'cleanLocal' 'nscplugin/publishLocal' 'nativelib/publishLocal' 'publishLocal' 'sandbox/run' 'demoNative/run' 'tests/run' 'tools/test' 'benchmarks/run' 'scripted' 'publishSnapshot'

benchmarks/src/main/scala/benchmarks/Benchmark.scala

@@ -0,0 +1,85 @@
+package benchmarks
+
+sealed abstract class BenchmarkResult(val name: String, val success: Boolean)
+
+case class BenchmarkCompleted(override val name: String,
+                              minNs: Long,
+                              maxNs: Long,
+                              avgNs: Long,
+                              iterations: Int,
+                              override val success: Boolean)
+    extends BenchmarkResult(name, success) {
+  override def toString: String = {
+    def format(n: Double, decimals: Int = 3): String = {
+      val s = n.toString
+      s.substring(0, s.indexOf('.') + decimals + 1)
+    }
+
+    val minMs = format(minNs / 1e6)
+    val maxMs = format(maxNs / 1e6)
+    val avgMs = format(avgNs / 1e6)
+    (if (success) "  [ok] " else "  [fail] ") + name +
+      s": $iterations iterations, min ${minMs}ms, max ${maxMs}ms," +
+      s" avg ${avgMs}ms"
+  }
+}
+
+case class BenchmarkFailed(override val name: String, cause: Throwable)
+    extends BenchmarkResult(name, false) {
+  override def toString: String =
+    s"""  [fail] $name threw an exception:
+       |$cause""".stripMargin
+}
+
+case class BenchmarkDisabled(override val name: String)
+    extends BenchmarkResult(name, true) {
+  override def toString: String =
+    s""" [???] $name is disabled."""
+}
+
+abstract class Benchmark[T] {
+  def run(): T
+  def check(t: T): Boolean
+
+  private class BenchmarkDisabledException extends Exception
+  final def disableBenchmark(): Nothing = throw new BenchmarkDisabledException
+
+  final def estimateTime(): Long = {
+    val start = System.nanoTime()
+    val _     = try { run() } catch { case _: Throwable => () }
+    System.nanoTime() - start
+  }
+
+  final def loop(iterations: Int): BenchmarkResult =
+    try {
+      var min: Long        = Long.MaxValue
+      var max: Long        = Long.MinValue
+      var success: Boolean = true
+
+      val t0 = System.nanoTime()
+      for (_ <- 1 to iterations) {
+        val start  = System.nanoTime()
+        val result = run()
+        val end    = System.nanoTime()
+
+        success = success && check(result)
+        val elapsed = end - start
+        if (elapsed > max) max = elapsed
+        if (elapsed < min) min = elapsed
+      }
+      val totalTime = System.nanoTime() - t0
+      val average   = totalTime / iterations
+
+      BenchmarkCompleted(this.getClass.getName,
+                         min,
+                         max,
+                         average,
+                         iterations,
+                         success)
+    } catch {
+      case _: BenchmarkDisabledException =>
+        BenchmarkDisabled(this.getClass.getName)
+      case t: Throwable =>
+        BenchmarkFailed(this.getClass.getName, t)
+    }
+}

benchmarks/src/main/scala/benchmarks/DummyBenchmark.scala

@@ -0,0 +1,9 @@
+package benchmarks
+
+class DummyBenchmark extends Benchmark[Int] {
+  override def run(): Int =
+    (1 to 1000).sum
+
+  override def check(t: Int): Boolean =
+    t == 500500
+}

benchmarks/src/main/scala/benchmarks/Main.scala

@@ -0,0 +1,30 @@
+package benchmarks
+
+import java.lang.System.exit
+
+object Main {
+
+  val nsPerBenchmark = 3e9.toLong
+
+  def main(args: Array[String]): Unit = {
+
+    val benchmarks = Discover.discovered
+
+    val success =
+      benchmarks forall { bench =>
+        // Run once to estimate how long this benchmark takes
+        val timeEstimate = bench.estimateTime()
+        val iterations   = (nsPerBenchmark / timeEstimate).toInt
+
+        val result = bench.loop(iterations)
+        println(result)
+
+        // Sleep between test to avoid error (???)
+        Thread.sleep(50)
+
+        result.success
+      }
+
+    if (success) exit(0) else exit(1)
+  }
+}

benchmarks/src/main/scala/bounce/BounceBenchmark.scala

@@ -0,0 +1,74 @@
+/* This code is based on the SOM class library.
+ *
+ * Copyright (c) 2001-2016 see AUTHORS.md file
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the 'Software'), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+package bounce
+
+import som.Random
+
+class BounceBenchmark extends benchmarks.Benchmark[Int] {
+  private class Ball(random: Random) {
+    private var x: Int    = random.next() % 500
+    private var y: Int    = random.next() % 500
+    private var xVel: Int = (random.next() % 300) - 150
+    private var yVel: Int = (random.next() % 300) - 150
+
+    def bounce(): Boolean = {
+      val xLimit: Int = 500
+      val yLimit: Int = 500
+      var bounced     = false
+
+      x += xVel;
+      y += yVel;
+      if (x > xLimit) {
+        x = xLimit; xVel = 0 - Math.abs(xVel); bounced = true;
+      }
+      if (x < 0) { x = 0; xVel = Math.abs(xVel); bounced = true; }
+      if (y > yLimit) {
+        y = yLimit; yVel = 0 - Math.abs(yVel); bounced = true;
+      }
+      if (y < 0) { y = 0; yVel = Math.abs(yVel); bounced = true; }
+
+      bounced
+    }
+  }
+
+  override def run(): Int = {
+    val random = new Random()
+
+    val ballCount = 100
+    var bounces   = 0
+    val balls     = Array.fill(ballCount)(new Ball(random))
+
+    (0 to 49).foreach { i =>
+      balls.foreach { ball =>
+        if (ball.bounce()) {
+          bounces += 1
+        }
+      }
+    }
+
+    bounces
+  }
+
+  override def check(result: Int): Boolean =
+    result == 1331
+}

benchmarks/src/main/scala/cd/Aircraft.scala

@@ -0,0 +1,3 @@
+package cd
+
+final class Aircraft(val callsign: CallSign, val position: Vector3D)

benchmarks/src/main/scala/cd/CDBenchmark.scala

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2001-2016 Stefan Marr
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the 'Software'), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+package cd
+
+import som._
+
+class CDBenchmark extends benchmarks.Benchmark[(Int, Int)] {
+
+  private val numAircrafts = Array(1000, 500, 250, 100, 10)
+  private var i            = 0
+
+  override def run(): (Int, Int) = {
+    disableBenchmark()
+    val aircrafts = numAircrafts(i % numAircrafts.length)
+    i = i + 1
+    (aircrafts, benchmark(aircrafts))
+  }
+
+  override def check(t: (Int, Int)): Boolean =
+    check(t._2, t._1)
+
+  def benchmark(numAircrafts: Int): Int = {
+    val numFrames        = 200
+    val simulator        = new Simulator(numAircrafts);
+    val detector         = new CollisionDetector();
+    var actualCollisions = 0
+
+    (0 until numFrames).map { i =>
+      val time       = i / 10.0
+      val collisions = detector.handleNewFrame(simulator.simulate(time))
+      actualCollisions += collisions.size()
+    }
+
+    actualCollisions
+  }
+
+  def check(actualCollisions: Int, numAircrafts: Int): Boolean = {
+    if (numAircrafts == 1000) { return actualCollisions == 14484 }
+    if (numAircrafts == 500) { return actualCollisions == 14484 }
+    if (numAircrafts == 250) { return actualCollisions == 10830 }
+    if (numAircrafts == 100) { return actualCollisions == 4305 }
+    if (numAircrafts == 10) { return actualCollisions == 390 }
+
+    System.out.println("No verification result for " + numAircrafts + " found")
+    System.out.println("Result is: " + actualCollisions)
+
+    return false
+  }
+}

benchmarks/src/main/scala/cd/CallSign.scala

@@ -0,0 +1,8 @@
+package cd
+
+final class CallSign(val value: Int) extends Comparable[CallSign] {
+  override def compareTo(other: CallSign) =
+    if (value == other.value) 0
+    else if (value < other.value) -1
+    else 1
+}

benchmarks/src/main/scala/cd/Collision.scala

@@ -0,0 +1,5 @@
+package cd
+
+final class Collision(val aircraftA: CallSign,
+                      val aircraftB: CallSign,
+                      val position: Vector3D)