Merge PR #73: Added benchmarks to measure impact of tracing on progra…

…m execution

CHANGELOG.md

@@ -6,13 +6,17 @@
    programming ([PR #52](https://github.com/smarr/SOMns/pull/52))
    - object model uses now a global safepoint to synchronize layout changes
    - array strategies are not safe yet
+
  - Turn writes to method arguments into errors. Before it was leading to 
    confusing setter sends and 'message not understood' errors.
+
  - Added Lee and Vacation benchmarks ([PR #78](https://github.com/smarr/SOMns/pull/78))
 
  - Configuration flag for actor tracing, -atcfg=<config>
    example: -atcfg=mt:mp:pc turns off message timestamps, message parameters and promises
 
+ - Added Validation benchmarks and a new Harness.
+
 ## 0.1.0 - 2016-12-15
 
 This is the first tagged version. For previous changes, please refer to the

core-lib/Benchmarks/ImpactHarness.som

@@ -0,0 +1,230 @@
+(* Copyright (c) 2001-2015 see AUTHORS file
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the 'Software'), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*)
+class ImpactHarness usingPlatform: platform = Value (
+(*  The Harness can be invoked on the command line and should be passed a list
+    of benchmarks to run (class names) as arguments. It will run each of them
+    and output single as well as a total. *)
+| private actors = platform actors.
+  private system = platform system.
+  private platform = platform.
+  private ObjectMirror = platform mirrors ObjectMirror.
+|
+)(
+
+  public class Benchmark = ()(
+
+    (* Needs to return a promise. *)
+    public benchmark: abortP  = ( self subclassResponsibility )
+
+    (* Needs to return a boolean *)
+    public verifyResult: result = ( self subclassResponsibility )
+
+    assert: expected equals: value = (
+      expected = value ifFalse: [
+        self error: 'Expected value (' + expected asString + 
+                    ') differs from actual (' + value asString + 
+                    ') benchmark result.'
+      ].
+      ^ true
+    )
+  )
+
+  public class Run = (
+    | public total           ::= 0.
+      public numIterations   ::= 1.
+      public numFailed       ::= 0.
+      public warmUp          ::= 0.
+      public problemSize     ::= 1.
+      public maxRuntime      ::= nil.
+      public printAll        ::= true.
+      public numThreads      ::= 1.
+      public benchmarkSuite  ::= nil.
+      public name            ::= nil.
+      public timeout         ::= 0.
+    |
+  )(
+
+    public runBenchmark = (
+      | result |
+
+      ('Starting ' + name + ' benchmark ... ') print.
+
+      ^ self doWarmup whenResolved: [:r |
+          self doRuns whenResolved: [:result |
+            self reportBenchmark: result.
+            total := total + result.
+            '' println ] ]
+    )
+
+    doWarmup = (
+      | numIterationsTmp printAllTmp maxRuntimeTmp |
+      warmUp = 0 ifTrue: [
+        | pp |
+        pp := actors createPromisePair.
+        pp resolve: self.
+        '' println.
+        ^ pp promise ].
+
+      numIterationsTmp := numIterations.
+      printAllTmp      := printAll.
+      maxRuntimeTmp    := maxRuntime.
+
+      numIterations := warmUp.
+      printAll      := false.
+      maxRuntime    := nil.
+
+      ' warmup ...' print.
+      ^ self doRuns whenResolved: [:r |
+        numIterations := numIterationsTmp.
+        printAll      := printAllTmp.
+        maxRuntime    := maxRuntimeTmp.
+
+        ' completed.' println
+      ]
+    )
+
+    doRuns = (
+      | total outerLoopP resultPP |
+      total := 0.
+      resultPP := actors createPromisePair.
+      numFailed := 0.
+      outerLoopP := actors async: 1 to: numIterations do: [:i |
+        | startTime endTime runTime bench pp|
+        bench := benchmarkSuite newInstance: problemSize.
+        startTime := system ticks.
+        pp := actors createPromisePair.
+
+        timeout > 0 ifTrue: [
+          actors after: timeout do: [ :r |
+            pp resolve: 1.
+        ] ].
+
+        (bench benchmark: pp promise) whenResolved: [:benchResult |
+          | success | 
+          success := (bench verifyResult: benchResult).
+          success ifFalse: [ numFailed := numFailed + 1 ].
+            endTime := system ticks.
+
+            runTime := endTime - startTime.
+            printAll ifTrue: [ self printRun: runTime success: success].
+
+            total := total + runTime.
+
+            maxRuntime ifNotNil: [
+              total > maxRuntime ifTrue: [
+                resultPP resolve: total ] ] ].
+      ].
+
+      outerLoopP whenResolved: [:r |
+        resultPP resolve: total.
+      ].
+
+      ^ resultPP promise
+    )
+
+    reportBenchmark: result = (
+      | out |
+      out :=  name.
+      out := out + ': iterations='.
+      out := out + numIterations.
+      out := out + ' average: '.
+      out := out + (result / numIterations * numThreads).
+      out := out + 'us'.
+      out := out + ' total: '.
+      out := out + result.
+      out := out + 'us'.
+      out := out + ' success: '.
+      out := out + ((1 - (numFailed//numIterations))*100) round.
+      out := out + '%\n'.
+      out println.
+    )
+
+    printRun: runTime success: success= (
+      (name + ': iterations=1' + ' runtime: ' + runTime + 'us success: ' + success) println
+    )
+
+    public printTotal = (
+      ('Total Runtime: ' + total asString + 'us') println.
+    )
+  )
+
+  processArguments: args for: run = (
+    (* First argument is the BenchmarkHarness *)
+    run name: (args at: 2).
+    run benchmarkSuite: (self loadBenchmarkSuite: (args at: 2)).
+    args size > 2 ifTrue: [
+      run numIterations: (args at: 3) asInteger.
+      args size > 3 ifTrue: [
+        run warmUp: (args at: 4) asInteger.
+        args size > 4 ifTrue: [
+          run problemSize: (args at: 5).
+          args size > 5 ifTrue: [
+            run timeout: (args at: 6) asInteger.
+            args size > 6 ifTrue: [
+              run numThreads: (args at: 7) asInteger.
+    ] ] ] ] ]
+  )
+
+  public main: args = (
+    | run |
+    args size < 2 ifTrue: [ self printUsage. ^ 1 ].
+
+    run := Run new.
+    self processArguments: args for: run.
+
+    ^ run runBenchmark ensure: [:r |
+      run printTotal.
+      system exit: 0
+    ]
+  )
+
+  printUsage = (
+    './som.sh core-lib/Benchmarks/Harness.som [benchmark] [num-iterations [warm-up [prob-size [num-threads]]]' println.
+    '' println.
+    '  benchmark      - benchmark class name (supports nested classes with ".") ' println.
+    '                   examples: Queens, Fibonacci, Sort.QuickSort, LanguageFeatures.Dispatch' println.
+    '  num-iterations - number of times to execute benchmark, default: 1' println.
+    '  warm-up        - number of times to execute benchmark before measuring, default: 0' println.
+    '  prob-size      - problem size, a benchmark specific value/value range, ' println.
+    '                   which is measured in total, default: 1' println.
+    '  timeout        - maximum time for one iteration in ms, default: 0 = disabled.' println.
+    '  num-threads    - number of threads executing, each doing num-iterations, default: 1' println.
+  )
+
+  loadModule: className = (
+    | file cls |
+    file := className + '.som'.
+    cls := system loadModule: file nextTo: self.
+    cls ifNil: [
+        self error: 'Failed loading benchmark: ' + className ].
+    ^ cls usingPlatform: platform andHarness: self.
+  )
+
+  loadBenchmarkSuite: className = (
+    | file module nested |
+    (className includes: '.')
+      ifFalse: [ ^ self loadModule: className ].
+
+    nested := className split: '.'.
+    module := self loadModule: (nested at: 1).
+    ^ (ObjectMirror reflecting: module) perform: (nested at: 2) asSymbol
+  )
+)