Branch the rubinius benchmarking tools into their own project

.autotest

@@ -0,0 +1,23 @@
+# -*- ruby -*-
+
+require 'autotest/restart'
+
+# Autotest.add_hook :initialize do |at|
+#   at.extra_files << "../some/external/dependency.rb"
+#
+#   at.libs << ":../some/external"
+#
+#   at.add_exception 'vendor'
+#
+#   at.add_mapping(/dependency.rb/) do |f, _|
+#     at.files_matching(/test_.*rb$/)
+#   end
+#
+#   %w(TestA TestB).each do |klass|
+#     at.extra_class_map[klass] = "test/test_misc.rb"
+#   end
+# end
+
+# Autotest.add_hook :run_command do |at|
+#   system "rake build"
+# end

History.txt

@@ -0,0 +1,6 @@
+=== 1.0.0 / 2011-04-05
+
+* 1 major enhancement
+
+  * Birthday!
+

Manifest.txt

@@ -0,0 +1,13 @@
+.autotest
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+bin/benchmark
+lib/benchmark/compare.rb
+lib/benchmark/ips.rb
+lib/benchmark/suite-run.rb
+lib/benchmark/suite.rb
+lib/benchmark/timing.rb
+lib/benchmark_suite.rb
+test/test_benchmark_suite.rb

README.txt

@@ -0,0 +1,84 @@
+= benchmark_suite
+
+* http://github.com/evanphx/benchmark_suite
+
+== DESCRIPTION:
+
+A set of enhancements to the standard library benchmark.rb 
+
+== FEATURES/PROBLEMS:
+
+ * benchmark/ips - benchmarks a blocks iterations/second. For short snippits
+   of code, ips automatically figures out how many times to run the code
+   to get interesting data. No more guessing at random iteration counts!
+
+ * benchmark - a cli tool for running multiple benchmarks against
+   multiple rubies.
+
+
+== SYNOPSIS:
+
+require 'benchmark/ips'
+
+Benchmark.ips do
+  # Typical mode, runs the block as many times as it can
+  x.report("addition") { 1 + 2 }
+
+  # To reduce overhead, the number of iterations is passed in
+  # and the block must run the code the specific number of times.
+  # Used for when the workload is very small and any overhead
+  # introduces incorrectable errors.
+  x.report("addition2") do |times|
+    i = 0
+    while i < times
+      1 + 2
+    end
+  end
+
+  # To reduce overhead even more, grafts the code given into
+  # the loop that performs the iterations internally to reduce
+  # overhead. Typically not needed, use the |times| form instead.
+  x.report("addition3", "1 + 2")
+end
+
+== REQUIREMENTS:
+
+* ruby-ffi on MRI
+
+== INSTALL:
+
+* gem install benchmark-suite
+
+== DEVELOPERS:
+
+After checking out the source, run:
+
+  $ rake newb
+
+This task will install any missing dependencies, run the tests/specs,
+and generate the RDoc.
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2011 Evan Phoenix
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Rakefile

@@ -0,0 +1,17 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'hoe'
+
+# Hoe.plugin :compiler
+# Hoe.plugin :gem_prelude_sucks
+# Hoe.plugin :inline
+# Hoe.plugin :inline
+# Hoe.plugin :racc
+# Hoe.plugin :rubyforge
+
+Hoe.spec 'benchmark_suite' do
+  developer('Evan Phoenix', 'evan@fallingsnow.net')
+end
+
+# vim: syntax=ruby

bin/benchmark

@@ -0,0 +1,177 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'tempfile'
+
+lib_path = File.expand_path("../../lib", __FILE__)
+
+$:.unshift lib_path
+
+require 'benchmark/suite'
+require 'benchmark/ips'
+
+targets = []
+extra = []
+at_end = false
+save = false
+
+compare_targets = []
+
+opt = OptionParser.new do |o|
+  o.on("-t", "--target TARGET", String,
+          "Use TARGET to compare against: r:ruby|r19:ruby19|x:rbx|j:jruby") do |t|
+    case t
+    when 'r', 'ruby'
+      targets << 'ruby'
+    when 'r19', 'ruby19'
+      targets << 'ruby19'
+    when 'x', 'rbx', 'rubinius'
+      targets << 'rbx'
+    when 'j', 'jruby'
+      targets << 'jruby'
+    else
+      # + disambiguates execution vs using a file
+      if t[0,1] == "+"
+        targets << t[1..-1]
+      elsif File.exists?(t)
+        begin
+          data = Marshal.load(File.read(t))
+          compare_targets << [t, data]
+        rescue TypeError
+          targets << t
+        end
+      else
+        targets << t
+      end
+    end
+  end
+
+  o.on("-p", "--profile", "Profile code while benchmarking (rbx only)") do
+    extra << "-Xprofile"
+  end
+
+  o.on("-e", "--end", "Report all stats after all suitse have run") do
+    at_end = true
+  end
+
+  o.on("-T OPTION", String, "Add more arguments to each target") do |t|
+    extra << t
+  end
+
+  o.on("-s", "--save PATH", String, "Save the results to a file") do |t|
+    save = t
+  end
+end
+
+opt.parse!
+
+if targets.empty?
+  targets << "ruby"
+end
+
+if save and targets.size > 1
+  STDOUT.puts "Save mode only available with one target."
+  exit 1
+end
+
+opts = []
+
+if at_end
+  opts << "--quiet"
+end
+
+results = targets.map do |t|
+  tf = Tempfile.new "benchmark"
+  tf.close
+  STDOUT.puts "=== #{t} ===" unless at_end
+
+  args = extra + ["-I#{lib_path}", "#{lib_path}/benchmark/suite-run.rb"]
+
+  args += opts
+  args << tf.path
+  args += ARGV
+
+  cmd, *rest = t.split(/\s+/)
+  args.unshift *rest
+
+  system cmd, *args
+
+  if $?.exitstatus != 0
+    puts "Error executing: #{cmd}"
+    nil
+  else
+    tf.open
+    [t, Marshal.load(tf.read)]
+  end
+end
+
+results.compact!
+
+if save
+  name, data = results.last
+
+  File.open save, "w" do |f|
+    f << Marshal.dump(data)
+  end
+
+  STDOUT.puts "[Saved results to '#{save}']"
+end
+
+if at_end
+  results.each do |name, suite|
+    STDOUT.puts "=== #{name} ==="
+    suite.display
+  end
+end
+
+results += compare_targets
+
+if results.size > 1
+  compared = Hash.new { |h,k| h[k] = [] }
+
+  results.each do |target, suite|
+    suite.reports.each do |name, reports|
+      reports.each do |rep|
+        compared["#{name}:#{rep.label}"] << [target, rep]
+      end
+    end
+  end
+
+  STDOUT.puts
+
+  compared.each do |name, reports|
+    if reports.size > 1
+      STDOUT.puts "Comparing #{name}:"
+
+      iter = false
+      sorted = reports.sort do |a,b|
+        if a[1].respond_to? :ips
+          iter = true
+          b[1].ips <=> a[1].ips
+        else
+          a[1].runtime <=> b[1].runtime
+        end
+      end
+
+      best_name, best_report = sorted.shift
+
+
+      if iter
+        STDOUT.printf "%20s: %10d i/s\n", best_name, best_report.ips
+      else
+        STDOUT.puts "#{best_name.rjust(20)}: #{best_report.runtime}s"
+      end
+
+      sorted.each do |entry|
+        name, report = entry
+        if iter
+          x = (best_report.ips.to_f / report.ips.to_f)
+          STDOUT.printf "%20s: %10d i/s - %.2fx slower\n", name, report.ips, x
+        else
+          x = "%.2f" % (report.ips.to_f / best_report.ips.to_f)
+          STDOUT.puts "#{name.rjust(20)}: #{report.runtime}s - #{x}x slower"
+        end
+      end
+    end
+  end
+end

lib/benchmark/compare.rb

@@ -0,0 +1,41 @@
+module Benchmark
+  def compare(*reports)
+    return if reports.size < 2
+
+    iter = false
+    sorted = reports.sort do |a,b|
+      if a.respond_to? :ips
+        iter = true
+        b.ips <=> a.ips
+      else
+        a.runtime <=> b.runtime
+      end
+    end
+
+    best = sorted.shift
+
+    STDOUT.puts "\nComparison:"
+
+    if iter
+      STDOUT.printf "%20s: %10.1f i/s\n", best.label, best.ips
+    else
+      STDOUT.puts "#{best.rjust(20)}: #{best.runtime}s"
+    end
+
+    sorted.each do |report|
+      name = report.label
+
+      if iter
+        x = (best.ips.to_f / report.ips.to_f)
+        STDOUT.printf "%20s: %10.1f i/s - %.2fx slower\n", name, report.ips, x
+      else
+        x = "%.2f" % (report.ips.to_f / best.ips.to_f)
+        STDOUT.puts "#{name.rjust(20)}: #{report.runtime}s - #{x}x slower"
+      end
+    end
+
+    STDOUT.puts
+  end
+
+  module_function :compare
+end