Skip to content
Fetching contributors…
Cannot retrieve contributors at this time
executable file 387 lines (300 sloc) 8.46 KB
#!/usr/bin/env ruby
# Must use our benchmark.rb, which has bug fixes -vs- Ruby 1.8.6 pl 111
require File.expand_path(File.join(File.dirname(__FILE__),'../lib/benchmark'))
include Benchmark
# Options handling
require 'optparse'
require 'ostruct'
# Benchmark test selector
# BM_GLOB = 'test/benchmark/*/bm*.rb'
BM_GLOB = File.expand_path(File.join(File.dirname(__FILE__),'../benchmark/*/bm*.rb'))
# Which Ruby runtimes to test: MRI should be first and Rubinius should be last
$interpreters = ['ruby','bin/rbx']
# WIDTH of test labels, required for nice formating
WIDTH = 50
REPORT_WIDTH = WIDTH - 6
COLUMN_WIDTH = 12
DECIMAL_FORMAT = '%4.4f'
now = Time.now
NOW = "#{now.year}-#{'%02d' % now.month}-#{'%02d' % now.day}-" +
"#{'%02d' % now.hour}-#{'%02d' % now.min}-#{'%02d' % now.sec}"
OPTIONS = OpenStruct.new( :size => 1024,
:path => './' )
options = OptionParser.new do |opts|
opts.on('-c', '--chart [FILENAME]', 'Output passing test times to SVG file at --path') do |c|
OPTIONS.chart = c || "#{NOW}-times.svg"
end
opts.on('-d', '--diff-chart [FILENAME]', 'Output speed differences to SVG file at --path') do |d|
OPTIONS.diffchart = d || "#{NOW}-difference.svg"
end
opts.on('-f', '--file [FILENAME]', 'Output report to text file at --path, instead of STDOUT') do |f|
OPTIONS.file = f || "#{NOW}-report.txt"
end
opts.on('-g', '--grep REGEX', 'Only run tests that match REGEX') do |g|
OPTIONS.grep = g
end
opts.on("-t", "--target TARGET", String,
"Use TARGET to compare against: r:ruby|r19:ruby19|x:rbx|j:jruby") do |t|
case t
when 'r', 'ruby'
target = 'ruby'
when 'r19', 'ruby19'
target = 'ruby19'
when 'x', 'rbx', 'rubinius'
target = 'bin/rbx'
when 'j', 'jruby'
target = 'jruby'
else
target = t
end
$interpreters.insert(1,target)
end
opts.on('-j', '--just', 'Benchmark just the target machine') do |j|
$interpreters.delete_at(0)
end
opts.on('-h', '--help', 'Output this help information') do |h|
OPTIONS.help = h
end
opts.on('-n', '--ngrep REGEX', 'Only run tests that do not match REGEX') do |n|
OPTIONS.ngrep = n
end
opts.on('-p', '--path PATH', 'Path prefix for --chart, --diff-chart, and --file') do |p|
OPTIONS.path = p
end
opts.on('-s', '--size PIXELS', 'Size, in pixels, of --chart and/or --diff-chart') do |s|
OPTIONS.size = s
end
opts.on('-w', '--warmup', 'Run each test twice and throw away the first result') do |w|
OPTIONS.warmup = w
end
end
options.parse!
if OPTIONS.help then
puts options.summarize
exit
end
class RbxBenchmarkFile
def initialize(interpreter,file)
@interpreter = interpreter
@file = file
end
def pretty_interpreter
@interpreter.match(%r{ruby19|rbx|ruby|jruby})[0]
end
def pretty_file
@file.match(%r{[^/]+/[^/]+$})[0]
end
def label
pretty_interpreter + ': ' + pretty_file
end
def command
command = "#{@interpreter} #{@file}"
case @file
when /borasky\/bm_MatrixBenchmark[.]rb/
command << %{ 64}
end
command
end
def quiet_command
command + ' > /dev/null 2>&1'
end
def run
spaces = WIDTH - label.length
spaces = spaces < 0 ? 0 : spaces
@result = if OPTIONS.warmup then
Benchmark.bmbm do |x|
x.report(label + ' ' * spaces) do
if system quiet_command then
@pass = true
else
@pass = false
system command
end
end
end.first
else
Benchmark.bm do |x|
x.report(label + ' ' * spaces) do
if system quiet_command then
@pass = true
else
@pass = false
system command
end
end
end
end
end
def passed?
@pass
end
def failed?
! @pass
end
def time
passed? ? @result.real : 0
end
def skip
@skipped = true
end
def skipped?
@skipped
end
def unchartable?
failed? or skipped?
end
def report
if skipped? then
'Skipped'
elsif passed? then
DECIMAL_FORMAT % time
else
'Failed'
end.rjust(COLUMN_WIDTH)
end
end
class RbxBenchmarkRun
attr_reader :interpreter
attr_reader :results
def initialize(interpreter)
@interpreter = interpreter
@results = []
@total = Benchmark::Tms.new
end
def clean
if @interpreter =~ /rubinius/ then
Dir[BM_GLOB + 'c'].each do |file|
File.unlink file
end
end
end
def run
clean
Dir[BM_GLOB].each do |file|
@results << RbxBenchmarkFile.new(@interpreter,file)
if OPTIONS.grep and ! file.match(OPTIONS.grep)
@results.last.skip
next
end
if OPTIONS.ngrep and file.match(OPTIONS.ngrep) then
@results.last.skip
next
end
@results.last.run
end
self
end
def time
@results.inject(0) { |total, result| total + result.time }
end
def chart_points
@results.collect do |result|
if result.unchartable?
nil
else
result.time
end
end
end
end
def difference(ruby,rubinius)
if ruby.unchartable? or rubinius.unchartable? then
nil
elsif ruby.time < rubinius.time then
-rubinius.time / ruby.time
else
ruby.time / rubinius.time
end
end
def report(runs)
header = 'file'.ljust(REPORT_WIDTH)
runs.each do |run|
header << (run.results.first.pretty_interpreter).rjust(COLUMN_WIDTH)
end
header << 'difference'.rjust(COLUMN_WIDTH)
file = nil
if OPTIONS.file
file = File.open(OPTIONS.path + OPTIONS.file,'w')
else
file = STDOUT
end
file.puts header
file.puts '-' * header.length
# Columnar result output
0.upto(runs.first.results.length - 1) do |i|
line = runs.first.results[i].pretty_file.ljust(REPORT_WIDTH)
results = []
runs.each do |run|
results << run.results[i]
line << run.results[i].report
end
diff = difference(results.first,results.last)
line << if diff.nil? then
'n/a'
else
DECIMAL_FORMAT % diff
end.rjust(COLUMN_WIDTH)
file.puts line
end
file.close unless file == STDOUT
end
runs = []
$interpreters.each do |i|
runs << RbxBenchmarkRun.new(i).run
end
report(runs)
if OPTIONS.chart or OPTIONS.diffchart then
require 'rubygems'
require 'scruffy'
end
if OPTIONS.chart then
chart_point_sets = runs.collect { |run| run.chart_points }
# Make a list of failed tests
deletes = []
for i in 0..chart_point_sets.length-1
delete = false
for j in 0..chart_point_sets.first.length-1
if chart_point_sets[i][j].nil? then
deletes << j
end
end
end
# Remove failed tests from each runtime
chart_point_sets.each do |cps|
deletes.uniq.sort.reverse.each { |d| cps.delete_at(d) }
end
graph = Scruffy::Graph.new
runs.each do |run|
graph << Scruffy::Layers::Line.new( :title => run.interpreter,
:points => chart_point_sets.shift )
end
# graph.point_markers = runs.first.results.reject do |result|
# result.unchartable?
# end.collect do |result|
# result.pretty_file
# end
puts "Rendering chart to #{OPTIONS.chart}"
graph.render( :width => OPTIONS.size, :to => OPTIONS.path + OPTIONS.chart )
end
if OPTIONS.diffchart then
differences = []
0.upto(runs.first.results.length - 1) do |i|
differences << difference(runs.first.results[i],runs.last.results[i])
end
# Remove where tests failed, easier this time, only a single list
differences.reject! { |d| d.nil? }
# Convert to percentages
# differences.collect { |d| d * 100 }
graph = Scruffy::Graph.new
graph << Scruffy::Layers::Line.new( :title => 'Zero',
:points => differences.collect { |d| 0 } )
graph << Scruffy::Layers::Line.new( :title => 'Rubinius times faster (-times slower)',
:points => differences )
# graph.point_markers = runs.first.results.reject do |result|
# result.unchartable?
# end.collect do |result|
# result.pretty_file
# end
puts "Rendering diffchart to #{OPTIONS.diffchart}"
graph.render( :width => OPTIONS.size, :to => OPTIONS.path + OPTIONS.diffchart )
end
Jump to Line
Something went wrong with that request. Please try again.