Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Fetching contributors…

Cannot retrieve contributors at this time

320 lines (255 sloc) 8.161 kB
require 'rakelib/git'
# Some tasks to automate running the Ruby Benchmark Suite (RBS)
# on Rubinius. The results are output to YAML and processed by
# the :results task into graphs on a webpage.
#
# The :run task does not depend on the :update tasks so the
# updates are done manually to ensure stability.
BASEDIR = File.expand_path(File.dirname(__FILE__) + "/..")
MONITOR = BASEDIR + "/benchmark/utils/monitor.rb"
RUNNER = BASEDIR + "/benchmark/utils/bench.rb"
RBS_DIR = BASEDIR + "/benchmark/rbs"
RESULTS_BASEDIR = BASEDIR + "/benchmark/results"
WEB_DIR = RESULTS_BASEDIR + "/web"
ITERATIONS = (ENV['ITERATIONS'] || 5).to_i
TIMEOUT = (ENV['TIMEOUT'] || 300).to_i
VM = ENV['VM'] || "#{BASEDIR}/bin/rbx"
ENV['VM'] = VM
BASELINE = ENV['BASELINE'] ? File.expand_path(ENV['BASELINE']) : nil
BASELINE_ID = ENV['BASELINE_ID'] || "no baseline id"
BASELINE_MIN = (ENV['BASELINE_MIN'] || 0).to_i
BASELINE_MAX = (ENV['BASELINE_MAX'] || 7).to_i
GROUP_NAME = ENV['GROUP'] || File.basename(VM.split.first)
RESULTS_DIR = ENV['RESULTS'] || RESULTS_BASEDIR
TEMPLATE = ENV['TEMPLATE'] || RESULTS_BASEDIR + "/templates/basic.erb"
HTMLOUT = ENV['HTMLOUT'] || WEB_DIR + "/index.html"
def command(name)
"ruby #{MONITOR} #{TIMEOUT} '#{VM}' #{RUNNER} #{name} #{ITERATIONS} #{report}"
end
# Cache the name so it is only generated once during an invocation.
# Eliminates having to save the name and pass it around.
def report
@report ||= "#{RESULTS_DIR}/#{GROUP_NAME}-#{Time.now.strftime "%d-%m-%Y-%H%M"}.yaml"
end
def report_name
report[(BASEDIR.size+1)..-1]
end
def field_name
field = ENV['FIELD'] || "min"
unless ["max", "min", "median", "mean"].include?(field)
raise "FIELD must be one of max, min, median, mean"
end
field
end
def parse_date(text)
if /-(\d+)-(\d+)-(\d+)-(\d+).yaml/ =~ text
Time.local($3, $2, $1).to_i * 1000
else
t = Time.now
Time.local(t.year, t.mon, t.day).to_i * 1000
end
end
def group_id(name)
/([^\d]+)-\d+/ =~ File.basename(name, ".*").split("/").last
$1
end
class Graph
class Point
attr_accessor :x, :y, :baseline
def initialize(x, y=0.0)
@x = x
@y = y
@valid = true
@baseline = 0.0
end
def valid?
@valid
end
def invalid
@valid = false
end
def data(delta=false)
return "null" unless valid?
(delta ? [x, @baseline.to_f / y] : [x, y]).inspect
end
end
class Line
attr_reader :label
def initialize(label)
/bm_([^.]*)\.rb-(\d+)$/ =~ label
input = $2
name = $1.gsub(/_/, " ")
@label = "#{name} (#{input})"
@points = Hash.new { |h,k| h[k] = Point.new(k) }
end
def set_point(x, y)
@points[x].y = y
end
def set_baseline(baseline)
@points.each_value { |p| p.baseline = baseline }
end
def invalid(date)
@points[date].invalid
end
def data(delta=false)
points = @points.values.sort { |a, b| a.x <=> b.x }
str = points.map { |p| p.data(delta) }.join(",")
%[{ label: "#{label}", data: [#{str}] }]
end
end
class LineSet
attr_reader :lines
def initialize(set_name)
@set_name = set_name
@lines = Hash.new { |h,k| h[k] = Line.new(k) }
end
def invalid(date)
@lines.each_value { |line| line.invalid date }
end
def data(delta=false)
@lines.values.map { |line| line.data(delta) }.join(", ")
end
end
attr_reader :id, :title, :linesets
def initialize(id)
@id = id.gsub(/\W/, "_")
@title = id.gsub(/\W/, " ")
@linesets = Hash.new { |h,k| h[k] = LineSet.new(k) }
end
def data(delta=false)
@linesets.values.map { |lineset| lineset.data(delta) }.join(", ")
end
end
class GraphEnvironment
attr_accessor :graphs, :field, :min_date, :max_date, :width, :height,
:baseline_id, :baseline_min, :baseline_max
def initialize
@min_date = Time.now.to_i * 1000
@max_date = 0
@width = 800
@height = 400
end
def get_binding
binding
end
end
desc "Run the RBS benchmarks"
task :bench => 'bench:run'
namespace :bench do
desc "Plot the RBS benchmark results"
task :results => :setup do
require 'yaml'
env = GraphEnvironment.new
env.field = field_name
env.graphs = Hash.new { |h,k| h[k] = Graph.new(k) }
Dir[RESULTS_DIR + "/**/*.yaml"].sort.each do |name|
next if BASELINE == name
graph = env.graphs[group_id(name)]
date = parse_date name
env.min_date = date if date < env.min_date
env.max_date = date if date > env.max_date
File.open name, "r" do |file|
YAML.load_documents file do |doc|
lineset = graph.linesets[doc["name"]]
if doc.key? "status" and doc["status"] != "success"
lineset.invalid date
end
next unless doc.key? env.field
line = lineset.lines["#{doc["name"]}-#{doc["parameter"]}"]
line.set_point date, doc[env.field]
end
end
end
if BASELINE
env.baseline_id = BASELINE_ID
env.baseline_min = BASELINE_MIN
env.baseline_max = BASELINE_MAX
File.open BASELINE, "r" do |file|
YAML.load_documents file do |doc|
next unless doc.key? env.field
env.graphs.each_value do |graph|
lineset = graph.linesets[doc["name"]]
line = lineset.lines["#{doc["name"]}-#{doc["parameter"]}"]
line.set_baseline doc[env.field]
end
end
end
end
require 'erb'
rhtml = ERB.new(IO.read(TEMPLATE))
File.open HTMLOUT, "w" do |f|
f.puts rhtml.result(env.get_binding)
end
end
desc "Generate a CSV file of results"
task :to_csv => :setup do
require 'yaml'
field = field_name
header = ["Benchmark file", "input"]
data = Hash.new { |h,k| h[k] = {} }
status = Hash.new { |h,k| h[k] = {} }
Dir[RESULTS_DIR + "/**/*.yaml"].sort.each do |name|
system = File.basename name, ".yaml"
header << system
File.open name, "r" do |file|
YAML.load_documents file do |doc|
bench_name = File.basename(doc["name"], ".rb")
status[bench_name][system] ||= doc["status"]
next unless doc.key? field
bench = [bench_name, doc["parameter"]]
data[bench]["input"] = doc["parameter"]
data[bench][system] = doc[field]
end
end
end
csv_report = "#{RESULTS_DIR}/report-#{Time.now.strftime "%d-%m-%Y-%H%M"}.csv"
File.open csv_report, "w" do |file|
file.puts(header.map { |h| h.inspect }.join(","))
header.shift
data.keys.sort.each do |key|
file.print key.first.inspect, ","
line = header.map do |h|
(data[key][h] || status[key.first][h]).inspect
end
file.puts line.join(",")
end
end
end
# Not public. Creates directories for results, etc.
task :setup do
mkdir_p RESULTS_DIR, :verbose => $verbose
mkdir_p WEB_DIR, :verbose => $verbose
end
task :run => :setup do
puts "Running Ruby Benchmark Suite"
puts " Writing report to #{report_name}"
Dir[RBS_DIR + "/**/bm_*.rb"].sort.each do |name|
Dir.chdir File.dirname(name) do
puts " Running #{File.basename name}"
system "#{command name}"
end
end
puts "Done"
end
desc "Run all the benchmarks in DIR"
task :dir => :setup do
dir = ENV['DIR'] || raise("bench:dir needs DIR to be a directory")
puts "Running all benchmarks in #{dir}"
puts " Writing report to #{report_name}"
Dir[dir + "/**/bm_*.rb"].sort.each do |name|
Dir.chdir File.dirname(name) do
puts " Running #{File.basename name}"
system "#{command name}"
end
end
puts "Done"
end
desc "Run only the benchmark specified by FILE"
task :file => :setup do
name = ENV['FILE'] || raise("bench:file needs FILE to be a filename")
Dir.chdir File.dirname(name) do
system "#{command name}"
end
puts "Writing report to #{report_name}"
end
end
Jump to Line
Something went wrong with that request. Please try again.