Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Fetching contributors…

Cannot retrieve contributors at this time

320 lines (255 sloc) 8.161 kb
require 'rakelib/git'
# Some tasks to automate running the Ruby Benchmark Suite (RBS)
# on Rubinius. The results are output to YAML and processed by
# the :results task into graphs on a webpage.
#
# The :run task does not depend on the :update tasks so the
# updates are done manually to ensure stability.
BASEDIR = File.expand_path(File.dirname(__FILE__) + "/..")
MONITOR = BASEDIR + "/benchmark/utils/monitor.rb"
RUNNER = BASEDIR + "/benchmark/utils/bench.rb"
RBS_DIR = BASEDIR + "/benchmark/rbs"
RESULTS_BASEDIR = BASEDIR + "/benchmark/results"
WEB_DIR = RESULTS_BASEDIR + "/web"
ITERATIONS = (ENV['ITERATIONS'] || 5).to_i
TIMEOUT = (ENV['TIMEOUT'] || 300).to_i
VM = ENV['VM'] || "#{BASEDIR}/bin/rbx"
ENV['VM'] = VM
BASELINE = ENV['BASELINE'] ? File.expand_path(ENV['BASELINE']) : nil
BASELINE_ID = ENV['BASELINE_ID'] || "no baseline id"
BASELINE_MIN = (ENV['BASELINE_MIN'] || 0).to_i
BASELINE_MAX = (ENV['BASELINE_MAX'] || 7).to_i
GROUP_NAME = ENV['GROUP'] || File.basename(VM.split.first)
RESULTS_DIR = ENV['RESULTS'] || RESULTS_BASEDIR
TEMPLATE = ENV['TEMPLATE'] || RESULTS_BASEDIR + "/templates/basic.erb"
HTMLOUT = ENV['HTMLOUT'] || WEB_DIR + "/index.html"
def command(name)
"ruby #{MONITOR} #{TIMEOUT} '#{VM}' #{RUNNER} #{name} #{ITERATIONS} #{report}"
end
# Cache the name so it is only generated once during an invocation.
# Eliminates having to save the name and pass it around.
def report
@report ||= "#{RESULTS_DIR}/#{GROUP_NAME}-#{Time.now.strftime "%d-%m-%Y-%H%M"}.yaml"
end
def report_name
report[(BASEDIR.size+1)..-1]
end
def field_name
field = ENV['FIELD'] || "min"
unless ["max", "min", "median", "mean"].include?(field)
raise "FIELD must be one of max, min, median, mean"
end
field
end
def parse_date(text)
if /-(\d+)-(\d+)-(\d+)-(\d+).yaml/ =~ text
Time.local($3, $2, $1).to_i * 1000
else
t = Time.now
Time.local(t.year, t.mon, t.day).to_i * 1000
end
end
def group_id(name)
/([^\d]+)-\d+/ =~ File.basename(name, ".*").split("/").last
$1
end
class Graph
class Point
attr_accessor :x, :y, :baseline
def initialize(x, y=0.0)
@x = x
@y = y
@valid = true
@baseline = 0.0
end
def valid?
@valid
end
def invalid
@valid = false
end
def data(delta=false)
return "null" unless valid?
(delta ? [x, @baseline.to_f / y] : [x, y]).inspect
end
end
class Line
attr_reader :label
def initialize(label)
/bm_([^.]*)\.rb-(\d+)$/ =~ label
input = $2
name = $1.gsub(/_/, " ")
@label = "#{name} (#{input})"
@points = Hash.new { |h,k| h[k] = Point.new(k) }
end
def set_point(x, y)
@points[x].y = y
end
def set_baseline(baseline)
@points.each_value { |p| p.baseline = baseline }
end
def invalid(date)
@points[date].invalid
end
def data(delta=false)
points = @points.values.sort { |a, b| a.x <=> b.x }
str = points.map { |p| p.data(delta) }.join(",")
%[{ label: "#{label}", data: [#{str}] }]
end
end
class LineSet
attr_reader :lines
def initialize(set_name)
@set_name = set_name
@lines = Hash.new { |h,k| h[k] = Line.new(k) }
end
def invalid(date)
@lines.each_value { |line| line.invalid date }
end
def data(delta=false)
@lines.values.map { |line| line.data(delta) }.join(", ")
end
end
attr_reader :id, :title, :linesets
def initialize(id)
@id = id.gsub(/\W/, "_")
@title = id.gsub(/\W/, " ")
@linesets = Hash.new { |h,k| h[k] = LineSet.new(k) }
end
def data(delta=false)
@linesets.values.map { |lineset| lineset.data(delta) }.join(", ")
end
end
class GraphEnvironment
attr_accessor :graphs, :field, :min_date, :max_date, :width, :height,
:baseline_id, :baseline_min, :baseline_max
def initialize
@min_date = Time.now.to_i * 1000
@max_date = 0
@width = 800
@height = 400
end
def get_binding
binding
end
end
desc "Run the RBS benchmarks"
task :bench => 'bench:run'
namespace :bench do
desc "Plot the RBS benchmark results"
task :results => :setup do
require 'yaml'
env = GraphEnvironment.new
env.field = field_name
env.graphs = Hash.new { |h,k| h[k] = Graph.new(k) }
Dir[RESULTS_DIR + "/**/*.yaml"].sort.each do |name|
next if BASELINE == name
graph = env.graphs[group_id(name)]
date = parse_date name
env.min_date = date if date < env.min_date
env.max_date = date if date > env.max_date
File.open name, "r" do |file|
YAML.load_documents file do |doc|
lineset = graph.linesets[doc["name"]]
if doc.key? "status" and doc["status"] != "success"
lineset.invalid date
end
next unless doc.key? env.field
line = lineset.lines["#{doc["name"]}-#{doc["parameter"]}"]
line.set_point date, doc[env.field]
end
end
end
if BASELINE
env.baseline_id = BASELINE_ID
env.baseline_min = BASELINE_MIN
env.baseline_max = BASELINE_MAX
File.open BASELINE, "r" do |file|
YAML.load_documents file do |doc|
next unless doc.key? env.field
env.graphs.each_value do |graph|
lineset = graph.linesets[doc["name"]]
line = lineset.lines["#{doc["name"]}-#{doc["parameter"]}"]
line.set_baseline doc[env.field]
end
end
end
end
require 'erb'
rhtml = ERB.new(IO.read(TEMPLATE))
File.open HTMLOUT, "w" do |f|
f.puts rhtml.result(env.get_binding)
end
end
desc "Generate a CSV file of results"
task :to_csv => :setup do
require 'yaml'
field = field_name
header = ["Benchmark file", "input"]
data = Hash.new { |h,k| h[k] = {} }
status = Hash.new { |h,k| h[k] = {} }
Dir[RESULTS_DIR + "/**/*.yaml"].sort.each do |name|
system = File.basename name, ".yaml"
header << system
File.open name, "r" do |file|
YAML.load_documents file do |doc|
bench_name = File.basename(doc["name"], ".rb")
status[bench_name][system] ||= doc["status"]
next unless doc.key? field
bench = [bench_name, doc["parameter"]]
data[bench]["input"] = doc["parameter"]
data[bench][system] = doc[field]
end
end
end
csv_report = "#{RESULTS_DIR}/report-#{Time.now.strftime "%d-%m-%Y-%H%M"}.csv"
File.open csv_report, "w" do |file|
file.puts(header.map { |h| h.inspect }.join(","))
header.shift
data.keys.sort.each do |key|
file.print key.first.inspect, ","
line = header.map do |h|
(data[key][h] || status[key.first][h]).inspect
end
file.puts line.join(",")
end
end
end
# Not public. Creates directories for results, etc.
task :setup do
mkdir_p RESULTS_DIR, :verbose => $verbose
mkdir_p WEB_DIR, :verbose => $verbose
end
task :run => :setup do
puts "Running Ruby Benchmark Suite"
puts " Writing report to #{report_name}"
Dir[RBS_DIR + "/**/bm_*.rb"].sort.each do |name|
Dir.chdir File.dirname(name) do
puts " Running #{File.basename name}"
system "#{command name}"
end
end
puts "Done"
end
desc "Run all the benchmarks in DIR"
task :dir => :setup do
dir = ENV['DIR'] || raise("bench:dir needs DIR to be a directory")
puts "Running all benchmarks in #{dir}"
puts " Writing report to #{report_name}"
Dir[dir + "/**/bm_*.rb"].sort.each do |name|
Dir.chdir File.dirname(name) do
puts " Running #{File.basename name}"
system "#{command name}"
end
end
puts "Done"
end
desc "Run only the benchmark specified by FILE"
task :file => :setup do
name = ENV['FILE'] || raise("bench:file needs FILE to be a filename")
Dir.chdir File.dirname(name) do
system "#{command name}"
end
puts "Writing report to #{report_name}"
end
end
Jump to Line
Something went wrong with that request. Please try again.