Skip to content
This repository
Fetching contributors…

Octocat-spinner-32-eaf2f5

Cannot retrieve contributors at this time

file 320 lines (255 sloc) 8.161 kb
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319
require 'rakelib/git'

# Some tasks to automate running the Ruby Benchmark Suite (RBS)
# on Rubinius. The results are output to YAML and processed by
# the :results task into graphs on a webpage.
#
# The :run task does not depend on the :update tasks so the
# updates are done manually to ensure stability.

BASEDIR = File.expand_path(File.dirname(__FILE__) + "/..")
MONITOR = BASEDIR + "/benchmark/utils/monitor.rb"
RUNNER = BASEDIR + "/benchmark/utils/bench.rb"
RBS_DIR = BASEDIR + "/benchmark/rbs"
RESULTS_BASEDIR = BASEDIR + "/benchmark/results"
WEB_DIR = RESULTS_BASEDIR + "/web"

ITERATIONS = (ENV['ITERATIONS'] || 5).to_i
TIMEOUT = (ENV['TIMEOUT'] || 300).to_i
VM = ENV['VM'] || "#{BASEDIR}/bin/rbx"
ENV['VM'] = VM

BASELINE = ENV['BASELINE'] ? File.expand_path(ENV['BASELINE']) : nil
BASELINE_ID = ENV['BASELINE_ID'] || "no baseline id"
BASELINE_MIN = (ENV['BASELINE_MIN'] || 0).to_i
BASELINE_MAX = (ENV['BASELINE_MAX'] || 7).to_i
GROUP_NAME = ENV['GROUP'] || File.basename(VM.split.first)
RESULTS_DIR = ENV['RESULTS'] || RESULTS_BASEDIR
TEMPLATE = ENV['TEMPLATE'] || RESULTS_BASEDIR + "/templates/basic.erb"
HTMLOUT = ENV['HTMLOUT'] || WEB_DIR + "/index.html"


def command(name)
  "ruby #{MONITOR} #{TIMEOUT} '#{VM}' #{RUNNER} #{name} #{ITERATIONS} #{report}"
end

# Cache the name so it is only generated once during an invocation.
# Eliminates having to save the name and pass it around.
def report
  @report ||= "#{RESULTS_DIR}/#{GROUP_NAME}-#{Time.now.strftime "%d-%m-%Y-%H%M"}.yaml"
end

def report_name
  report[(BASEDIR.size+1)..-1]
end

def field_name
  field = ENV['FIELD'] || "min"
  unless ["max", "min", "median", "mean"].include?(field)
    raise "FIELD must be one of max, min, median, mean"
  end
  field
end

def parse_date(text)
  if /-(\d+)-(\d+)-(\d+)-(\d+).yaml/ =~ text
    Time.local($3, $2, $1).to_i * 1000
  else
    t = Time.now
    Time.local(t.year, t.mon, t.day).to_i * 1000
  end
end

def group_id(name)
  /([^\d]+)-\d+/ =~ File.basename(name, ".*").split("/").last
  $1
end

class Graph
  class Point
    attr_accessor :x, :y, :baseline

    def initialize(x, y=0.0)
      @x = x
      @y = y
      @valid = true
      @baseline = 0.0
    end

    def valid?
      @valid
    end

    def invalid
      @valid = false
    end

    def data(delta=false)
      return "null" unless valid?
      (delta ? [x, @baseline.to_f / y] : [x, y]).inspect
    end
  end

  class Line
    attr_reader :label

    def initialize(label)
      /bm_([^.]*)\.rb-(\d+)$/ =~ label
      input = $2
      name = $1.gsub(/_/, " ")
      @label = "#{name} (#{input})"
      @points = Hash.new { |h,k| h[k] = Point.new(k) }
    end

    def set_point(x, y)
      @points[x].y = y
    end

    def set_baseline(baseline)
      @points.each_value { |p| p.baseline = baseline }
    end

    def invalid(date)
      @points[date].invalid
    end

    def data(delta=false)
      points = @points.values.sort { |a, b| a.x <=> b.x }
      str = points.map { |p| p.data(delta) }.join(",")
      %[{ label: "#{label}", data: [#{str}] }]
    end
  end

  class LineSet
    attr_reader :lines

    def initialize(set_name)
      @set_name = set_name
      @lines = Hash.new { |h,k| h[k] = Line.new(k) }
    end

    def invalid(date)
      @lines.each_value { |line| line.invalid date }
    end

    def data(delta=false)
      @lines.values.map { |line| line.data(delta) }.join(", ")
    end
  end

  attr_reader :id, :title, :linesets

  def initialize(id)
    @id = id.gsub(/\W/, "_")
    @title = id.gsub(/\W/, " ")
    @linesets = Hash.new { |h,k| h[k] = LineSet.new(k) }
  end

  def data(delta=false)
    @linesets.values.map { |lineset| lineset.data(delta) }.join(", ")
  end
end

class GraphEnvironment
  attr_accessor :graphs, :field, :min_date, :max_date, :width, :height,
                :baseline_id, :baseline_min, :baseline_max

  def initialize
    @min_date = Time.now.to_i * 1000
    @max_date = 0
    @width = 800
    @height = 400
  end

  def get_binding
    binding
  end
end

desc "Run the RBS benchmarks"
task :bench => 'bench:run'

namespace :bench do
  desc "Plot the RBS benchmark results"
  task :results => :setup do
    require 'yaml'

    env = GraphEnvironment.new
    env.field = field_name
    env.graphs = Hash.new { |h,k| h[k] = Graph.new(k) }

    Dir[RESULTS_DIR + "/**/*.yaml"].sort.each do |name|
      next if BASELINE == name

      graph = env.graphs[group_id(name)]
      date = parse_date name

      env.min_date = date if date < env.min_date
      env.max_date = date if date > env.max_date

      File.open name, "r" do |file|
        YAML.load_documents file do |doc|
          lineset = graph.linesets[doc["name"]]

          if doc.key? "status" and doc["status"] != "success"
            lineset.invalid date
          end

          next unless doc.key? env.field

          line = lineset.lines["#{doc["name"]}-#{doc["parameter"]}"]
          line.set_point date, doc[env.field]
        end
      end
    end

    if BASELINE
      env.baseline_id = BASELINE_ID
      env.baseline_min = BASELINE_MIN
      env.baseline_max = BASELINE_MAX

      File.open BASELINE, "r" do |file|
        YAML.load_documents file do |doc|
          next unless doc.key? env.field

          env.graphs.each_value do |graph|
            lineset = graph.linesets[doc["name"]]
            line = lineset.lines["#{doc["name"]}-#{doc["parameter"]}"]
            line.set_baseline doc[env.field]
          end
        end
      end
    end

    require 'erb'
    rhtml = ERB.new(IO.read(TEMPLATE))

    File.open HTMLOUT, "w" do |f|
      f.puts rhtml.result(env.get_binding)
    end
  end

  desc "Generate a CSV file of results"
  task :to_csv => :setup do
    require 'yaml'

    field = field_name
    header = ["Benchmark file", "input"]
    data = Hash.new { |h,k| h[k] = {} }
    status = Hash.new { |h,k| h[k] = {} }

    Dir[RESULTS_DIR + "/**/*.yaml"].sort.each do |name|
      system = File.basename name, ".yaml"
      header << system

      File.open name, "r" do |file|
        YAML.load_documents file do |doc|
          bench_name = File.basename(doc["name"], ".rb")
          status[bench_name][system] ||= doc["status"]

          next unless doc.key? field

          bench = [bench_name, doc["parameter"]]
          data[bench]["input"] = doc["parameter"]
          data[bench][system] = doc[field]
        end
      end
    end

    csv_report = "#{RESULTS_DIR}/report-#{Time.now.strftime "%d-%m-%Y-%H%M"}.csv"
    File.open csv_report, "w" do |file|
      file.puts(header.map { |h| h.inspect }.join(","))
      header.shift

      data.keys.sort.each do |key|
        file.print key.first.inspect, ","
        line = header.map do |h|
          (data[key][h] || status[key.first][h]).inspect
        end
        file.puts line.join(",")
      end
    end
  end

  # Not public. Creates directories for results, etc.
  task :setup do
    mkdir_p RESULTS_DIR, :verbose => $verbose
    mkdir_p WEB_DIR, :verbose => $verbose
  end

  task :run => :setup do
    puts "Running Ruby Benchmark Suite"
    puts " Writing report to #{report_name}"

    Dir[RBS_DIR + "/**/bm_*.rb"].sort.each do |name|
      Dir.chdir File.dirname(name) do
        puts " Running #{File.basename name}"
        system "#{command name}"
      end
    end

    puts "Done"
  end

  desc "Run all the benchmarks in DIR"
  task :dir => :setup do
    dir = ENV['DIR'] || raise("bench:dir needs DIR to be a directory")

    puts "Running all benchmarks in #{dir}"
    puts " Writing report to #{report_name}"

    Dir[dir + "/**/bm_*.rb"].sort.each do |name|
      Dir.chdir File.dirname(name) do
        puts " Running #{File.basename name}"
        system "#{command name}"
      end
    end

    puts "Done"
  end

  desc "Run only the benchmark specified by FILE"
  task :file => :setup do
    name = ENV['FILE'] || raise("bench:file needs FILE to be a filename")
    Dir.chdir File.dirname(name) do
      system "#{command name}"
    end
    puts "Writing report to #{report_name}"
  end
end
Something went wrong with that request. Please try again.