Skip to content
111 changes: 88 additions & 23 deletions lib/benchmark_runner.rb
Original file line number Diff line number Diff line change
Expand Up @@ -3,40 +3,105 @@
require 'csv'
require 'json'
require 'rbconfig'
require_relative 'table_formatter'

# Extracted helper methods from run_benchmarks.rb for testing
module BenchmarkRunner
module_function
class << self
# Determine output path - either use the override or find a free file number
def output_path(out_path_dir, out_override: nil)
if out_override
out_override
else
# If no out path is specified, find a free file index for the output files
file_no = free_file_no(out_path_dir)
File.join(out_path_dir, "output_%03d" % file_no)
end
end

# Find the first available file number for output files
def free_file_no(directory)
(1..).each do |file_no|
out_path = File.join(directory, "output_%03d.csv" % file_no)
return file_no unless File.exist?(out_path)
# Write benchmark data to JSON file
def write_json(output_path, ruby_descriptions, bench_data)
out_json_path = "#{output_path}.json"
out_data = {
metadata: ruby_descriptions,
raw_data: bench_data,
}
File.write(out_json_path, JSON.generate(out_data))
out_json_path
end
end

# Render a graph from JSON benchmark data
def render_graph(json_path)
png_path = json_path.sub(/\.json$/, '.png')
require_relative 'graph_renderer'
GraphRenderer.render(json_path, png_path)
end
# Write benchmark results to CSV file
def write_csv(output_path, ruby_descriptions, table)
out_csv_path = "#{output_path}.csv"

CSV.open(out_csv_path, "wb") do |csv|
ruby_descriptions.each do |key, value|
csv << [key, value]
end
csv << []
table.each do |row|
csv << row
end
end

out_csv_path
end

# Build output text string with metadata, table, and legend
def build_output_text(ruby_descriptions, table, format, bench_failures)
base_name, *other_names = ruby_descriptions.keys

output_str = +""

ruby_descriptions.each do |key, value|
output_str << "#{key}: #{value}\n"
end

# Checked system - error or return info if the command fails
def check_call(command, env: {}, raise_error: true, quiet: false)
puts("+ #{command}") unless quiet
output_str << "\n"
output_str << TableFormatter.new(table, format, bench_failures).to_s + "\n"

result = {}
unless other_names.empty?
output_str << "Legend:\n"
other_names.each do |name|
output_str << "- #{name} 1st itr: ratio of #{base_name}/#{name} time for the first benchmarking iteration.\n"
output_str << "- #{base_name}/#{name}: ratio of #{base_name}/#{name} time. Higher is better for #{name}. Above 1 represents a speedup.\n"
end
end

result[:success] = system(env, command)
result[:status] = $?
output_str
end

unless result[:success]
puts "Command #{command.inspect} failed with exit code #{result[:status].exitstatus} in directory #{Dir.pwd}"
raise RuntimeError.new if raise_error
# Render a graph from JSON benchmark data
def render_graph(json_path)
png_path = json_path.sub(/\.json$/, '.png')
require_relative 'graph_renderer'
GraphRenderer.render(json_path, png_path)
end

result
# Checked system - error or return info if the command fails
def check_call(command, env: {}, raise_error: true, quiet: false)
puts("+ #{command}") unless quiet

result = {}

result[:success] = system(env, command)
result[:status] = $?

unless result[:success]
puts "Command #{command.inspect} failed with exit code #{result[:status].exitstatus} in directory #{Dir.pwd}"
raise RuntimeError.new if raise_error
end

result
end

private

def free_file_no(directory)
(1..).each do |file_no|
out_path = File.join(directory, "output_%03d.csv" % file_no)
return file_no unless File.exist?(out_path)
end
end
end
end
51 changes: 5 additions & 46 deletions run_benchmarks.rb
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
require_relative 'lib/cpu_config'
require_relative 'lib/benchmark_runner'
require_relative 'lib/benchmark_suite'
require_relative 'lib/table_formatter'
require_relative 'lib/argument_parser'
require_relative 'lib/results_table_builder'

Expand Down Expand Up @@ -57,65 +56,25 @@
puts

# Build results table
all_names = args.executables.keys
base_name, *other_names = all_names
builder = ResultsTableBuilder.new(
executable_names: all_names,
executable_names: ruby_descriptions.keys,
bench_data: bench_data,
include_rss: args.rss
)
table, format = builder.build

output_path = nil
if args.out_override
output_path = args.out_override
else
# If no out path is specified, find a free file index for the output files
file_no = BenchmarkRunner.free_file_no(args.out_path)
output_path = File.join(args.out_path, "output_%03d" % file_no)
end
output_path = BenchmarkRunner.output_path(args.out_path, out_override: args.out_override)

# Save the raw data as JSON
out_json_path = output_path + ".json"
File.open(out_json_path, "w") do |file|
out_data = {
metadata: ruby_descriptions,
raw_data: bench_data,
}
json_str = JSON.generate(out_data)
file.write json_str
end
out_json_path = BenchmarkRunner.write_json(output_path, ruby_descriptions, bench_data)

# Save data as CSV so we can produce tables/graphs in a spreasheet program
# NOTE: we don't do any number formatting for the output file because
# we don't want to lose any precision
output_rows = []
ruby_descriptions.each do |key, value|
output_rows.append([key, value])
end
output_rows.append([])
output_rows.concat(table)
out_tbl_path = output_path + ".csv"
CSV.open(out_tbl_path, "wb") do |csv|
output_rows.each do |row|
csv << row
end
end
BenchmarkRunner.write_csv(output_path, ruby_descriptions, table)

# Save the output in a text file that we can easily refer to
output_str = ""
ruby_descriptions.each do |key, value|
output_str << "#{key}: #{value}\n"
end
output_str += "\n"
output_str += TableFormatter.new(table, format, bench_failures).to_s + "\n"
unless other_names.empty?
output_str << "Legend:\n"
other_names.each do |name|
output_str << "- #{name} 1st itr: ratio of #{base_name}/#{name} time for the first benchmarking iteration.\n"
output_str << "- #{base_name}/#{name}: ratio of #{base_name}/#{name} time. Higher is better for #{name}. Above 1 represents a speedup.\n"
end
end
output_str = BenchmarkRunner.build_output_text(ruby_descriptions, table, format, bench_failures)
out_txt_path = output_path + ".txt"
File.open(out_txt_path, "w") { |f| f.write output_str }

Expand Down
Loading
Loading