Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 28 additions & 4 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,27 @@ jobs:
if: ${{ github.event_name != 'schedule' || github.repository == 'ruby/ruby-bench' }}
steps:
- uses: actions/checkout@v3
- name: Cache apt packages
uses: actions/cache@v4
with:
path: |
/var/cache/apt/archives
/var/lib/apt/lists
key: ${{ runner.os }}-apt-imagemagick-${{ hashFiles('.github/workflows/test.yml') }}
restore-keys: |
${{ runner.os }}-apt-imagemagick-
- name: Install ImageMagick dependencies
run: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends libmagickwand-dev
- name: Set up Ruby
uses: ruby/setup-ruby@v1
with:
ruby-version: ${{ matrix.ruby }}

- name: Run tests
run: rake test
continue-on-error: ${{ matrix.ruby == 'truffleruby' }}

benchmark-default:
runs-on: ubuntu-latest
Expand Down Expand Up @@ -95,16 +109,26 @@ jobs:
if: ${{ github.event_name != 'schedule' || github.repository == 'ruby/ruby-bench' }}
steps:
- uses: actions/checkout@v3
- name: Cache apt packages
uses: actions/cache@v4
with:
path: |
/var/cache/apt/archives
/var/lib/apt/lists
key: ${{ runner.os }}-apt-imagemagick-${{ hashFiles('.github/workflows/test.yml') }}
restore-keys: |
${{ runner.os }}-apt-imagemagick-
- name: Install ImageMagick dependencies
run: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends libmagickwand-dev
- name: Set up Ruby
uses: ruby/setup-ruby@v1
with:
ruby-version: ruby

- name: Test run_benchmarks.rb --graph
run: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends libmagickwand-dev
./run_benchmarks.rb --graph fib
run: ./run_benchmarks.rb --graph fib
env:
WARMUP_ITRS: '1'
MIN_BENCH_ITRS: '1'
Expand Down
7 changes: 7 additions & 0 deletions lib/benchmark_runner.rb
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,13 @@ def free_file_no(directory)
end
end

# Render a graph from JSON benchmark data
def render_graph(json_path)
png_path = json_path.sub(/\.json$/, '.png')
require_relative 'graph_renderer'
GraphRenderer.render(json_path, png_path)
end

# Checked system - error or return info if the command fails
def check_call(command, env: {}, raise_error: true, quiet: false)
puts("+ #{command}") unless quiet
Expand Down
73 changes: 73 additions & 0 deletions lib/graph_renderer.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
# frozen_string_literal: true

require_relative '../misc/stats'
require 'json'
begin
require 'gruff'
rescue LoadError
Gem.install('gruff')
gem 'gruff'
require 'gruff'
end

# Renders benchmark data as a graph
class GraphRenderer
DEFAULT_WIDTH = 1600
COLOR_PALETTE = %w[#3285e1 #489d32 #e2c13e #8A6EAF #D1695E].freeze
THEME = {
colors: COLOR_PALETTE,
marker_color: '#dddddd',
font_color: 'black',
background_colors: 'white'
}.freeze
DEFAULT_BOTTOM_MARGIN = 30.0
DEFAULT_LEGEND_MARGIN = 4.0

class << self
def render(json_path, png_path, title_font_size: 16.0, legend_font_size: 12.0, marker_font_size: 10.0)
ruby_descriptions, data, baseline, bench_names = load_benchmark_data(json_path)

graph = Gruff::Bar.new(DEFAULT_WIDTH)
configure_graph(graph, ruby_descriptions, bench_names, title_font_size, legend_font_size, marker_font_size)

ruby_descriptions.each do |ruby, description|
speedups = calculate_speedups(data, baseline, ruby, bench_names)
graph.data "#{ruby}: #{description}", speedups
end
graph.write(png_path)
png_path
end

private

def load_benchmark_data(json_path)
json = JSON.load_file(json_path)
ruby_descriptions = json.fetch("metadata")
data = json.fetch("raw_data")
baseline = ruby_descriptions.first.first
bench_names = data.first.last.keys

[ruby_descriptions, data, baseline, bench_names]
end

def configure_graph(graph, ruby_descriptions, bench_names, title_font_size, legend_font_size, marker_font_size)
graph.title = "Speedup ratio relative to #{ruby_descriptions.keys.first}"
graph.title_font_size = title_font_size
graph.theme = THEME
graph.labels = bench_names.map.with_index { |bench, index| [index, bench] }.to_h
graph.show_labels_for_bar_values = true
graph.bottom_margin = DEFAULT_BOTTOM_MARGIN
graph.legend_margin = DEFAULT_LEGEND_MARGIN
graph.legend_font_size = legend_font_size
graph.marker_font_size = marker_font_size
end

def calculate_speedups(data, baseline, ruby, bench_names)
bench_names.map { |bench|
baseline_times = data.fetch(baseline).fetch(bench).fetch("bench")
times = data.fetch(ruby).fetch(bench).fetch("bench")
Stats.new(baseline_times).mean / Stats.new(times).mean
}
end
end
end
51 changes: 4 additions & 47 deletions misc/graph.rb
Original file line number Diff line number Diff line change
@@ -1,52 +1,9 @@
#!/usr/bin/env ruby

require_relative 'stats'
require 'json'
begin
require 'gruff'
rescue LoadError
Gem.install('gruff')
gem 'gruff'
require 'gruff'
end

def render_graph(json_path, png_path, title_font_size: 16.0, legend_font_size: 12.0, marker_font_size: 10.0)
json = JSON.load_file(json_path)
ruby_descriptions = json.fetch("metadata")
data = json.fetch("raw_data")
baseline = ruby_descriptions.first.first
bench_names = data.first.last.keys

# ruby_descriptions, bench_names, table
g = Gruff::Bar.new(1600)
g.title = "Speedup ratio relative to #{ruby_descriptions.keys.first}"
g.title_font_size = title_font_size
g.theme = {
colors: %w[#3285e1 #489d32 #e2c13e #8A6EAF #D1695E],
marker_color: '#dddddd',
font_color: 'black',
background_colors: 'white'
}
g.labels = bench_names.map.with_index { |bench, index| [index, bench] }.to_h
g.show_labels_for_bar_values = true
g.bottom_margin = 30.0
g.legend_margin = 4.0
g.legend_font_size = legend_font_size
g.marker_font_size = marker_font_size

ruby_descriptions.each do |ruby, description|
speedups = bench_names.map { |bench|
baseline_times = data.fetch(baseline).fetch(bench).fetch("bench")
times = data.fetch(ruby).fetch(bench).fetch("bench")
Stats.new(baseline_times).mean / Stats.new(times).mean
}
g.data "#{ruby}: #{description}", speedups
end
g.write(png_path)
end
require_relative '../lib/graph_renderer'

# This file may be used as a standalone command as well.
if $0 == __FILE__
# Standalone command-line interface for rendering graphs
if __FILE__ == $0
require 'optparse'

args = {}
Expand All @@ -68,7 +25,7 @@ def render_graph(json_path, png_path, title_font_size: 16.0, legend_font_size: 1
abort parser.help if json_path.nil?

png_path = json_path.sub(/\.json\z/, '.png')
render_graph(json_path, png_path, **args)
GraphRenderer.render(json_path, png_path, **args)

open = %w[open xdg-open].find { |open| system("which #{open} >/dev/null 2>/dev/null") }
system(open, png_path) if open
Expand Down
6 changes: 2 additions & 4 deletions run_benchmarks.rb
Original file line number Diff line number Diff line change
Expand Up @@ -126,11 +126,9 @@
puts
puts "Output:"
puts out_json_path

if args.graph
require_relative 'misc/graph'
out_graph_path = output_path + ".png"
render_graph(out_json_path, out_graph_path)
puts out_graph_path
puts BenchmarkRunner.render_graph(out_json_path)
end

if !bench_failures.empty?
Expand Down
20 changes: 20 additions & 0 deletions test/benchmark_runner_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -157,4 +157,24 @@
end
end
end

describe '.render_graph' do
it 'delegates to GraphRenderer and returns calculated png_path' do
Dir.mktmpdir do |dir|
json_path = File.join(dir, 'test.json')
expected_png_path = File.join(dir, 'test.png')

json_data = {
metadata: { 'ruby-a' => 'version A' },
raw_data: { 'ruby-a' => { 'bench1' => { 'bench' => [1.0] } } }
}
File.write(json_path, JSON.generate(json_data))

result = BenchmarkRunner.render_graph(json_path)

assert_equal expected_png_path, result
assert File.exist?(expected_png_path)
end
end
end
end
84 changes: 84 additions & 0 deletions test/graph_renderer_test.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
require_relative 'test_helper'
require_relative '../lib/graph_renderer'
require 'tempfile'
require 'tmpdir'
require 'json'

describe GraphRenderer do
describe '.render' do
it 'creates a PNG file from JSON data' do
Dir.mktmpdir do |dir|
json_path = File.join(dir, 'test.json')
png_path = File.join(dir, 'test.png')

# Create test JSON file with minimal benchmark data
json_data = {
metadata: {
'ruby-a' => 'version A'
},
raw_data: {
'ruby-a' => {
'bench1' => {
'bench' => [1.0, 1.1, 0.9]
}
}
}
}
File.write(json_path, JSON.generate(json_data))

result = GraphRenderer.render(json_path, png_path)

assert_equal png_path, result
assert File.exist?(png_path), 'PNG file should be created'
assert File.size(png_path) > 0, 'PNG file should not be empty'
end
end

it 'returns the png_path' do
Dir.mktmpdir do |dir|
json_path = File.join(dir, 'test.json')
png_path = File.join(dir, 'test.png')

json_data = {
metadata: { 'ruby-a' => 'version A' },
raw_data: { 'ruby-a' => { 'bench1' => { 'bench' => [1.0] } } }
}
File.write(json_path, JSON.generate(json_data))

result = GraphRenderer.render(json_path, png_path)

assert_equal png_path, result
end
end

it 'handles multiple rubies and benchmarks' do
Dir.mktmpdir do |dir|
json_path = File.join(dir, 'test.json')
png_path = File.join(dir, 'test.png')

json_data = {
metadata: {
'ruby-a' => 'version A',
'ruby-b' => 'version B'
},
raw_data: {
'ruby-a' => {
'bench1' => { 'bench' => [1.0, 1.1] },
'bench2' => { 'bench' => [2.0, 2.1] }
},
'ruby-b' => {
'bench1' => { 'bench' => [0.9, 1.0] },
'bench2' => { 'bench' => [1.8, 1.9] }
}
}
}
File.write(json_path, JSON.generate(json_data))

GraphRenderer.render(json_path, png_path)

assert File.exist?(png_path)
assert File.size(png_path) > 0
end
end
end
end
Loading