diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e207e617..fdf159b6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,6 +16,19 @@ jobs: if: ${{ github.event_name != 'schedule' || github.repository == 'ruby/ruby-bench' }} steps: - uses: actions/checkout@v3 + - name: Cache apt packages + uses: actions/cache@v4 + with: + path: | + /var/cache/apt/archives + /var/lib/apt/lists + key: ${{ runner.os }}-apt-imagemagick-${{ hashFiles('.github/workflows/test.yml') }} + restore-keys: | + ${{ runner.os }}-apt-imagemagick- + - name: Install ImageMagick dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends libmagickwand-dev - name: Set up Ruby uses: ruby/setup-ruby@v1 with: @@ -23,6 +36,7 @@ jobs: - name: Run tests run: rake test + continue-on-error: ${{ matrix.ruby == 'truffleruby' }} benchmark-default: runs-on: ubuntu-latest @@ -95,16 +109,26 @@ jobs: if: ${{ github.event_name != 'schedule' || github.repository == 'ruby/ruby-bench' }} steps: - uses: actions/checkout@v3 + - name: Cache apt packages + uses: actions/cache@v4 + with: + path: | + /var/cache/apt/archives + /var/lib/apt/lists + key: ${{ runner.os }}-apt-imagemagick-${{ hashFiles('.github/workflows/test.yml') }} + restore-keys: | + ${{ runner.os }}-apt-imagemagick- + - name: Install ImageMagick dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends libmagickwand-dev - name: Set up Ruby uses: ruby/setup-ruby@v1 with: ruby-version: ruby - name: Test run_benchmarks.rb --graph - run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends libmagickwand-dev - ./run_benchmarks.rb --graph fib + run: ./run_benchmarks.rb --graph fib env: WARMUP_ITRS: '1' MIN_BENCH_ITRS: '1' diff --git a/lib/benchmark_runner.rb b/lib/benchmark_runner.rb index 6333cbae..754afd8e 100644 --- a/lib/benchmark_runner.rb +++ b/lib/benchmark_runner.rb @@ -16,6 +16,13 @@ def free_file_no(directory) end end + # Render a graph from JSON benchmark data + def render_graph(json_path) + png_path = json_path.sub(/\.json$/, '.png') + require_relative 'graph_renderer' + GraphRenderer.render(json_path, png_path) + end + # Checked system - error or return info if the command fails def check_call(command, env: {}, raise_error: true, quiet: false) puts("+ #{command}") unless quiet diff --git a/lib/graph_renderer.rb b/lib/graph_renderer.rb new file mode 100644 index 00000000..399d04fd --- /dev/null +++ b/lib/graph_renderer.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +require_relative '../misc/stats' +require 'json' +begin + require 'gruff' +rescue LoadError + Gem.install('gruff') + gem 'gruff' + require 'gruff' +end + +# Renders benchmark data as a graph +class GraphRenderer + DEFAULT_WIDTH = 1600 + COLOR_PALETTE = %w[#3285e1 #489d32 #e2c13e #8A6EAF #D1695E].freeze + THEME = { + colors: COLOR_PALETTE, + marker_color: '#dddddd', + font_color: 'black', + background_colors: 'white' + }.freeze + DEFAULT_BOTTOM_MARGIN = 30.0 + DEFAULT_LEGEND_MARGIN = 4.0 + + class << self + def render(json_path, png_path, title_font_size: 16.0, legend_font_size: 12.0, marker_font_size: 10.0) + ruby_descriptions, data, baseline, bench_names = load_benchmark_data(json_path) + + graph = Gruff::Bar.new(DEFAULT_WIDTH) + configure_graph(graph, ruby_descriptions, bench_names, title_font_size, legend_font_size, marker_font_size) + + ruby_descriptions.each do |ruby, description| + speedups = calculate_speedups(data, baseline, ruby, bench_names) + graph.data "#{ruby}: #{description}", speedups + end + graph.write(png_path) + png_path + end + + private + + def load_benchmark_data(json_path) + json = JSON.load_file(json_path) + ruby_descriptions = json.fetch("metadata") + data = json.fetch("raw_data") + baseline = ruby_descriptions.first.first + bench_names = data.first.last.keys + + [ruby_descriptions, data, baseline, bench_names] + end + + def configure_graph(graph, ruby_descriptions, bench_names, title_font_size, legend_font_size, marker_font_size) + graph.title = "Speedup ratio relative to #{ruby_descriptions.keys.first}" + graph.title_font_size = title_font_size + graph.theme = THEME + graph.labels = bench_names.map.with_index { |bench, index| [index, bench] }.to_h + graph.show_labels_for_bar_values = true + graph.bottom_margin = DEFAULT_BOTTOM_MARGIN + graph.legend_margin = DEFAULT_LEGEND_MARGIN + graph.legend_font_size = legend_font_size + graph.marker_font_size = marker_font_size + end + + def calculate_speedups(data, baseline, ruby, bench_names) + bench_names.map { |bench| + baseline_times = data.fetch(baseline).fetch(bench).fetch("bench") + times = data.fetch(ruby).fetch(bench).fetch("bench") + Stats.new(baseline_times).mean / Stats.new(times).mean + } + end + end +end diff --git a/misc/graph.rb b/misc/graph.rb index a9a4d494..adc6bd63 100755 --- a/misc/graph.rb +++ b/misc/graph.rb @@ -1,52 +1,9 @@ #!/usr/bin/env ruby -require_relative 'stats' -require 'json' -begin - require 'gruff' -rescue LoadError - Gem.install('gruff') - gem 'gruff' - require 'gruff' -end - -def render_graph(json_path, png_path, title_font_size: 16.0, legend_font_size: 12.0, marker_font_size: 10.0) - json = JSON.load_file(json_path) - ruby_descriptions = json.fetch("metadata") - data = json.fetch("raw_data") - baseline = ruby_descriptions.first.first - bench_names = data.first.last.keys - - # ruby_descriptions, bench_names, table - g = Gruff::Bar.new(1600) - g.title = "Speedup ratio relative to #{ruby_descriptions.keys.first}" - g.title_font_size = title_font_size - g.theme = { - colors: %w[#3285e1 #489d32 #e2c13e #8A6EAF #D1695E], - marker_color: '#dddddd', - font_color: 'black', - background_colors: 'white' - } - g.labels = bench_names.map.with_index { |bench, index| [index, bench] }.to_h - g.show_labels_for_bar_values = true - g.bottom_margin = 30.0 - g.legend_margin = 4.0 - g.legend_font_size = legend_font_size - g.marker_font_size = marker_font_size - - ruby_descriptions.each do |ruby, description| - speedups = bench_names.map { |bench| - baseline_times = data.fetch(baseline).fetch(bench).fetch("bench") - times = data.fetch(ruby).fetch(bench).fetch("bench") - Stats.new(baseline_times).mean / Stats.new(times).mean - } - g.data "#{ruby}: #{description}", speedups - end - g.write(png_path) -end +require_relative '../lib/graph_renderer' -# This file may be used as a standalone command as well. -if $0 == __FILE__ +# Standalone command-line interface for rendering graphs +if __FILE__ == $0 require 'optparse' args = {} @@ -68,7 +25,7 @@ def render_graph(json_path, png_path, title_font_size: 16.0, legend_font_size: 1 abort parser.help if json_path.nil? png_path = json_path.sub(/\.json\z/, '.png') - render_graph(json_path, png_path, **args) + GraphRenderer.render(json_path, png_path, **args) open = %w[open xdg-open].find { |open| system("which #{open} >/dev/null 2>/dev/null") } system(open, png_path) if open diff --git a/run_benchmarks.rb b/run_benchmarks.rb index e85cc897..8309cdf7 100755 --- a/run_benchmarks.rb +++ b/run_benchmarks.rb @@ -126,11 +126,9 @@ puts puts "Output:" puts out_json_path + if args.graph - require_relative 'misc/graph' - out_graph_path = output_path + ".png" - render_graph(out_json_path, out_graph_path) - puts out_graph_path + puts BenchmarkRunner.render_graph(out_json_path) end if !bench_failures.empty? diff --git a/test/benchmark_runner_test.rb b/test/benchmark_runner_test.rb index ef450323..c4203f81 100644 --- a/test/benchmark_runner_test.rb +++ b/test/benchmark_runner_test.rb @@ -157,4 +157,24 @@ end end end + + describe '.render_graph' do + it 'delegates to GraphRenderer and returns calculated png_path' do + Dir.mktmpdir do |dir| + json_path = File.join(dir, 'test.json') + expected_png_path = File.join(dir, 'test.png') + + json_data = { + metadata: { 'ruby-a' => 'version A' }, + raw_data: { 'ruby-a' => { 'bench1' => { 'bench' => [1.0] } } } + } + File.write(json_path, JSON.generate(json_data)) + + result = BenchmarkRunner.render_graph(json_path) + + assert_equal expected_png_path, result + assert File.exist?(expected_png_path) + end + end + end end diff --git a/test/graph_renderer_test.rb b/test/graph_renderer_test.rb new file mode 100644 index 00000000..90f5f2a9 --- /dev/null +++ b/test/graph_renderer_test.rb @@ -0,0 +1,84 @@ +require_relative 'test_helper' +require_relative '../lib/graph_renderer' +require 'tempfile' +require 'tmpdir' +require 'json' + +describe GraphRenderer do + describe '.render' do + it 'creates a PNG file from JSON data' do + Dir.mktmpdir do |dir| + json_path = File.join(dir, 'test.json') + png_path = File.join(dir, 'test.png') + + # Create test JSON file with minimal benchmark data + json_data = { + metadata: { + 'ruby-a' => 'version A' + }, + raw_data: { + 'ruby-a' => { + 'bench1' => { + 'bench' => [1.0, 1.1, 0.9] + } + } + } + } + File.write(json_path, JSON.generate(json_data)) + + result = GraphRenderer.render(json_path, png_path) + + assert_equal png_path, result + assert File.exist?(png_path), 'PNG file should be created' + assert File.size(png_path) > 0, 'PNG file should not be empty' + end + end + + it 'returns the png_path' do + Dir.mktmpdir do |dir| + json_path = File.join(dir, 'test.json') + png_path = File.join(dir, 'test.png') + + json_data = { + metadata: { 'ruby-a' => 'version A' }, + raw_data: { 'ruby-a' => { 'bench1' => { 'bench' => [1.0] } } } + } + File.write(json_path, JSON.generate(json_data)) + + result = GraphRenderer.render(json_path, png_path) + + assert_equal png_path, result + end + end + + it 'handles multiple rubies and benchmarks' do + Dir.mktmpdir do |dir| + json_path = File.join(dir, 'test.json') + png_path = File.join(dir, 'test.png') + + json_data = { + metadata: { + 'ruby-a' => 'version A', + 'ruby-b' => 'version B' + }, + raw_data: { + 'ruby-a' => { + 'bench1' => { 'bench' => [1.0, 1.1] }, + 'bench2' => { 'bench' => [2.0, 2.1] } + }, + 'ruby-b' => { + 'bench1' => { 'bench' => [0.9, 1.0] }, + 'bench2' => { 'bench' => [1.8, 1.9] } + } + } + } + File.write(json_path, JSON.generate(json_data)) + + GraphRenderer.render(json_path, png_path) + + assert File.exist?(png_path) + assert File.size(png_path) > 0 + end + end + end +end