From 71cdca079d25bf6daac9eb4430f74c80e7d88de9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Mendon=C3=A7a=20Fran=C3=A7a?= Date: Tue, 18 Nov 2025 22:38:58 +0000 Subject: [PATCH] Build BenchmarkSuite only once per CLI run Only the execution of the benchmarks depends on the Ruby executable being used. --- lib/benchmark_runner/cli.rb | 22 ++++++------ lib/benchmark_suite.rb | 52 ++++++++++++++-------------- test/benchmark_suite_test.rb | 66 ++++++------------------------------ 3 files changed, 48 insertions(+), 92 deletions(-) diff --git a/lib/benchmark_runner/cli.rb b/lib/benchmark_runner/cli.rb index ab0d0692..92206a48 100644 --- a/lib/benchmark_runner/cli.rb +++ b/lib/benchmark_runner/cli.rb @@ -28,6 +28,16 @@ def run ruby_descriptions = {} + suite = BenchmarkSuite.new( + categories: args.categories, + name_filters: args.name_filters, + excludes: args.excludes, + out_path: args.out_path, + harness: args.harness, + pre_init: args.with_pre_init, + no_pinning: args.no_pinning + ) + # Benchmark with and without YJIT bench_start_time = Time.now.to_f bench_data = {} @@ -35,18 +45,10 @@ def run args.executables.each do |name, executable| ruby_descriptions[name] = `#{executable.shelljoin} -v`.chomp - suite = BenchmarkSuite.new( + bench_data[name], failures = suite.run( ruby: executable, - ruby_description: ruby_descriptions[name], - categories: args.categories, - name_filters: args.name_filters, - excludes: args.excludes, - out_path: args.out_path, - harness: args.harness, - pre_init: args.with_pre_init, - no_pinning: args.no_pinning + ruby_description: ruby_descriptions[name] ) - bench_data[name], failures = suite.run # Make it easier to query later. bench_failures[name] = failures unless failures.empty? end diff --git a/lib/benchmark_suite.rb b/lib/benchmark_suite.rb index 4143e109..30c4a853 100644 --- a/lib/benchmark_suite.rb +++ b/lib/benchmark_suite.rb @@ -19,11 +19,9 @@ class BenchmarkSuite RACTOR_CATEGORY = ["ractor"].freeze RACTOR_HARNESS = "harness-ractor" - attr_reader :ruby, :ruby_description, :categories, :name_filters, :excludes, :out_path, :harness, :pre_init, :no_pinning, :bench_dir, :ractor_bench_dir + attr_reader :categories, :name_filters, :excludes, :out_path, :harness, :pre_init, :no_pinning, :bench_dir, :ractor_bench_dir - def initialize(ruby:, ruby_description:, categories:, name_filters:, excludes: [], out_path:, harness:, pre_init: nil, no_pinning: false) - @ruby = ruby - @ruby_description = ruby_description + def initialize(categories:, name_filters:, excludes: [], out_path:, harness:, pre_init: nil, no_pinning: false) @categories = categories @name_filters = name_filters @excludes = excludes @@ -38,17 +36,19 @@ def initialize(ruby:, ruby_description:, categories:, name_filters:, excludes: [ # Run all the benchmarks and record execution times # Returns [bench_data, bench_failures] - def run + def run(ruby:, ruby_description:) bench_data = {} bench_failures = {} benchmark_entries = discover_benchmarks + cmd_prefix = base_cmd(ruby_description) + env = benchmark_env(ruby) benchmark_entries.each_with_index do |entry, idx| puts("Running benchmark \"#{entry.name}\" (#{idx+1}/#{benchmark_entries.length})") result_json_path = File.join(out_path, "temp#{Process.pid}.json") - result = run_single_benchmark(entry.script_path, result_json_path) + result = run_single_benchmark(entry.script_path, result_json_path, ruby, cmd_prefix, env) if result[:success] bench_data[entry.name] = process_benchmark_result(result_json_path, result[:command]) @@ -142,7 +142,7 @@ def filter_entries(entries, categories:, name_filters:, excludes:, directory_map entries.select { |entry| filter.match?(entry.name) } end - def run_single_benchmark(script_path, result_json_path) + def run_single_benchmark(script_path, result_json_path, ruby, cmd_prefix, env) # Fix for jruby/jruby#7394 in JRuby 9.4.2.0 script_path = File.expand_path(script_path) @@ -150,7 +150,7 @@ def run_single_benchmark(script_path, result_json_path) ENV["RESULT_JSON_PATH"] = result_json_path # Set up the benchmarking command - cmd = base_cmd + [ + cmd = cmd_prefix + [ *ruby, "-I", harness, *pre_init, @@ -158,31 +158,29 @@ def run_single_benchmark(script_path, result_json_path) ].compact # Do the benchmarking - result = BenchmarkRunner.check_call(cmd.shelljoin, env: benchmark_env, raise_error: false) + result = BenchmarkRunner.check_call(cmd.shelljoin, env: env, raise_error: false) result[:command] = cmd.shelljoin result end - def benchmark_env - @benchmark_env ||= begin - # When the Ruby running this script is not the first Ruby in PATH, shell commands - # like `bundle install` in a child process will not use the Ruby being benchmarked. - # It overrides PATH to guarantee the commands of the benchmarked Ruby will be used. - env = {} - ruby_path = `#{ruby.shelljoin} -e 'print RbConfig.ruby' 2> #{File::NULL}` + def benchmark_env(ruby) + # When the Ruby running this script is not the first Ruby in PATH, shell commands + # like `bundle install` in a child process will not use the Ruby being benchmarked. + # It overrides PATH to guarantee the commands of the benchmarked Ruby will be used. + env = {} + ruby_path = `#{ruby.shelljoin} -e 'print RbConfig.ruby' 2> #{File::NULL}` - if ruby_path != RbConfig.ruby - env["PATH"] = "#{File.dirname(ruby_path)}:#{ENV["PATH"]}" + if ruby_path != RbConfig.ruby + env["PATH"] = "#{File.dirname(ruby_path)}:#{ENV["PATH"]}" - # chruby sets GEM_HOME and GEM_PATH in your shell. We have to unset it in the child - # process to avoid installing gems to the version that is running run_benchmarks.rb. - ["GEM_HOME", "GEM_PATH"].each do |var| - env[var] = nil if ENV.key?(var) - end + # chruby sets GEM_HOME and GEM_PATH in your shell. We have to unset it in the child + # process to avoid installing gems to the version that is running run_benchmarks.rb. + ["GEM_HOME", "GEM_PATH"].each do |var| + env[var] = nil if ENV.key?(var) end - - env end + + env end def benchmarks_metadata @@ -199,8 +197,8 @@ def linux? end # Set up the base command with CPU pinning if needed - def base_cmd - @base_cmd ||= if linux? + def base_cmd(ruby_description) + if linux? cmd = setarch_prefix # Pin the process to one given core to improve caching and reduce variance on CRuby diff --git a/test/benchmark_suite_test.rb b/test/benchmark_suite_test.rb index ce953b91..cd9eee3b 100644 --- a/test/benchmark_suite_test.rb +++ b/test/benchmark_suite_test.rb @@ -47,16 +47,12 @@ describe '#initialize' do it 'sets all required attributes' do suite = BenchmarkSuite.new( - ruby: ['ruby'], - ruby_description: 'ruby 3.2.0', categories: ['micro'], name_filters: [], out_path: @out_path, harness: 'harness' ) - assert_equal ['ruby'], suite.ruby - assert_equal 'ruby 3.2.0', suite.ruby_description assert_equal ['micro'], suite.categories assert_equal [], suite.name_filters assert_equal @out_path, suite.out_path @@ -67,8 +63,6 @@ it 'accepts optional parameters' do suite = BenchmarkSuite.new( - ruby: ['ruby'], - ruby_description: 'ruby 3.2.0', categories: [], name_filters: [], out_path: @out_path, @@ -81,8 +75,6 @@ it 'sets bench_dir to BENCHMARKS_DIR by default' do suite = BenchmarkSuite.new( - ruby: ['ruby'], - ruby_description: 'ruby 3.2.0', categories: ['micro'], name_filters: [], out_path: @out_path, @@ -97,8 +89,6 @@ it 'sets bench_dir to ractor directory and updates harness when ractor-only category is used' do suite = BenchmarkSuite.new( - ruby: ['ruby'], - ruby_description: 'ruby 3.2.0', categories: ['ractor-only'], name_filters: [], out_path: @out_path, @@ -113,8 +103,6 @@ it 'keeps bench_dir as BENCHMARKS_DIR when ractor category is used' do suite = BenchmarkSuite.new( - ruby: ['ruby'], - ruby_description: 'ruby 3.2.0', categories: ['ractor'], name_filters: [], out_path: @out_path, @@ -131,8 +119,6 @@ describe '#run' do it 'returns bench_data and bench_failures as a tuple' do suite = BenchmarkSuite.new( - ruby: [RbConfig.ruby], - ruby_description: 'ruby 3.2.0', categories: [], name_filters: ['simple'], out_path: @out_path, @@ -142,7 +128,7 @@ result = nil capture_io do - result = suite.run + result = suite.run(ruby: [RbConfig.ruby], ruby_description: 'ruby 3.2.0') end assert_instance_of Array, result @@ -155,8 +141,6 @@ it 'runs matching benchmarks and collects results' do suite = BenchmarkSuite.new( - ruby: [RbConfig.ruby], - ruby_description: 'ruby 3.2.0', categories: [], name_filters: ['simple'], out_path: @out_path, @@ -166,7 +150,7 @@ bench_data, bench_failures = nil capture_io do - bench_data, bench_failures = suite.run + bench_data, bench_failures = suite.run(ruby: [RbConfig.ruby], ruby_description: 'ruby 3.2.0') end assert_includes bench_data, 'simple' @@ -180,8 +164,6 @@ it 'prints progress messages while running' do suite = BenchmarkSuite.new( - ruby: [RbConfig.ruby], - ruby_description: 'ruby 3.2.0', categories: [], name_filters: ['simple'], out_path: @out_path, @@ -190,7 +172,7 @@ ) output = capture_io do - suite.run + suite.run(ruby: [RbConfig.ruby], ruby_description: 'ruby 3.2.0') end assert_includes output[0], 'Running benchmark "simple"' @@ -203,8 +185,6 @@ RUBY suite = BenchmarkSuite.new( - ruby: [RbConfig.ruby], - ruby_description: 'ruby 3.2.0', categories: [], name_filters: ['failing'], out_path: @out_path, @@ -214,7 +194,7 @@ bench_data, bench_failures = nil capture_io do - bench_data, bench_failures = suite.run + bench_data, bench_failures = suite.run(ruby: [RbConfig.ruby], ruby_description: 'ruby 3.2.0') end assert_empty bench_data @@ -236,8 +216,6 @@ RUBY suite = BenchmarkSuite.new( - ruby: [RbConfig.ruby], - ruby_description: 'ruby 3.2.0', categories: [], name_filters: ['subdir'], out_path: @out_path, @@ -247,7 +225,7 @@ bench_data, bench_failures = nil capture_io do - bench_data, bench_failures = suite.run + bench_data, bench_failures = suite.run(ruby: [RbConfig.ruby], ruby_description: 'ruby 3.2.0') end assert_includes bench_data, 'subdir' @@ -267,8 +245,6 @@ RUBY suite = BenchmarkSuite.new( - ruby: [RbConfig.ruby], - ruby_description: 'ruby 3.2.0', categories: ['ractor-only'], name_filters: [], out_path: @out_path, @@ -278,7 +254,7 @@ bench_data, bench_failures = nil capture_io do - bench_data, bench_failures = suite.run + bench_data, bench_failures = suite.run(ruby: [RbConfig.ruby], ruby_description: 'ruby 3.2.0') end # When ractor-only is specified, it should use benchmarks-ractor directory @@ -301,8 +277,6 @@ RUBY suite = BenchmarkSuite.new( - ruby: [RbConfig.ruby], - ruby_description: 'ruby 3.2.0', categories: ['ractor'], name_filters: [], out_path: @out_path, @@ -312,7 +286,7 @@ bench_data = nil capture_io do - bench_data, _ = suite.run + bench_data, _ = suite.run(ruby: [RbConfig.ruby], ruby_description: 'ruby 3.2.0') end # With ractor category, both directories should be scanned @@ -326,8 +300,6 @@ File.write(pre_init_file, "# Pre-initialization code\n") suite = BenchmarkSuite.new( - ruby: [RbConfig.ruby], - ruby_description: 'ruby 3.2.0', categories: [], name_filters: ['simple'], out_path: @out_path, @@ -350,8 +322,6 @@ File.write(pre_init_file, "# Config code\n") suite = BenchmarkSuite.new( - ruby: [RbConfig.ruby], - ruby_description: 'ruby 3.2.0', categories: [], name_filters: ['simple'], out_path: @out_path, @@ -372,8 +342,6 @@ File.write(pre_init_file, "# Setup code\n") suite = BenchmarkSuite.new( - ruby: [RbConfig.ruby], - ruby_description: 'ruby 3.2.0', categories: [], name_filters: ['simple'], out_path: @out_path, @@ -391,8 +359,6 @@ output = capture_io do assert_raises(SystemExit) do BenchmarkSuite.new( - ruby: [RbConfig.ruby], - ruby_description: 'ruby 3.2.0', categories: [], name_filters: ['simple'], out_path: @out_path, @@ -409,8 +375,6 @@ output = capture_io do assert_raises(SystemExit) do BenchmarkSuite.new( - ruby: [RbConfig.ruby], - ruby_description: 'ruby 3.2.0', categories: [], name_filters: ['simple'], out_path: @out_path, @@ -425,8 +389,6 @@ it 'stores command_line in benchmark results' do suite = BenchmarkSuite.new( - ruby: [RbConfig.ruby], - ruby_description: 'ruby 3.2.0', categories: [], name_filters: ['simple'], out_path: @out_path, @@ -436,7 +398,7 @@ bench_data, _ = nil capture_io do - bench_data, _ = suite.run + bench_data, _ = suite.run(ruby: [RbConfig.ruby], ruby_description: 'ruby 3.2.0') end assert_includes bench_data['simple'], 'command_line' @@ -446,8 +408,6 @@ it 'cleans up temporary JSON files after successful run' do suite = BenchmarkSuite.new( - ruby: [RbConfig.ruby], - ruby_description: 'ruby 3.2.0', categories: [], name_filters: ['simple'], out_path: @out_path, @@ -456,7 +416,7 @@ ) capture_io do - suite.run + suite.run(ruby: [RbConfig.ruby], ruby_description: 'ruby 3.2.0') end # Temporary files should be cleaned up @@ -479,8 +439,6 @@ RUBY suite = BenchmarkSuite.new( - ruby: [RbConfig.ruby], - ruby_description: 'ruby 3.2.0', categories: [], name_filters: ['bench_a'], out_path: @out_path, @@ -490,7 +448,7 @@ bench_data = nil capture_io do - bench_data, _ = suite.run + bench_data, _ = suite.run(ruby: [RbConfig.ruby], ruby_description: 'ruby 3.2.0') end assert_includes bench_data, 'bench_a' @@ -514,8 +472,6 @@ File.write('benchmarks.yml', YAML.dump(metadata)) suite = BenchmarkSuite.new( - ruby: [RbConfig.ruby], - ruby_description: 'ruby 3.2.0', categories: ['micro'], name_filters: [], out_path: @out_path, @@ -525,7 +481,7 @@ bench_data = nil capture_io do - bench_data, _ = suite.run + bench_data, _ = suite.run(ruby: [RbConfig.ruby], ruby_description: 'ruby 3.2.0') end # Should only include micro category benchmarks