From d0dd7a2badebc38bb1ab80579a70e393aa2c1742 Mon Sep 17 00:00:00 2001 From: Randy Stauner Date: Wed, 22 May 2024 17:02:08 -0700 Subject: [PATCH 1/4] Benchmark against CRuby 3.3 to use as a baseline --- basic_benchmark.rb | 11 +++++++++-- lib/yjit-metrics/bench-results.rb | 7 +++++++ .../report_templates/blog_speed_details.html.erb | 4 ++-- .../report_types/bloggable_speed_report.rb | 4 +++- .../report_types/variable_warmup_report.rb | 4 +++- 5 files changed, 24 insertions(+), 6 deletions(-) diff --git a/basic_benchmark.rb b/basic_benchmark.rb index 65f5520fb..15e27b85b 100755 --- a/basic_benchmark.rb +++ b/basic_benchmark.rb @@ -46,6 +46,8 @@ MJIT_PER_OS_OPTS = SETARCH_OPTS TRUFFLE_PER_OS_OPTS = {} +PREV_RUBY_BUILD = "ruby-3.3.1" + # These are "config roots" because they define a configuration # in a non-platform-specific way. They're really several *variables* # that partially define a configuration. @@ -96,8 +98,13 @@ opts: MJIT_ENABLED_OPTS + [ "--mjit-verbose=1" ], per_os_prefix: MJIT_PER_OS_OPTS, }, + "prev_ruby_no_jit" => { + build: PREV_RUBY_BUILD, + opts: NO_JIT_OPTS, + per_os_prefix: CRUBY_PER_OS_OPTS, + }, "prev_ruby_yjit" => { - build: "ruby-3.3.1", + build: PREV_RUBY_BUILD, opts: YJIT_ENABLED_OPTS, per_os_prefix: YJIT_PER_OS_OPTS, }, @@ -143,7 +150,7 @@ min_bench_itrs: DEFAULT_MIN_BENCH_ITRS, min_bench_time: DEFAULT_MIN_BENCH_TIME, } -DEFAULT_CONFIGS = %w(yjit_stats prod_ruby_with_yjit prod_ruby_no_jit prev_ruby_yjit) +DEFAULT_CONFIGS = %w(yjit_stats prod_ruby_with_yjit prod_ruby_no_jit prev_ruby_yjit prev_ruby_no_jit) configs_to_test = DEFAULT_CONFIGS.map { |config| "#{YJITMetrics::PLATFORM}_#{config}"} bench_data = nil when_error = :report diff --git a/lib/yjit-metrics/bench-results.rb b/lib/yjit-metrics/bench-results.rb index 1a3d9ee2f..3c1a2abe5 100644 --- a/lib/yjit-metrics/bench-results.rb +++ b/lib/yjit-metrics/bench-results.rb @@ -194,6 +194,7 @@ def platforms "prod_ruby_with_mjit" => "MJIT", "ruby_30_with_mjit" => "MJIT-3.0", "prod_ruby_no_jit" => "CRuby ", + "prev_ruby_no_jit" => "CRuby ", "truffleruby" => "TruffleRuby", "yjit_stats" => "YJIT Stats", } @@ -662,6 +663,9 @@ module YJITMetrics "x86_64_prod_ruby_with_yjit" => { max_warmup_itrs: 30, }, + "x86_64_prev_ruby_no_jit" => { + max_warmup_itrs: 30, + }, "x86_64_prev_ruby_yjit" => { max_warmup_itrs: 30, }, @@ -678,6 +682,9 @@ module YJITMetrics "aarch64_prod_ruby_with_yjit" => { max_warmup_itrs: 30, }, + "aarch64_prev_ruby_no_jit" => { + max_warmup_itrs: 30, + }, "aarch64_prev_ruby_yjit" => { max_warmup_itrs: 30, }, diff --git a/lib/yjit-metrics/report_templates/blog_speed_details.html.erb b/lib/yjit-metrics/report_templates/blog_speed_details.html.erb index adad11799..49086a146 100644 --- a/lib/yjit-metrics/report_templates/blog_speed_details.html.erb +++ b/lib/yjit-metrics/report_templates/blog_speed_details.html.erb @@ -25,9 +25,9 @@

- Tested Ruby version for YJIT and No-JIT: <%= @ruby_metadata_by_config[@with_yjit_config]["RUBY_DESCRIPTION"] %>
+ Tested Ruby version for development CRuby and YJIT: <%= @ruby_metadata_by_config[@with_yjit_config]["RUBY_DESCRIPTION"] %>
<% if @with_prev_yjit_config %> - Tested Ruby version for Ruby 3.3 YJIT: <%= @ruby_metadata_by_config[@with_prev_yjit_config]["RUBY_DESCRIPTION"] %>
+ Tested Ruby version for stable CRuby and YJIT: <%= @ruby_metadata_by_config[@with_prev_yjit_config]["RUBY_DESCRIPTION"] %>
<% end %> <% if @with_mjit_latest_config %> Tested Ruby version for Ruby latest MJIT: <%= @ruby_metadata_by_config[@with_mjit_latest_config]["RUBY_DESCRIPTION"] %>
diff --git a/lib/yjit-metrics/report_types/bloggable_speed_report.rb b/lib/yjit-metrics/report_types/bloggable_speed_report.rb index 18ca8a32b..9bceaa2a1 100644 --- a/lib/yjit-metrics/report_types/bloggable_speed_report.rb +++ b/lib/yjit-metrics/report_types/bloggable_speed_report.rb @@ -44,6 +44,7 @@ def look_up_data_by_ruby(only_platforms: YJITMetrics::PLATFORMS, in_runs: false) raise "No data files for platform(s) #{only_platforms.inspect} in #{@config_names}!" if config_names.empty? @with_yjit_config = exactly_one_config_with_name(config_names, "prod_ruby_with_yjit", "with-YJIT") + @prev_no_jit_config = exactly_one_config_with_name(config_names, "prev_ruby_no_jit", "prev-CRuby", none_okay: true) @with_prev_yjit_config = exactly_one_config_with_name(config_names, "prev_ruby_yjit", "prev-YJIT", none_okay: true) @with_mjit30_config = exactly_one_config_with_name(config_names, "ruby_30_with_mjit", "with-MJIT3.0", none_okay: true) @with_mjit_latest_config = exactly_one_config_with_name(config_names, "prod_ruby_with_mjit", "with-MJIT", none_okay: true) @@ -52,6 +53,7 @@ def look_up_data_by_ruby(only_platforms: YJITMetrics::PLATFORMS, in_runs: false) # Order matters here - we push No-JIT, then MJIT(s), then YJIT and finally TruffleRuby when present @configs_with_human_names = [ + ["CRuby ", @prev_no_jit_config], ["CRuby ", @no_jit_config], ["MJIT3.0", @with_mjit30_config], ["MJIT", @with_mjit_latest_config], @@ -431,7 +433,7 @@ def svg_object(benchmarks: @benchmark_names) # Set up the top legend with coloured boxes and Ruby config names top_legend_box_height = 0.03 - top_legend_box_width = 0.1 + top_legend_box_width = 0.12 top_legend_text_height = 0.025 # Turns out we can't directly specify this... legend_box_stroke_colour = "#888" top_legend_item_width = plot_effective_width / n_configs diff --git a/lib/yjit-metrics/report_types/variable_warmup_report.rb b/lib/yjit-metrics/report_types/variable_warmup_report.rb index 92f0faf6f..59eb2730d 100644 --- a/lib/yjit-metrics/report_types/variable_warmup_report.rb +++ b/lib/yjit-metrics/report_types/variable_warmup_report.rb @@ -30,7 +30,9 @@ def look_up_data_by_ruby # It matters because we want the output reports to be stable with no churn in Git. bench_configs = YJITMetrics::DEFAULT_YJIT_BENCH_CI_SETTINGS["configs"] configs = @result_set.config_names - config_order = configs.select { |c| c["prod_ruby_no_jit"] }.sort + config_order = [] + config_order += configs.select { |c| c["prev_ruby_no_jit"] }.sort # optional + config_order += configs.select { |c| c["prod_ruby_no_jit"] }.sort config_order += configs.select { |c| c["prod_ruby_with_mjit"] }.sort # MJIT is optional, may be empty config_order += configs.select { |c| c["prev_ruby_yjit"] }.sort # optional config_order += configs.select { |c| c["prod_ruby_with_yjit"] }.sort From 4f933f9912da977cb7da1e9308f32df1543c145d Mon Sep 17 00:00:00 2001 From: Randy Stauner Date: Fri, 24 May 2024 15:04:41 -0700 Subject: [PATCH 2/4] Remove all references to mem_ratio_by_config We already weren't using it and as we add more configs it makes less sense. --- .../report_types/bloggable_speed_report.rb | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/lib/yjit-metrics/report_types/bloggable_speed_report.rb b/lib/yjit-metrics/report_types/bloggable_speed_report.rb index 9bceaa2a1..09e2d9a3f 100644 --- a/lib/yjit-metrics/report_types/bloggable_speed_report.rb +++ b/lib/yjit-metrics/report_types/bloggable_speed_report.rb @@ -162,10 +162,8 @@ def calc_speed_stats_by_config def calc_mem_stats_by_config @peak_mb_by_config = {} - @mem_ratio_by_config = {} @configs_with_human_names.map { |name, config| config }.each do |config| @peak_mb_by_config[config] = [] - @mem_ratio_by_config[config] = [] end @mem_overhead_factor_by_benchmark = [] @@ -175,21 +173,12 @@ def calc_mem_stats_by_config one_mib = 1024 * 1024.0 # As a float @benchmark_names.each.with_index do |benchmark_name, idx| - no_jit_bytes = mean(@peak_mem_by_config[@no_jit_config][benchmark_name]) @configs_with_human_names.each do |name, config| if @peak_mem_by_config[config][benchmark_name].nil? @peak_mb_by_config[config].push nil - if config != @no_jit_config - @mem_ratio_by_config[config].push nil - end else this_config_bytes = mean(@peak_mem_by_config[config][benchmark_name]) @peak_mb_by_config[config].push(this_config_bytes / one_mib) - - # Total mem ratios - not currently displayed - if config != @no_jit_config - @mem_ratio_by_config[config].push(this_config_bytes / no_jit_bytes) - end end end @@ -709,7 +698,6 @@ def report_table_data @configs_with_human_names.map { |name, config| @peak_mb_by_config[config][idx] } + [ @inline_mem_used[idx], @outline_mem_used[idx] ] #[ "#{"%d" % (@peak_mb_by_config[@with_yjit_config][idx] - 256)} + #{@inline_mem_used[idx]}/128 + #{@outline_mem_used[idx]}/128" ] - #@configs_with_human_names.flat_map { |name, config| config == @no_jit_config ? [] : @mem_ratio_by_config[config][idx] } end end @@ -726,7 +714,6 @@ def details_report_table_data @configs_with_human_names.map { |name, config| @peak_mb_by_config[config][idx] } + [ @inline_mem_used[idx], @outline_mem_used[idx], @mem_overhead_factor_by_benchmark[idx] * 100.0 ] #[ "#{"%d" % (@peak_mb_by_config[@with_yjit_config][idx] - 256)} + #{@inline_mem_used[idx]}/128 + #{@outline_mem_used[idx]}/128" ] - #@configs_with_human_names.flat_map { |name, config| config == @no_jit_config ? [] : @mem_ratio_by_config[config][idx] } end end From 5df07671fcd26275e26279cfb074fd29cb39617b Mon Sep 17 00:00:00 2001 From: Randy Stauner Date: Fri, 24 May 2024 15:28:29 -0700 Subject: [PATCH 3/4] Use prev_no_jit as baseline config if present, else current no_jit --- .../report_types/bloggable_speed_report.rb | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/lib/yjit-metrics/report_types/bloggable_speed_report.rb b/lib/yjit-metrics/report_types/bloggable_speed_report.rb index 09e2d9a3f..d59dc03a5 100644 --- a/lib/yjit-metrics/report_types/bloggable_speed_report.rb +++ b/lib/yjit-metrics/report_types/bloggable_speed_report.rb @@ -51,6 +51,9 @@ def look_up_data_by_ruby(only_platforms: YJITMetrics::PLATFORMS, in_runs: false) @no_jit_config = exactly_one_config_with_name(config_names, "prod_ruby_no_jit", "no-JIT") @truffle_config = exactly_one_config_with_name(config_names, "truffleruby", "Truffle", none_okay: true) + # Prefer previous CRuby if present otherwise current CRuby. + @baseline_config = @prev_no_jit_config || @no_jit_config + # Order matters here - we push No-JIT, then MJIT(s), then YJIT and finally TruffleRuby when present @configs_with_human_names = [ ["CRuby ", @prev_no_jit_config], @@ -113,7 +116,7 @@ def calc_speed_stats_by_config @mean_by_config[config] = [] @rsd_pct_by_config[config] = [] @total_time_by_config[config] = 0.0 - @speedup_by_config[config] = [] unless config == @no_jit_config + @speedup_by_config[config] = [] unless config == @baseline_config end @yjit_ratio = [] @@ -128,11 +131,11 @@ def calc_speed_stats_by_config @rsd_pct_by_config[config].push this_config_rel_stddev_pct end - no_jit_mean = @mean_by_config[@no_jit_config][-1] # Last pushed -- the one for this benchmark - no_jit_rel_stddev_pct = @rsd_pct_by_config[@no_jit_config][-1] - no_jit_rel_stddev = no_jit_rel_stddev_pct / 100.0 # Get ratio, not percent + baseline_mean = @mean_by_config[@baseline_config][-1] # Last pushed -- the one for this benchmark + baseline_rel_stddev_pct = @rsd_pct_by_config[@baseline_config][-1] + baseline_rel_stddev = baseline_rel_stddev_pct / 100.0 # Get ratio, not percent @configs_with_human_names.each do |name, config| - next if config == @no_jit_config + next if config == @baseline_config this_config_mean = @mean_by_config[config][-1] @@ -141,8 +144,8 @@ def calc_speed_stats_by_config else this_config_rel_stddev_pct = @rsd_pct_by_config[config][-1] this_config_rel_stddev = this_config_rel_stddev_pct / 100.0 # Get ratio, not percent - speed_ratio = no_jit_mean / this_config_mean - speed_rel_stddev = Math.sqrt(no_jit_rel_stddev * no_jit_rel_stddev + this_config_rel_stddev * this_config_rel_stddev) + speed_ratio = baseline_mean / this_config_mean + speed_rel_stddev = Math.sqrt(baseline_rel_stddev * baseline_rel_stddev + this_config_rel_stddev * this_config_rel_stddev) @speedup_by_config[config].push [ speed_ratio, speed_rel_stddev * 100.0 ] end @@ -182,6 +185,9 @@ def calc_mem_stats_by_config end end + # Here we use @with_yjit_config and @no_jit_config directly (not @baseline_config) + # to compare the memory difference of yjit vs no_jit on the same version. + yjit_mem_usage = @peak_mem_by_config[@with_yjit_config][benchmark_name].sum no_jit_mem_usage = @peak_mem_by_config[@no_jit_config][benchmark_name].sum @mem_overhead_factor_by_benchmark[idx] = (yjit_mem_usage.to_f / no_jit_mem_usage) - 1.0 @@ -238,7 +244,7 @@ def initialize(orig_config_names, platform, results, benchmarks: []) @headings = [ "bench" ] + @configs_with_human_names.flat_map { |name, config| [ "#{name} (ms)", "#{name} RSD" ] } + - @configs_with_human_names.flat_map { |name, config| config == @no_jit_config ? [] : [ "#{name} spd", "#{name} spd RSD" ] } + + @configs_with_human_names.flat_map { |name, config| config == @baseline_config ? [] : [ "#{name} spd", "#{name} spd RSD" ] } + [ "% in YJIT" ] # Col formats are only used when formatting entries for a text table, not for CSV @col_formats = [ "%s" ] + # Benchmark name @@ -266,7 +272,7 @@ def report_table_data @benchmark_names.map.with_index do |bench_name, idx| [ bench_name ] + @configs_with_human_names.flat_map { |name, config| [ @mean_by_config[config][idx], @rsd_pct_by_config[config][idx] ] } + - @configs_with_human_names.flat_map { |name, config| config == @no_jit_config ? [] : @speedup_by_config[config][idx] } + + @configs_with_human_names.flat_map { |name, config| config == @baseline_config ? [] : @speedup_by_config[config][idx] } + [ @yjit_ratio[idx] ] end end @@ -282,7 +288,7 @@ def details_report_table_data end [ "#{bench_name}" ] + @configs_with_human_names.flat_map { |name, config| [ @mean_by_config[config][idx], @rsd_pct_by_config[config][idx] ] } + - @configs_with_human_names.flat_map { |name, config| config == @no_jit_config ? [] : @speedup_by_config[config][idx] } + + @configs_with_human_names.flat_map { |name, config| config == @baseline_config ? [] : @speedup_by_config[config][idx] } + [ @yjit_ratio[idx] ] end end @@ -449,15 +455,15 @@ def svg_object(benchmarks: @benchmark_names) benchmarks.each.with_index do |bench_name, bench_short_idx| bench_idx = @benchmark_names.index(bench_name) - no_jit_mean = @mean_by_config[@no_jit_config][bench_idx] + baseline_mean = @mean_by_config[@baseline_config][bench_idx] bars_width_start = bench_left_edge[bench_short_idx] ruby_configs.each.with_index do |config, config_idx| human_name = ruby_human_names[config_idx] - if config == @no_jit_config + if config == @baseline_config speedup = 1.0 # No-JIT is always exactly 1x No-JIT - rsd_pct = @rsd_pct_by_config[@no_jit_config][bench_idx] + rsd_pct = @rsd_pct_by_config[@baseline_config][bench_idx] else speedup, rsd_pct = @speedup_by_config[config][bench_idx] end @@ -681,7 +687,7 @@ def initialize(config_names, platform, results, benchmarks: []) @headings = [ "bench" ] + @configs_with_human_names.map { |name, config| "#{name} mem (MiB)"} + [ "Inline Code", "Outlined Code", "YJIT Mem overhead" ] - #@configs_with_human_names.flat_map { |name, config| config == @no_jit_config ? [] : [ "#{name} mem ratio" ] } + #@configs_with_human_names.flat_map { |name, config| config == @baseline_config ? [] : [ "#{name} mem ratio" ] } # Col formats are only used when formatting entries for a text table, not for CSV @col_formats = [ "%s" ] + # Benchmark name [ "%d" ] * @configs_with_human_names.size + # Mem usage per-Ruby @@ -1065,6 +1071,8 @@ def initialize(config_names, results, benchmarks: []) calc_speed_stats_by_config + # For these ratios we compare current yjit and no_jit directly (not @baseline_config). + # "Ratio of total times" method #@yjit_vs_cruby_ratio = @total_time_by_config[@no_jit_config] / @total_time_by_config[@with_yjit_config] #@yjit_vs_mjit_ratio = @total_time_by_config[@with_mjit_config] / @total_time_by_config[@with_yjit_config] From f9e7f867f6fc8cdbdae1a0fa13e2747a763e5b17 Mon Sep 17 00:00:00 2001 From: Randy Stauner Date: Fri, 24 May 2024 15:41:32 -0700 Subject: [PATCH 4/4] Remove commented assignment of unused var --- lib/yjit-metrics/report_types/bloggable_speed_report.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/yjit-metrics/report_types/bloggable_speed_report.rb b/lib/yjit-metrics/report_types/bloggable_speed_report.rb index d59dc03a5..bafb0b5e8 100644 --- a/lib/yjit-metrics/report_types/bloggable_speed_report.rb +++ b/lib/yjit-metrics/report_types/bloggable_speed_report.rb @@ -1075,7 +1075,6 @@ def initialize(config_names, results, benchmarks: []) # "Ratio of total times" method #@yjit_vs_cruby_ratio = @total_time_by_config[@no_jit_config] / @total_time_by_config[@with_yjit_config] - #@yjit_vs_mjit_ratio = @total_time_by_config[@with_mjit_config] / @total_time_by_config[@with_yjit_config] headline_runtimes = headline_benchmarks.map do |bench_name| bench_idx = @benchmark_names.index(bench_name)