Skip to content
17 changes: 17 additions & 0 deletions lib/benchmark_runner.rb
Original file line number Diff line number Diff line change
Expand Up @@ -80,4 +80,21 @@ def setarch_prefix

prefix
end

# Checked system - error or return info if the command fails
def check_call(command, env: {}, raise_error: true, quiet: false)
puts("+ #{command}") unless quiet

result = {}

result[:success] = system(env, command)
result[:status] = $?

unless result[:success]
puts "Command #{command.inspect} failed with exit code #{result[:status].exitstatus} in directory #{Dir.pwd}"
raise RuntimeError.new if raise_error
end

result
end
end
161 changes: 161 additions & 0 deletions lib/cpu_config.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
require_relative 'benchmark_runner'

# Manages CPU frequency and turbo boost configuration for benchmark consistency
class CPUConfig
class << self
# Configure CPU for benchmarking: disable frequency scaling and verify settings
def configure_for_benchmarking(turbo:)
build.configure_for_benchmarking(turbo: turbo)
end

def build
if File.exist?(IntelCPUConfig::PSTATE_DIR)
IntelCPUConfig.new
elsif File.exist?(AMDCPUConfig::BOOST_PATH)
AMDCPUConfig.new
else
NullCPUConfig.new
end
end
end

def configure_for_benchmarking(turbo:)
disable_frequency_scaling(turbo: turbo)
check_pstate(turbo: turbo)
end

private

def disable_frequency_scaling(turbo:)
disable_turbo_boost unless turbo || turbo_disabled?
maximize_frequency unless frequency_maximized?
end

def turbo_disabled?
# Override in subclasses
false
end

def frequency_maximized?
# Override in subclasses
false
end

def disable_turbo_boost
# Override in subclasses
end

def maximize_frequency
# Override in subclasses
end

def check_pstate(turbo:)
# Override in subclasses
end
end

# Intel CPU configuration
class IntelCPUConfig < CPUConfig
PSTATE_DIR = '/sys/devices/system/cpu/intel_pstate'
NO_TURBO_PATH = "#{PSTATE_DIR}/no_turbo"
MIN_PERF_PCT_PATH = "#{PSTATE_DIR}/min_perf_pct"
TURBO_DISABLED_VALUE = '1'
FREQUENCY_MAXIMIZED_VALUE = '100'

private

def disable_turbo_boost
# sudo requires the flag '-S' in order to take input from stdin
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see that it's always been like this but I don't know why we are using -S.

BenchmarkRunner.check_call("sudo -S sh -c 'echo #{TURBO_DISABLED_VALUE} > #{NO_TURBO_PATH}'")
at_exit { BenchmarkRunner.check_call("sudo -S sh -c 'echo 0 > #{NO_TURBO_PATH}'", quiet: true) }
end

def maximize_frequency
# Disabling Turbo Boost reduces the CPU frequency, so this should be run after that.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I feel like this comment should actually go in disable_frequency_scaling since that's where the order is important.

BenchmarkRunner.check_call("sudo -S sh -c 'echo #{FREQUENCY_MAXIMIZED_VALUE} > #{MIN_PERF_PCT_PATH}'")
end

def turbo_disabled?
@turbo_disabled ||= File.exist?(NO_TURBO_PATH) &&
File.read(NO_TURBO_PATH).strip == TURBO_DISABLED_VALUE
end

def frequency_maximized?
@frequency_maximized ||= File.exist?(MIN_PERF_PCT_PATH) &&
File.read(MIN_PERF_PCT_PATH).strip == FREQUENCY_MAXIMIZED_VALUE
end

def check_pstate(turbo:)
unless turbo || turbo_disabled?
puts("You forgot to disable turbo:")
puts(" sudo sh -c 'echo #{TURBO_DISABLED_VALUE} > #{NO_TURBO_PATH}'")
exit(-1)
end

unless frequency_maximized?
puts("You forgot to set the min perf percentage to 100:")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
puts("You forgot to set the min perf percentage to 100:")
puts("You forgot to set the min perf percentage to #{FREQUENCY_MAXIMIZED_VALUE}:")

puts(" sudo sh -c 'echo #{FREQUENCY_MAXIMIZED_VALUE} > #{MIN_PERF_PCT_PATH}'")
exit(-1)
end
end
end

# AMD CPU configuration
class AMDCPUConfig < CPUConfig
CPUFREQ_DIR = '/sys/devices/system/cpu/cpufreq'
BOOST_PATH = "#{CPUFREQ_DIR}/boost"
SCALING_GOVERNOR_GLOB = '/sys/devices/system/cpu/cpu*/cpufreq/scaling_governor'
TURBO_DISABLED_VALUE = '0'
TURBO_ENABLED_VALUE = '1'
PERFORMANCE_GOVERNOR = 'performance'

private

def disable_turbo_boost
# sudo requires the flag '-S' in order to take input from stdin
BenchmarkRunner.check_call("sudo -S sh -c 'echo #{TURBO_DISABLED_VALUE} > #{BOOST_PATH}'")
at_exit { BenchmarkRunner.check_call("sudo -S sh -c 'echo #{TURBO_ENABLED_VALUE} > #{BOOST_PATH}'", quiet: true) }
end

def maximize_frequency
BenchmarkRunner.check_call("sudo -S cpupower frequency-set -g performance")
end

def turbo_disabled?
@turbo_disabled ||= File.exist?(BOOST_PATH) &&
File.read(BOOST_PATH).strip == TURBO_DISABLED_VALUE
end

def frequency_maximized?
@frequency_maximized ||= Dir.glob(SCALING_GOVERNOR_GLOB).all? do |governor|
File.read(governor).strip == PERFORMANCE_GOVERNOR
end
end

def check_pstate(turbo:)
unless turbo || turbo_disabled?
puts("You forgot to disable boost:")
puts(" sudo sh -c 'echo #{TURBO_DISABLED_VALUE} > #{BOOST_PATH}'")
exit(-1)
end

unless frequency_maximized?
puts("You forgot to set the performance governor:")
puts(" sudo cpupower frequency-set -g #{PERFORMANCE_GOVERNOR}")
exit(-1)
end
end
end

# Null object for unsupported CPUs
class NullCPUConfig < CPUConfig
private

def disable_frequency_scaling(turbo:)
# Do nothing
end

def check_pstate(turbo:)
# Do nothing
end
end
105 changes: 6 additions & 99 deletions run_benchmarks.rb
Original file line number Diff line number Diff line change
Expand Up @@ -11,101 +11,16 @@
require 'etc'
require 'yaml'
require_relative 'misc/stats'
require_relative 'lib/cpu_config'
require_relative 'lib/benchmark_runner'
require_relative 'lib/table_formatter'
require_relative 'lib/benchmark_filter'

# Checked system - error or return info if the command fails
def check_call(command, env: {}, raise_error: true, quiet: false)
puts("+ #{command}") unless quiet

result = {}

result[:success] = system(env, command)
result[:status] = $?

unless result[:success]
puts "Command #{command.inspect} failed with exit code #{result[:status].exitstatus} in directory #{Dir.pwd}"
raise RuntimeError.new if raise_error
end

result
end

def check_output(*command)
IO.popen(*command, &:read)
end

def have_yjit?(ruby)
ruby_version = check_output("#{ruby} -v --yjit", err: File::NULL).strip
ruby_version = `#{ruby} -v --yjit 2> #{File::NULL}`.strip
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why not continue using check_output here?

Copy link
Collaborator Author

@rafaelfranca rafaelfranca Nov 20, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

popen is overkill. It is the same answer as the Process case. Ruby has Kernel#` to do what we want, get the stdout of a subshell. There is no reason to reimplement it using popen

ruby_version.downcase.include?("yjit")
end

# Disable Turbo Boost while running benchmarks. Maximize the CPU frequency.
def set_bench_config(turbo:)
# sudo requires the flag '-S' in order to take input from stdin
if File.exist?('/sys/devices/system/cpu/intel_pstate') # Intel
unless intel_no_turbo? || turbo
check_call("sudo -S sh -c 'echo 1 > /sys/devices/system/cpu/intel_pstate/no_turbo'")
at_exit { check_call("sudo -S sh -c 'echo 0 > /sys/devices/system/cpu/intel_pstate/no_turbo'", quiet: true) }
end
# Disabling Turbo Boost reduces the CPU frequency, so this should be run after that.
check_call("sudo -S sh -c 'echo 100 > /sys/devices/system/cpu/intel_pstate/min_perf_pct'") unless intel_perf_100pct?
elsif File.exist?('/sys/devices/system/cpu/cpufreq/boost') # AMD
unless amd_no_boost? || turbo
check_call("sudo -S sh -c 'echo 0 > /sys/devices/system/cpu/cpufreq/boost'")
at_exit { check_call("sudo -S sh -c 'echo 1 > /sys/devices/system/cpu/cpufreq/boost'", quiet: true) }
end
check_call("sudo -S cpupower frequency-set -g performance") unless performance_governor?
end
end

def check_pstate(turbo:)
if File.exist?('/sys/devices/system/cpu/intel_pstate') # Intel
unless turbo || intel_no_turbo?
puts("You forgot to disable turbo:")
puts(" sudo sh -c 'echo 1 > /sys/devices/system/cpu/intel_pstate/no_turbo'")
exit(-1)
end

unless intel_perf_100pct?
puts("You forgot to set the min perf percentage to 100:")
puts(" sudo sh -c 'echo 100 > /sys/devices/system/cpu/intel_pstate/min_perf_pct'")
exit(-1)
end
elsif File.exist?('/sys/devices/system/cpu/cpufreq/boost') # AMD
unless turbo || amd_no_boost?
puts("You forgot to disable boost:")
puts(" sudo sh -c 'echo 0 > /sys/devices/system/cpu/cpufreq/boost'")
exit(-1)
end

unless performance_governor?
puts("You forgot to set the performance governor:")
puts(" sudo cpupower frequency-set -g performance")
exit(-1)
end
end
end

def intel_no_turbo?
File.read('/sys/devices/system/cpu/intel_pstate/no_turbo').strip == '1'
end

def intel_perf_100pct?
File.read('/sys/devices/system/cpu/intel_pstate/min_perf_pct').strip == '100'
end

def amd_no_boost?
File.read('/sys/devices/system/cpu/cpufreq/boost').strip == '0'
end

def performance_governor?
Dir.glob('/sys/devices/system/cpu/cpu*/cpufreq/scaling_governor').all? do |governor|
File.read(governor).strip == 'performance'
end
end

def mean(values)
Stats.new(values).mean
end
Expand All @@ -132,10 +47,6 @@ def sort_benchmarks(bench_names)
BenchmarkRunner.sort_benchmarks(bench_names, benchmarks_metadata)
end

def setarch_prefix
BenchmarkRunner.setarch_prefix
end

# Run all the benchmarks and record execution times
def run_benchmarks(ruby:, ruby_description:, categories:, name_filters:, out_path:, harness:, pre_init:, no_pinning:)
bench_data = {}
Expand Down Expand Up @@ -193,7 +104,7 @@ def run_benchmarks(ruby:, ruby_description:, categories:, name_filters:, out_pat
# Set up the benchmarking command
cmd = []
if BenchmarkRunner.os == :linux
cmd += setarch_prefix
cmd += BenchmarkRunner.setarch_prefix

# Pin the process to one given core to improve caching and reduce variance on CRuby
# Other Rubies need to use multiple cores, e.g., for JIT threads
Expand Down Expand Up @@ -230,7 +141,7 @@ def run_benchmarks(ruby:, ruby_description:, categories:, name_filters:, out_pat
end

# Do the benchmarking
result = check_call(cmd.shelljoin, env: env, raise_error: false)
result = BenchmarkRunner.check_call(cmd.shelljoin, env: env, raise_error: false)

if result[:success]
bench_data[bench_name] = JSON.parse(File.read(result_json_path)).tap do |json|
Expand Down Expand Up @@ -387,18 +298,14 @@ def run_benchmarks(ruby:, ruby_description:, categories:, name_filters:, out_pat
end
end

# Disable CPU frequency scaling
set_bench_config(turbo: args.turbo)

# Check pstate status
check_pstate(turbo: args.turbo)
CPUConfig.configure_for_benchmarking(turbo: args.turbo)

# Create the output directory
FileUtils.mkdir_p(args.out_path)

ruby_descriptions = {}
args.executables.each do |name, executable|
ruby_descriptions[name] = check_output([*executable, "-v"]).chomp
ruby_descriptions[name] = `#{executable.shelljoin} -v`.chomp
end

# Benchmark with and without YJIT
Expand Down
Loading
Loading