benchmark

#!/usr/bin/env ruby
# frozen_string_literal: true

# This helps benchmark current performance of Dalli
# as well as compare performance of optimizated and non-optimized calls like multi-set vs set
#
# run with:
# bundle exec bin/benchmark
# RUBY_YJIT_ENABLE=1 BENCH_TARGET=get bundle exec bin/benchmark
require 'bundler/inline'
require 'json'

gemfile do
  source 'https://rubygems.org'
  gem 'benchmark-ips'
  gem 'logger'
  gem 'opentelemetry-sdk'
end

require_relative '../lib/dalli'
require 'benchmark/ips'
require 'monitor'

##
# StringSerializer is a serializer that avoids the overhead of Marshal or JSON.
##
class StringSerializer
  def self.dump(value)
    value
  end

  def self.load(value)
    value
  end
end

dalli_url = ENV['BENCH_CACHE_URL'] || "127.0.0.1:11211"

if dalli_url.include?('unix')
  ENV['BENCH_CACHE_URL'].gsub('unix://','')
end
bench_target = ENV['BENCH_TARGET'] || 'set'
bench_time = (ENV['BENCH_TIME'] || 10).to_i
bench_warmup = (ENV['BENCH_WARMUP'] || 3).to_i
bench_payload_size = (ENV['BENCH_PAYLOAD_SIZE'] || 700_000).to_i
payload = 'B' * bench_payload_size
TERMINATOR = "\r\n"
puts "yjit: #{RubyVM::YJIT.enabled?}"

client = Dalli::Client.new('localhost', serializer: StringSerializer, compress: false, raw: true)
multi_client = Dalli::Client.new('localhost:11211,localhost:11222', serializer: StringSerializer, compress: false, raw: true)

# The raw socket implementation is used to benchmark the performance of dalli & the overhead of the various abstractions
# in the library.
sock = TCPSocket.new('127.0.0.1', '11211', connect_timeout: 1)
sock.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, true)
sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, true)
# Benchmarks didn't see any performance gains from increasing the SO_RCVBUF buffer size
# sock.setsockopt(Socket::SOL_SOCKET, ::Socket::SO_RCVBUF, 1024 * 1024 * 8)
# Benchamrks did see an improvement in performance when increasing the SO_SNDBUF buffer size
sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDBUF, 1024 * 1024 * 8)

# ensure the clients are all connected and working
client.set('key', payload)
sock.write("set sock_key 0 3600 #{payload.bytesize}\r\n")
sock.write(payload)
sock.write(TERMINATOR)
sock.flush
sock.readline # clear the buffer

# ensure we have basic data for the benchmarks and get calls
payload_smaller = 'B' * 50_000
pairs = {}
100.times do |i|
  pairs["multi_#{i}"] = payload_smaller
end
client.quiet do
  pairs.each do |key, value|
    client.set(key, value, 3600, raw: true)
  end
end

###
# GC Suite
# benchmark without GC skewing things
###
class GCSuite
  def warming(*)
    run_gc
  end

  def running(*)
    run_gc
  end

  def warmup_stats(*); end

  def add_report(*); end

  private

  def run_gc
    GC.enable
    GC.start
    GC.disable
  end
end
suite = GCSuite.new

def sock_get_multi(sock, pairs)
  count = pairs.length
  pairs.each_key do |key|
    count -= 1
    tail = count.zero? ? '' : 'q'
    sock.write("mg #{key} v f k #{tail}\r\n")
  end
  sock.flush
  # read all the memcached responses back and build a hash of key value pairs
  results = {}
  last_result = false
  while (line = sock.readline.chomp!(TERMINATOR)) != ''
    last_result = true if line.start_with?('EN ')
    next unless line.start_with?('VA ') || last_result

    _, value_length, _flags, key = line.split
    results[key[1..]] = sock.read(value_length.to_i)
    sock.read(TERMINATOR.length)
    break if results.size == pairs.size
    break if last_result
  end
  results
end


if %w[all set].include?(bench_target)
  Benchmark.ips do |x|
    x.config(warmup: bench_warmup, time: bench_time, suite: suite)
    x.report('client set') { client.set('key', payload) }
    #x.report('multi client set') { multi_client.set('string_key', payload) }
    x.report('raw sock set') do
      sock.write("ms sock_key #{payload.bytesize} T3600 MS\r\n")
      sock.write(payload)
      sock.write("\r\n")
      sock.flush
      sock.readline # clear the buffer
    end
    x.compare!
  end
end

@lock = Monitor.new
if %w[all get].include?(bench_target)
  Benchmark.ips do |x|
    x.config(warmup: bench_warmup, time: bench_time, suite: suite)
    x.report('get dalli') { client.get('key') }
    # NOTE: while this is the fastest it is not thread safe and is blocking vs IO sharing friendly
    x.report('get sock') do
      sock.write("get sock_key\r\n")
      sock.readline
      sock.read(payload.bytesize)
    end
    # NOTE: This shows that when adding thread safety & non-blocking IO we are slower for single process/thread use case
    x.report('get sock non-blocking') do
      @lock.synchronize do
        sock.write("get sock_key\r\n")
        sock.readline
        count = payload.bytesize
        value = String.new(capacity: count + 1)
        loop do
          begin
            value << sock.read_nonblock(count - value.bytesize)
          rescue Errno::EAGAIN
            IO.select([sock])
            retry
          rescue EOFError
            puts "EOFError"
            break
          end
          break if value.bytesize == count
        end
      end
    end
    x.compare!
  end
end

if %w[all get_multi].include?(bench_target)
  Benchmark.ips do |x|
    x.config(warmup: bench_warmup, time: bench_time, suite: suite)
    x.report('get 100 keys') { client.get_multi(pairs.keys) }
    x.report('get 100 keys raw sock') { sock_get_multi(sock, pairs) }
    x.compare!
  end
end

if %w[all set_multi].include?(bench_target)
  Benchmark.ips do |x|
    x.config(warmup: bench_warmup, time: bench_time, suite: suite)
    x.report('write 100 keys simple') do
      client.quiet do
        pairs.each do |key, value|
          client.set(key, value, 3600, raw: true)
        end
      end
    end
    x.report('multi client set_multi 100') do
      multi_client.set_multi(pairs, 3600, raw: true)
    end
    x.report('write 100 keys rawsock') do
      count = pairs.length
      tail = ''
      value_bytesize = payload_smaller.bytesize
      ttl = 3600

      pairs.each do |key, value|
        count -= 1
        tail = count.zero? ? '' : 'q'
        sock.write(String.new("ms #{key} #{value_bytesize} c F0 T#{ttl} MS #{tail}\r\n",
                              capacity: key.size + value_bytesize + 40) << value << TERMINATOR)
      end
      sock.flush
      sock.gets(TERMINATOR) # clear the buffer
    end
    x.report('write_mutli 100 keys') { client.set_multi(pairs, 3600, raw: true) }
    x.compare!
  end
end

if %w[all delete_multi].include?(bench_target)
  Benchmark.ips do |x|
    x.config(warmup: bench_warmup, time: bench_time, suite: suite)
    x.report('delete_multi 100 keys') { client.delete_multi(pairs.keys) }
    x.report('delete 100 keys') { pairs.keys.each { |key| client.delete(key) } }
    x.compare!
  end
end