Setup dummy test server

bin/bench

@@ -0,0 +1,107 @@
+#!/usr/bin/env ruby
+# ActiveModelSerializers Benchmark driver
+
+require 'json'
+require 'pathname'
+require 'optparse'
+require 'digest'
+require 'pathname'
+
+class BenchmarkDriver
+  ROOT = Pathname File.expand_path(File.join('..', '..'), __FILE__)
+  BASE = ENV.fetch('BASE') { ROOT.join('test', 'dummy') }
+
+  def self.benchmark(options)
+    self.new(options).run
+  end
+
+  def initialize(options)
+    @repeat_count = options[:repeat_count]
+    @pattern = options[:pattern]
+    @env = Array(options[:env]).join(' ')
+  end
+
+  def run
+    files.each do |path|
+      next if !@pattern.empty? && /#{@pattern.join('|')}/ !~ File.basename(path)
+      run_single(path)
+    end
+  end
+
+  private
+
+  def files
+    Dir[File.join(BASE, 'bm_*')]
+  end
+
+  def run_single(path)
+    script = "RAILS_ENV=production #{@env} ruby #{path}"
+
+    # FIXME: ` provides the full output but it'll return failed output as well.
+    results = measure(script)
+
+    # TODO:
+    # "vs. earliest ref (#{first_entry_ref[0..8]}) (x faster)",
+    # "caching speedup (on / off)"
+    commit_hash = ENV['COMMIT_HASH'] || `git rev-parse --short HEAD`.chomp
+
+    environment = `ruby -v`.chomp.strip[/\d+\.\d+\.\d+\w+/]
+
+    puts "Benchmark results:"
+    results.each do |json|
+      result = {
+        'name' => json['label'],
+        'commit_hash' => commit_hash,
+        'version' => json['version'],
+        "total_allocated_objects_per_measurement" => json["total_allocated_objects_per_measurement"],
+        'environment' => environment
+      }
+      if json['error']
+        result['error'] = json['error']
+      else
+        result['time[user]'] = Float(json['user']).round(5)
+      end
+      puts result
+    end
+  end
+
+  def measure(script)
+    results = []
+
+    @repeat_count.times do
+      output = `#{script}`
+      result = output.split("\n").map {|line|
+        JSON.parse(line)
+      }
+      results << result
+    end
+
+    results.sort_by do |result|
+      result.first['time[user]']
+    end.last
+  end
+end
+
+options = {
+  repeat_count: 1,
+  pattern: [],
+  env: "CACHE_ON=on"
+}
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: bin/bench [options]"
+
+  opts.on("-r", "--repeat-count [NUM]", "Run benchmarks [NUM] times taking the best result") do |value|
+    options[:repeat_count] = value.to_i
+  end
+
+  opts.on("-p", "--pattern <PATTERN1,PATTERN2,PATTERN3>", "Benchmark name pattern") do |value|
+    options[:pattern] = value.split(',')
+  end
+
+  opts.on("-e", "--env <var1=val1,var2=val2,var3=vale>", "ENV variables to pass in") do |value|
+    options[:env] = value.split(',')
+  end
+end.parse!(ARGV)
+
+BenchmarkDriver.benchmark(options)

bin/revision_runner

@@ -0,0 +1,162 @@
+#!/usr/bin/env ruby
+require 'fileutils'
+require 'pathname'
+require 'shellwords'
+require 'English'
+require 'net/http'
+require 'json'
+
+############################
+#
+# A wrapper around git-bisect that stashes some code (bin/bench,test/dummy/*) for use in making requests
+# running against each revision.
+#
+# USAGE
+#
+# bin/revision_runner <ref1> <ref2> <cmd>
+#   <ref1> defaults to the current branch
+#   <ref2> defaults to the master branch
+# Runs <cmd> across every revisiion in the range, inclusive.
+# TODO: Aborts when <cmd> exit code is non-zero.
+#
+# EXAMPLE
+#
+# bin/revision_runner 792fb8a9053f8db3c562dae4f40907a582dd1720 master bin/bench -r 2 -e CACHE_ON=off
+###########################
+
+class RevisionRunner
+  ROOT = Pathname File.expand_path(['..', '..'].join(File::Separator), __FILE__)
+  TMP_DIR_BASE = File.join('tmp', 'revision_runner')
+  TMP_DIR = File.join(ROOT, TMP_DIR_BASE)
+
+  attr_accessor :url_base
+
+  def initialize
+    refresh_temp_dir
+  end
+
+  def refresh_temp_dir
+    empty_temp_dir
+    fill_temp_dir
+  end
+
+  def temp_dir_empty?
+    Dir[File.join(TMP_DIR, '*')].none?
+  end
+
+  def empty_temp_dir
+    FileUtils.mkdir_p(TMP_DIR)
+    Dir[File.join(TMP_DIR, '*')].each do |file|
+      if File.directory?(file)
+        FileUtils.rm_rf(file)
+      else
+        FileUtils.rm(file)
+      end
+    end
+  end
+
+  def fill_temp_dir
+    Dir[File.join(ROOT, 'test', 'dummy', '*.{rb,ru}')].each do |file|
+      FileUtils.cp(file, File.join(TMP_DIR, File.basename(file)))
+    end
+    file = File.join('bin', 'bench')
+    FileUtils.cp(file, File.join(TMP_DIR, File.basename(file)))
+    at_exit { empty_temp_dir }
+  end
+
+
+  module GitCommands
+    module_function
+
+    def current_branch
+      @current_branch ||= `cat .git/HEAD | cut -d/ -f3,4,5`.chomp
+    end
+
+    def revisions(start_ref, end_ref)
+      cmd = "git rev-list --reverse #{start_ref}..#{end_ref}"
+      `#{cmd}`.chomp.split("\n")
+    end
+
+    def checkout_ref(ref)
+      `git checkout #{ref}`.chomp
+      abort "Checkout failed: #{ref}, #{$CHILD_STATUS.exitstatus}" unless $CHILD_STATUS.success?
+    end
+
+    def revision_description(rev)
+      `git log --oneline -1 #{rev}`.chomp
+    end
+
+    def bundle
+      `rm -f Gemfile.lock; bundle check || bundle --local --quiet || bundle --quiet`
+    end
+
+    def clean_head
+      system('git reset --hard --quiet')
+    end
+  end
+  include GitCommands
+
+  def run_revisions(ref1: nil, ref2: nil, cmd:)
+    ref0 = current_branch
+    ref1 ||= current_branch
+    ref2 ||= 'master'
+    reports = {}
+
+    revisions(ref1, ref2).each do |rev|
+      STDERR.puts "Checking out: #{revision_description(rev)}"
+
+      reports[rev] = run_at_ref(rev, cmd)
+      clean_head
+    end
+    checkout_ref(ref0)
+    debug { "OK for all revisions!" }
+    reports
+  rescue Exception # rubocop:disable Lint/RescueException
+    STDERR.puts $!.message
+    checkout_ref(ref0)
+    raise
+  ensure
+    return reports
+  end
+
+  def run_at_ref(ref, cmd)
+    checkout_ref(ref)
+    bundle
+    base = Shellwords.shellescape(TMP_DIR_BASE)
+    cmd = "COMMIT_HASH=#{ref} BASE=#{base} #{Shellwords.shelljoin(cmd)}"
+    cmd.sub('bin/bench', 'tmp/revision_runner/bench')
+    debug { cmd }
+    system(cmd)
+    $CHILD_STATUS.exitstatus
+  end
+
+  # TODO: useful?
+  # def restart_server
+  #   server_script = File.join(TMP_DIR_BASE, 'serve_dummy')
+  #   system("#{server_script} stop")
+  #   at_exit { system("#{server_script} stop") }
+  #   config_ru = Shellwords.shellescape(File.join(TMP_DIR_BASE, 'config.ru'))
+  #   pid = `CONFIG_RU=#{config_ru} #{server_script} start`.chomp
+  #   abort "No pid" if pid.empty?
+  #   pid = Integer(pid)
+  #   Process.kill(0, pid) # confirm running, else it raises
+  # end
+
+  def debug(msg = '')
+    if block_given? && ENV['DEBUG'] =~ /\Atrue|on|0\z/i
+      STDOUT.puts yield
+    else
+      STDOUT.puts msg
+    end
+  end
+
+end
+
+if $PROGRAM_NAME == __FILE__
+  runner = RevisionRunner.new
+  args = ARGV.dup
+  reports = runner.run_revisions(ref1: args.shift, ref2: args.shift, cmd: args)
+  reports.each do |name, value|
+    STDERR.puts "revision: #{name}\n\t#{value}"
+  end
+end

bin/serve_dummy

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+set -e
+
+case "$1" in
+
+  start)
+  config="${CONFIG_RU:-test/dummy/config.ru}"
+  bundle exec ruby -Ilib -S rackup "$config" --daemonize --pid tmp/dummy_app.pid --warn --server webrick
+  until [ -f 'tmp/dummy_app.pid' ]; do
+    sleep 0.1 # give it time to start.. I don't know a better way
+  done
+  cat tmp/dummy_app.pid
+  true
+  ;;
+
+  stop)
+  if [ -f 'tmp/dummy_app.pid' ]; then
+    kill -TERM $(cat tmp/dummy_app.pid)
+  else
+    echo 'No pidfile'
+    false
+  fi
+  ;;
+
+  status)
+  if [ -f 'tmp/dummy_app.pid' ]; then
+    kill -0 $(cat tmp/dummy_app.pid)
+    [ "$?" -eq 0 ]
+  else
+    echo 'No pidfile'
+    false
+  fi
+  ;;
+
+  *)
+  echo "Usage: $0 [start|stop|status]"
+  ;;
+
+esac

test/dummy/app.rb

@@ -0,0 +1,87 @@
+# https://github.com/rails-api/active_model_serializers/pull/872
+# approx ref 792fb8a9053f8db3c562dae4f40907a582dd1720 to test against
+require 'bundler/setup'
+
+require 'rails'
+require 'active_model'
+require 'active_support'
+require 'active_support/json'
+require 'action_controller'
+require 'action_controller/test_case'
+require 'action_controller/railtie'
+abort "Rails application already defined: #{Rails.application.class}" if Rails.application
+
+class NullLogger < Logger
+  def initialize(*_args)
+  end
+
+  def add(*_args, &_block)
+  end
+end
+class DummyLogger < ActiveSupport::Logger
+  def initialize
+    @file = StringIO.new
+    super(@file)
+  end
+
+  def messages
+    @file.rewind
+    @file.read
+  end
+end
+# ref: https://gist.github.com/bf4/8744473
+class DummyApp < Rails::Application
+  # CONFIG: CACHE_ON={on,off}
+  config.action_controller.perform_caching = ENV['CACHE_ON'] != 'off'
+  config.action_controller.cache_store = ActiveSupport::Cache.lookup_store(:memory_store)
+
+  # Set up production configuration
+  config.eager_load = true
+  config.cache_classes = true
+
+  config.active_support.test_order = :random
+  config.secret_token = '1234'
+  config.secret_key_base = 'abc123'
+  config.logger = NullLogger.new
+end
+
+require 'active_model_serializers'
+
+# Initialize app before any serializers are defined, for sanity's sake.
+# Otherwise, you have to manually set perform caching.
+#
+# Details:
+#
+# 1. Upon load, when AMS.config.perform_caching is true,
+#    serializers inherit the cache store from ActiveModelSerializers.config.cache_store
+# 1. If the serializers are loaded before Rails is initialized (`Rails.application.initialize!`),
+#    these values are nil, and are not applied to the already loaded serializers
+# 1. If Rails is initialized before any serializers are loaded, then the configs are set,
+#    and are used when serializers are loaded
+# 1. In either case, `ActiveModelSerializers.config.cache_store`, and
+#    `ActiveModelSerializers.config.perform_caching` can be set at any time before the serializers
+#    are loaded,
+#    e.g.  `ActiveModel::Serializer.config.cache_store ||=
+#      ActiveSupport::Cache.lookup_store(ActionController::Base.cache_store ||
+#      Rails.cache || :memory_store)`
+#    and `ActiveModelSerializers.config.perform_caching = true`
+# 1. If the serializers are loaded before Rails is initialized, then,
+#    you can set the `_cache` store directly on the serializers.
+#    `ActiveModel::Serializer._cache ||=
+#      ActiveSupport::Cache.lookup_store(ActionController::Base.cache_store ||
+#      Rails.cache || :memory_store`
+#    is sufficient.
+#    Setting `_cache` to a truthy value will cause the CachedSerializer
+#    to consider it cached, which will apply to all serializers (bug? :bug: )
+#
+# This happens, in part, because the cache store is set for a serializer
+# when `cache` is called, and cache is usually called when the serializer is defined.
+#
+# So, there's now a 'workaround', something to debug, and a starting point.
+Rails.application.initialize!
+
+# HACK: Serializer::cache depends on the ActionController-dependent configs being set.
+ActiveSupport.on_load(:action_controller) do
+  require_relative 'fixtures'
+end
+require_relative 'controllers'