diff --git a/bin/bench b/bin/bench new file mode 100755 index 000000000..89a8ffa75 --- /dev/null +++ b/bin/bench @@ -0,0 +1,107 @@ +#!/usr/bin/env ruby +# ActiveModelSerializers Benchmark driver + +require 'json' +require 'pathname' +require 'optparse' +require 'digest' +require 'pathname' + +class BenchmarkDriver + ROOT = Pathname File.expand_path(File.join('..', '..'), __FILE__) + BASE = ENV.fetch('BASE') { ROOT.join('test', 'dummy') } + + def self.benchmark(options) + self.new(options).run + end + + def initialize(options) + @repeat_count = options[:repeat_count] + @pattern = options[:pattern] + @env = Array(options[:env]).join(' ') + end + + def run + files.each do |path| + next if !@pattern.empty? && /#{@pattern.join('|')}/ !~ File.basename(path) + run_single(path) + end + end + + private + + def files + Dir[File.join(BASE, 'bm_*')] + end + + def run_single(path) + script = "RAILS_ENV=production #{@env} ruby #{path}" + + # FIXME: ` provides the full output but it'll return failed output as well. + results = measure(script) + + # TODO: + # "vs. earliest ref (#{first_entry_ref[0..8]}) (x faster)", + # "caching speedup (on / off)" + commit_hash = ENV['COMMIT_HASH'] || `git rev-parse --short HEAD`.chomp + + environment = `ruby -v`.chomp.strip[/\d+\.\d+\.\d+\w+/] + + puts "Benchmark results:" + results.each do |json| + result = { + 'name' => json['label'], + 'commit_hash' => commit_hash, + 'version' => json['version'], + "total_allocated_objects_per_measurement" => json["total_allocated_objects_per_measurement"], + 'environment' => environment + } + if json['error'] + result['error'] = json['error'] + else + result['time[user]'] = Float(json['user']).round(5) + end + puts result + end + end + + def measure(script) + results = [] + + @repeat_count.times do + output = `#{script}` + result = output.split("\n").map {|line| + JSON.parse(line) + } + results << result + end + + results.sort_by do |result| + result.first['time[user]'] + end.last + end +end + +options = { + repeat_count: 1, + pattern: [], + env: "CACHE_ON=on" +} + +OptionParser.new do |opts| + opts.banner = "Usage: bin/bench [options]" + + opts.on("-r", "--repeat-count [NUM]", "Run benchmarks [NUM] times taking the best result") do |value| + options[:repeat_count] = value.to_i + end + + opts.on("-p", "--pattern ", "Benchmark name pattern") do |value| + options[:pattern] = value.split(',') + end + + opts.on("-e", "--env ", "ENV variables to pass in") do |value| + options[:env] = value.split(',') + end +end.parse!(ARGV) + +BenchmarkDriver.benchmark(options) diff --git a/bin/revision_runner b/bin/revision_runner new file mode 100755 index 000000000..2fffafe34 --- /dev/null +++ b/bin/revision_runner @@ -0,0 +1,162 @@ +#!/usr/bin/env ruby +require 'fileutils' +require 'pathname' +require 'shellwords' +require 'English' +require 'net/http' +require 'json' + +############################ +# +# A wrapper around git-bisect that stashes some code (bin/bench,test/dummy/*) for use in making requests +# running against each revision. +# +# USAGE +# +# bin/revision_runner +# defaults to the current branch +# defaults to the master branch +# Runs across every revisiion in the range, inclusive. +# TODO: Aborts when exit code is non-zero. +# +# EXAMPLE +# +# bin/revision_runner 792fb8a9053f8db3c562dae4f40907a582dd1720 master bin/bench -r 2 -e CACHE_ON=off +########################### + +class RevisionRunner + ROOT = Pathname File.expand_path(['..', '..'].join(File::Separator), __FILE__) + TMP_DIR_BASE = File.join('tmp', 'revision_runner') + TMP_DIR = File.join(ROOT, TMP_DIR_BASE) + + attr_accessor :url_base + + def initialize + refresh_temp_dir + end + + def refresh_temp_dir + empty_temp_dir + fill_temp_dir + end + + def temp_dir_empty? + Dir[File.join(TMP_DIR, '*')].none? + end + + def empty_temp_dir + FileUtils.mkdir_p(TMP_DIR) + Dir[File.join(TMP_DIR, '*')].each do |file| + if File.directory?(file) + FileUtils.rm_rf(file) + else + FileUtils.rm(file) + end + end + end + + def fill_temp_dir + Dir[File.join(ROOT, 'test', 'dummy', '*.{rb,ru}')].each do |file| + FileUtils.cp(file, File.join(TMP_DIR, File.basename(file))) + end + file = File.join('bin', 'bench') + FileUtils.cp(file, File.join(TMP_DIR, File.basename(file))) + at_exit { empty_temp_dir } + end + + + module GitCommands + module_function + + def current_branch + @current_branch ||= `cat .git/HEAD | cut -d/ -f3,4,5`.chomp + end + + def revisions(start_ref, end_ref) + cmd = "git rev-list --reverse #{start_ref}..#{end_ref}" + `#{cmd}`.chomp.split("\n") + end + + def checkout_ref(ref) + `git checkout #{ref}`.chomp + abort "Checkout failed: #{ref}, #{$CHILD_STATUS.exitstatus}" unless $CHILD_STATUS.success? + end + + def revision_description(rev) + `git log --oneline -1 #{rev}`.chomp + end + + def bundle + `rm -f Gemfile.lock; bundle check || bundle --local --quiet || bundle --quiet` + end + + def clean_head + system('git reset --hard --quiet') + end + end + include GitCommands + + def run_revisions(ref1: nil, ref2: nil, cmd:) + ref0 = current_branch + ref1 ||= current_branch + ref2 ||= 'master' + reports = {} + + revisions(ref1, ref2).each do |rev| + STDERR.puts "Checking out: #{revision_description(rev)}" + + reports[rev] = run_at_ref(rev, cmd) + clean_head + end + checkout_ref(ref0) + debug { "OK for all revisions!" } + reports + rescue Exception # rubocop:disable Lint/RescueException + STDERR.puts $!.message + checkout_ref(ref0) + raise + ensure + return reports + end + + def run_at_ref(ref, cmd) + checkout_ref(ref) + bundle + base = Shellwords.shellescape(TMP_DIR_BASE) + cmd = "COMMIT_HASH=#{ref} BASE=#{base} #{Shellwords.shelljoin(cmd)}" + cmd.sub('bin/bench', 'tmp/revision_runner/bench') + debug { cmd } + system(cmd) + $CHILD_STATUS.exitstatus + end + + # TODO: useful? + # def restart_server + # server_script = File.join(TMP_DIR_BASE, 'serve_dummy') + # system("#{server_script} stop") + # at_exit { system("#{server_script} stop") } + # config_ru = Shellwords.shellescape(File.join(TMP_DIR_BASE, 'config.ru')) + # pid = `CONFIG_RU=#{config_ru} #{server_script} start`.chomp + # abort "No pid" if pid.empty? + # pid = Integer(pid) + # Process.kill(0, pid) # confirm running, else it raises + # end + + def debug(msg = '') + if block_given? && ENV['DEBUG'] =~ /\Atrue|on|0\z/i + STDOUT.puts yield + else + STDOUT.puts msg + end + end + +end + +if $PROGRAM_NAME == __FILE__ + runner = RevisionRunner.new + args = ARGV.dup + reports = runner.run_revisions(ref1: args.shift, ref2: args.shift, cmd: args) + reports.each do |name, value| + STDERR.puts "revision: #{name}\n\t#{value}" + end +end diff --git a/bin/serve_dummy b/bin/serve_dummy new file mode 100755 index 000000000..960a7126d --- /dev/null +++ b/bin/serve_dummy @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -e + +case "$1" in + + start) + config="${CONFIG_RU:-test/dummy/config.ru}" + bundle exec ruby -Ilib -S rackup "$config" --daemonize --pid tmp/dummy_app.pid --warn --server webrick + until [ -f 'tmp/dummy_app.pid' ]; do + sleep 0.1 # give it time to start.. I don't know a better way + done + cat tmp/dummy_app.pid + true + ;; + + stop) + if [ -f 'tmp/dummy_app.pid' ]; then + kill -TERM $(cat tmp/dummy_app.pid) + else + echo 'No pidfile' + false + fi + ;; + + status) + if [ -f 'tmp/dummy_app.pid' ]; then + kill -0 $(cat tmp/dummy_app.pid) + [ "$?" -eq 0 ] + else + echo 'No pidfile' + false + fi + ;; + + *) + echo "Usage: $0 [start|stop|status]" + ;; + +esac diff --git a/test/dummy/app.rb b/test/dummy/app.rb new file mode 100644 index 000000000..f19aedc18 --- /dev/null +++ b/test/dummy/app.rb @@ -0,0 +1,87 @@ +# https://github.com/rails-api/active_model_serializers/pull/872 +# approx ref 792fb8a9053f8db3c562dae4f40907a582dd1720 to test against +require 'bundler/setup' + +require 'rails' +require 'active_model' +require 'active_support' +require 'active_support/json' +require 'action_controller' +require 'action_controller/test_case' +require 'action_controller/railtie' +abort "Rails application already defined: #{Rails.application.class}" if Rails.application + +class NullLogger < Logger + def initialize(*_args) + end + + def add(*_args, &_block) + end +end +class DummyLogger < ActiveSupport::Logger + def initialize + @file = StringIO.new + super(@file) + end + + def messages + @file.rewind + @file.read + end +end +# ref: https://gist.github.com/bf4/8744473 +class DummyApp < Rails::Application + # CONFIG: CACHE_ON={on,off} + config.action_controller.perform_caching = ENV['CACHE_ON'] != 'off' + config.action_controller.cache_store = ActiveSupport::Cache.lookup_store(:memory_store) + + # Set up production configuration + config.eager_load = true + config.cache_classes = true + + config.active_support.test_order = :random + config.secret_token = '1234' + config.secret_key_base = 'abc123' + config.logger = NullLogger.new +end + +require 'active_model_serializers' + +# Initialize app before any serializers are defined, for sanity's sake. +# Otherwise, you have to manually set perform caching. +# +# Details: +# +# 1. Upon load, when AMS.config.perform_caching is true, +# serializers inherit the cache store from ActiveModelSerializers.config.cache_store +# 1. If the serializers are loaded before Rails is initialized (`Rails.application.initialize!`), +# these values are nil, and are not applied to the already loaded serializers +# 1. If Rails is initialized before any serializers are loaded, then the configs are set, +# and are used when serializers are loaded +# 1. In either case, `ActiveModelSerializers.config.cache_store`, and +# `ActiveModelSerializers.config.perform_caching` can be set at any time before the serializers +# are loaded, +# e.g. `ActiveModel::Serializer.config.cache_store ||= +# ActiveSupport::Cache.lookup_store(ActionController::Base.cache_store || +# Rails.cache || :memory_store)` +# and `ActiveModelSerializers.config.perform_caching = true` +# 1. If the serializers are loaded before Rails is initialized, then, +# you can set the `_cache` store directly on the serializers. +# `ActiveModel::Serializer._cache ||= +# ActiveSupport::Cache.lookup_store(ActionController::Base.cache_store || +# Rails.cache || :memory_store` +# is sufficient. +# Setting `_cache` to a truthy value will cause the CachedSerializer +# to consider it cached, which will apply to all serializers (bug? :bug: ) +# +# This happens, in part, because the cache store is set for a serializer +# when `cache` is called, and cache is usually called when the serializer is defined. +# +# So, there's now a 'workaround', something to debug, and a starting point. +Rails.application.initialize! + +# HACK: Serializer::cache depends on the ActionController-dependent configs being set. +ActiveSupport.on_load(:action_controller) do + require_relative 'fixtures' +end +require_relative 'controllers' diff --git a/test/dummy/benchmarking_support.rb b/test/dummy/benchmarking_support.rb new file mode 100644 index 000000000..998b83ac2 --- /dev/null +++ b/test/dummy/benchmarking_support.rb @@ -0,0 +1,65 @@ +require 'benchmark' +require 'json' + +module Benchmark + module ActiveModelSerializers + module TestMethods + def request(method, path) + response = Rack::MockRequest.new(DummyApp).send(method, path) + if response.status.in?([404, 500]) + fail "omg, #{method}, #{path}, '#{response.status}', '#{response.body}'" + end + response + end + end + def ams(label = nil, time: 2_000, warmup: 10, &block) + fail ArgumentError.new, 'block should be passed' unless block_given? + + # run_gc + GC.enable + GC.start + GC.disable + warmup.times do + yield + end + measurement = Benchmark.measure do + time.times do + yield + end + end + + user = measurement.utime + system = measurement.stime + total = measurement.total + real = measurement.real + output = { + label: label, + real: real, + total: total, + user: user, + system: system, + version: ::ActiveModel::Serializer::VERSION, + total_allocated_objects_per_measurement: total_allocated_objects(&block) + }.to_json + + puts output + end + + def total_allocated_objects + return unless block_given? + key = + if RUBY_VERSION < '2.2' + :total_allocated_object + else + :total_allocated_objects + end + + before = GC.stat[key] + yield + after = GC.stat[key] + after - before + end + end + + extend Benchmark::ActiveModelSerializers +end diff --git a/test/dummy/bm_caching.rb b/test/dummy/bm_caching.rb new file mode 100644 index 000000000..2aa5a67a6 --- /dev/null +++ b/test/dummy/bm_caching.rb @@ -0,0 +1,112 @@ +require_relative './benchmarking_support' +require_relative './app' + +class ApiAssertion + include Benchmark::ActiveModelSerializers::TestMethods + BadRevisionError = Class.new(StandardError) + + def valid? + caching = get_caching + caching[:body].delete('meta') + non_caching = get_non_caching + non_caching[:body].delete('meta') + assert_responses(caching, non_caching) + rescue BadRevisionError => e + msg = e.message + STDOUT.puts msg + exit 1 + end + + def get_status(on_off = 'on'.freeze) + get("/status/#{on_off}") + end + + def clear + get('/clear') + end + + def get_caching(on_off = 'on'.freeze) + get("/caching/#{on_off}") + end + + def get_non_caching(on_off = 'on'.freeze) + get("/non_caching/#{on_off}") + end + + private + + def assert_responses(caching, non_caching) + assert_equal(caching[:code], 200, "Caching response failed: #{caching}") + assert_equal(caching[:body], expected, "Caching response format failed: \n+ #{caching[:body]}\n- #{expected}") + assert_equal(caching[:content_type], 'application/json; charset=utf-8', "Caching response content type failed: \n+ #{caching[:content_type]}\n- application/json") + assert_equal(non_caching[:code], 200, "Non caching response failed: #{non_caching}") + assert_equal(non_caching[:body], expected, "Non Caching response format failed: \n+ #{non_caching[:body]}\n- #{expected}") + assert_equal(non_caching[:content_type], 'application/json; charset=utf-8', "Non caching response content type failed: \n+ #{non_caching[:content_type]}\n- application/json") + end + + def get(url) + response = request(:get, url) + { code: response.status, body: JSON.load(response.body), content_type: response.content_type } + end + + def expected + @expected ||= + { + 'post' => { + 'id' => 1, + 'title' => 'New Post', + 'body' => 'Body', + 'comments' => [ + { + 'id' => 1, + 'body' => 'ZOMG A COMMENT' + } + ], + 'blog' => { + 'id' => 999, + 'name' => 'Custom blog' + }, + 'author' => { + 'id' => 1, + 'name' => 'Joao Moura.' + } + } + } + end + + def assert_equal(expected, actual, message) + return true if expected == actual + fail BadRevisionError, message + end + + def debug(msg = '') + if block_given? && ENV['DEBUG'] =~ /\Atrue|on|0\z/i + STDOUT.puts yield + else + STDOUT.puts msg + end + end +end +assertion = ApiAssertion.new +assertion.valid? + +# STDERR.puts assertion.get_status +Benchmark.ams('caching on: caching serializers') do + assertion.get_caching('on') +end +# STDERR.puts assertion.get_status +assertion.clear +Benchmark.ams('caching off: caching serializers') do + assertion.get_caching('off') +end +# STDERR.puts assertion.get_status +assertion.clear +Benchmark.ams('caching on: non-caching serializers') do + assertion.get_non_caching('on') +end +# STDERR.puts assertion.get_status +assertion.clear +Benchmark.ams('caching off: non-caching serializers') do + assertion.get_non_caching('off') +end +# STDERR.puts assertion.get_status diff --git a/test/dummy/config.ru b/test/dummy/config.ru new file mode 100644 index 000000000..908eb28c4 --- /dev/null +++ b/test/dummy/config.ru @@ -0,0 +1,3 @@ +require File.expand_path(['..', 'app'].join(File::SEPARATOR), __FILE__) + +run Rails.application diff --git a/test/dummy/controllers.rb b/test/dummy/controllers.rb new file mode 100644 index 000000000..8b1fda8bf --- /dev/null +++ b/test/dummy/controllers.rb @@ -0,0 +1,69 @@ +class PostController < ActionController::Base + POST = + begin + comment = Comment.new(id: 1, body: 'ZOMG A COMMENT') + author = Author.new(id: 1, name: 'Joao Moura.') + Post.new(id: 1, title: 'New Post', blog: nil, body: 'Body', comments: [comment], author: author) + end + + def render_with_caching_serializer + toggle_cache_status + render json: POST, serializer: CachingPostSerializer, adapter: :json, meta: { caching: perform_caching } + end + + def render_with_non_caching_serializer + toggle_cache_status + render json: POST, adapter: :json, meta: { caching: perform_caching } + end + + def render_cache_status + toggle_cache_status + # Uncomment to debug + # STDERR.puts cache_store.class + # STDERR.puts cache_dependencies + # ActiveSupport::Cache::Store.logger.debug [ActiveModelSerializers.config.cache_store, ActiveModelSerializers.config.perform_caching, CachingPostSerializer._cache, perform_caching, params].inspect + render json: { caching: perform_caching, meta: { cache_log: cache_messages, cache_status: cache_status } }.to_json + end + + def clear + ActionController::Base.cache_store.clear + # Test caching is on + # logger = DummyLogger.new + # ActiveSupport::Cache::Store.logger = logger # seems to be the best way + # the below is used in some rails tests but isn't available/working in all versions, so far as I can tell + # https://github.com/rails/rails/pull/15943 + # ActiveSupport::Notifications.subscribe(/^cache_(.*)\.active_support$/) do |*args| + # logger.debug ActiveSupport::Notifications::Event.new(*args) + # end + render json: 'ok'.to_json + end + + private + + def cache_status + { + controller: perform_caching, + app: Rails.configuration.action_controller.perform_caching, + serializers: Rails.configuration.serializers.each_with_object({}) { |serializer, data| data[serializer.name] = serializer._cache.present? } + } + end + + def cache_messages + ActiveSupport::Cache::Store.logger.is_a?(DummyLogger) && ActiveSupport::Cache::Store.logger.messages.split("\n") + end + + def toggle_cache_status + case params[:on] + when 'on'.freeze then self.perform_caching = true + when 'off'.freeze then self.perform_caching = false + else nil # no-op + end + end +end + +Rails.application.routes.draw do + get '/status(/:on)' => 'post#render_cache_status' + get '/clear' => 'post#clear' + get '/caching(/:on)' => 'post#render_with_caching_serializer' + get '/non_caching(/:on)' => 'post#render_with_non_caching_serializer' +end diff --git a/test/dummy/fixtures.rb b/test/dummy/fixtures.rb new file mode 100644 index 000000000..f0795e183 --- /dev/null +++ b/test/dummy/fixtures.rb @@ -0,0 +1,114 @@ +Rails.configuration.serializers = [] +class AuthorSerializer < ActiveModel::Serializer + attributes :id, :name + + has_many :posts, embed: :ids + has_one :bio +end +Rails.configuration.serializers << AuthorSerializer + +class BlogSerializer < ActiveModel::Serializer + attributes :id, :name +end +Rails.configuration.serializers << BlogSerializer + +class CommentSerializer < ActiveModel::Serializer + attributes :id, :body + + belongs_to :post + belongs_to :author +end +Rails.configuration.serializers << CommentSerializer + +class PostSerializer < ActiveModel::Serializer + attributes :id, :title, :body + + has_many :comments, serializer: CommentSerializer + belongs_to :blog, serializer: BlogSerializer + belongs_to :author, serializer: AuthorSerializer + + def blog + Blog.new(id: 999, name: 'Custom blog') + end +end +Rails.configuration.serializers << PostSerializer + +class CachingAuthorSerializer < AuthorSerializer + cache key: 'writer', only: [:name], skip_digest: true +end +Rails.configuration.serializers << CachingAuthorSerializer + +class CachingCommentSerializer < CommentSerializer + cache expires_in: 1.day, skip_digest: true +end +Rails.configuration.serializers << CachingCommentSerializer + +class CachingPostSerializer < PostSerializer + cache key: 'post', expires_in: 0.1, skip_digest: true + belongs_to :blog, serializer: BlogSerializer + belongs_to :author, serializer: CachingAuthorSerializer + has_many :comments, serializer: CachingCommentSerializer +end +Rails.configuration.serializers << CachingPostSerializer + +# ActiveModelSerializers::Model is a convenient +# serializable class to inherit from when making +# serializable non-activerecord objects. +class DummyModel + include ActiveModel::Model + include ActiveModel::Serializers::JSON + + attr_reader :attributes + + def initialize(attributes = {}) + @attributes = attributes + super + end + + # Defaults to the downcased model name. + def id + attributes.fetch(:id) { self.class.name.downcase } + end + + # Defaults to the downcased model name and updated_at + def cache_key + attributes.fetch(:cache_key) { "#{self.class.name.downcase}/#{id}-#{updated_at.strftime("%Y%m%d%H%M%S%9N")}" } + end + + # Defaults to the time the serializer file was modified. + def updated_at + attributes.fetch(:updated_at) { File.mtime(__FILE__) } + end + + def read_attribute_for_serialization(key) + if key == :id || key == 'id' + attributes.fetch(key) { id } + else + attributes[key] + end + end +end + +class Comment < DummyModel + attr_accessor :id, :body + + def cache_key + "#{self.class.name.downcase}/#{id}" + end +end + +class Author < DummyModel + attr_accessor :id, :name, :posts +end + +class Post < DummyModel + attr_accessor :id, :title, :body, :comments, :blog, :author + + def cache_key + 'benchmarking::post/1-20151215212620000000000' + end +end + +class Blog < DummyModel + attr_accessor :id, :name +end