Skip to content
This repository
Browse code

Performance: integration test benchmarking and profiling. [Jeremy Kem…

…per]
  • Loading branch information...
commit eab71208db1afead6803501c8d51d77625e5ad6e 1 parent ba0f38f
Jeremy Kemper authored
9  actionpack/lib/action_controller/integration.rb
... ...
@@ -1,9 +1,10 @@
1  
-require 'stringio'
2  
-require 'uri'
3  
-
  1
+require 'active_support/test_case'
4 2
 require 'action_controller/dispatcher'
5 3
 require 'action_controller/test_process'
6 4
 
  5
+require 'stringio'
  6
+require 'uri'
  7
+
7 8
 module ActionController
8 9
   module Integration #:nodoc:
9 10
     # An integration Session instance represents a set of requests and responses
@@ -580,7 +581,7 @@ def method_missing(sym, *args, &block)
580 581
   #         end
581 582
   #       end
582 583
   #   end
583  
-  class IntegrationTest < Test::Unit::TestCase
  584
+  class IntegrationTest < ActiveSupport::TestCase
584 585
     include Integration::Runner
585 586
 
586 587
     # Work around a bug in test/unit caused by the default test being named
16  actionpack/lib/action_controller/performance_test.rb
... ...
@@ -0,0 +1,16 @@
  1
+require 'action_controller/integration'
  2
+require 'active_support/testing/performance'
  3
+require 'active_support/testing/default'
  4
+
  5
+module ActionController
  6
+  # An integration test that runs a code profiler on your test methods.
  7
+  # Profiling output for combinations of each test method, measurement, and
  8
+  # output format are written to your tmp/performance directory.
  9
+  #
  10
+  # By default, process_time is measured and both flat and graph_html output
  11
+  # formats are written, so you'll have two output files per test method.
  12
+  class PerformanceTest < ActionController::IntegrationTest
  13
+    include ActiveSupport::Testing::Performance
  14
+    include ActiveSupport::Testing::Default
  15
+  end
  16
+end
226  activesupport/lib/active_support/testing/performance.rb
... ...
@@ -0,0 +1,226 @@
  1
+require 'rubygems'
  2
+gem 'ruby-prof', '>= 0.6.1'
  3
+require 'ruby-prof'
  4
+
  5
+require 'fileutils'
  6
+require 'rails/version'
  7
+
  8
+module ActiveSupport
  9
+  module Testing
  10
+    module Performance
  11
+      benchmark = ARGV.include?('--benchmark')  # HAX for rake test
  12
+
  13
+      DEFAULTS = {
  14
+        :benchmark => benchmark,
  15
+        :runs => benchmark ? 20 : 4,
  16
+        :min_percent => 0.05,
  17
+        :metrics => [:process_time, :memory, :allocations],
  18
+        :formats => [:flat, :graph_html, :call_tree],
  19
+        :output => 'tmp/performance' }
  20
+
  21
+      def self.included(base)
  22
+        base.extend ClassMethods
  23
+        base.class_inheritable_accessor :profile_options
  24
+        base.profile_options = DEFAULTS.dup
  25
+      end
  26
+
  27
+      def run(result)
  28
+        return if method_name =~ /^default_test$/
  29
+
  30
+        yield(self.class::STARTED, name)
  31
+        @_result = result
  32
+
  33
+        run_warmup
  34
+
  35
+        self.class.measure_modes.each do |measure_mode|
  36
+          data = run_profile(measure_mode)
  37
+          self.class.report_profile_total(data, measure_mode)
  38
+          self.class.record_results(full_test_name, data, measure_mode)
  39
+          result.add_run
  40
+        end
  41
+
  42
+        yield(self.class::FINISHED, name)
  43
+      end
  44
+
  45
+      protected
  46
+        def full_test_name
  47
+          "#{self.class.name}##{@method_name}"
  48
+        end
  49
+
  50
+        def run_test
  51
+          run_callbacks :setup
  52
+          setup
  53
+          yield
  54
+        rescue ::Test::Unit::AssertionFailedError => e
  55
+          add_failure(e.message, e.backtrace)
  56
+        rescue StandardError, ScriptError
  57
+          add_error($!)
  58
+        ensure
  59
+          begin
  60
+            teardown
  61
+            run_callbacks :teardown, :enumerator => :reverse_each
  62
+          rescue ::Test::Unit::AssertionFailedError => e
  63
+            add_failure(e.message, e.backtrace)
  64
+          rescue StandardError, ScriptError
  65
+            add_error($!)
  66
+          end
  67
+        end
  68
+
  69
+        def run_warmup
  70
+          puts
  71
+          print full_test_name
  72
+
  73
+          run_test do
  74
+            bench = Benchmark.realtime do
  75
+              __send__(@method_name)
  76
+            end
  77
+            puts " (%.2fs warmup)" % bench
  78
+          end
  79
+        end
  80
+
  81
+        def run_profile(measure_mode)
  82
+          RubyProf.benchmarking = profile_options[:benchmark]
  83
+          RubyProf.measure_mode = measure_mode
  84
+
  85
+          print '  '
  86
+          profile_options[:runs].times do |i|
  87
+            run_test do
  88
+              begin
  89
+                GC.disable
  90
+                RubyProf.resume { __send__(@method_name) }
  91
+                print '.'
  92
+                $stdout.flush
  93
+              ensure
  94
+                GC.enable
  95
+              end
  96
+            end
  97
+          end
  98
+
  99
+          RubyProf.stop
  100
+        end
  101
+
  102
+      module ClassMethods
  103
+        def record_results(test_name, data, measure_mode)
  104
+          if RubyProf.benchmarking?
  105
+            record_benchmark(test_name, data, measure_mode)
  106
+          else
  107
+            record_profile(test_name, data, measure_mode)
  108
+          end
  109
+        end
  110
+
  111
+        def report_profile_total(data, measure_mode)
  112
+          total_time =
  113
+            if RubyProf.benchmarking?
  114
+              data
  115
+            else
  116
+              data.threads.values.sum(0) do |method_infos|
  117
+                method_infos.sort.last.total_time
  118
+              end
  119
+            end
  120
+
  121
+          format =
  122
+            case measure_mode
  123
+              when RubyProf::PROCESS_TIME, RubyProf::WALL_TIME
  124
+                "%.2f seconds"
  125
+              when RubyProf::MEMORY
  126
+                "%.2f bytes"
  127
+              when RubyProf::ALLOCATIONS
  128
+                "%d allocations"
  129
+              else
  130
+                "%.2f #{measure_mode}"
  131
+            end
  132
+
  133
+          total = format % total_time
  134
+          puts "\n  #{ActiveSupport::Testing::Performance::Util.metric_name(measure_mode)}: #{total}\n"
  135
+        end
  136
+
  137
+        def measure_modes
  138
+          ActiveSupport::Testing::Performance::Util.measure_modes(profile_options[:metrics])
  139
+        end
  140
+
  141
+        def printer_classes
  142
+          ActiveSupport::Testing::Performance::Util.printer_classes(profile_options[:formats])
  143
+        end
  144
+
  145
+        private
  146
+          def record_benchmark(test_name, data, measure_mode)
  147
+            bench_filename = "#{profile_options[:output]}/benchmarks.csv"
  148
+
  149
+            if new_file = !File.exist?(bench_filename)
  150
+              FileUtils.mkdir_p(File.dirname(bench_filename))
  151
+            end
  152
+
  153
+            File.open(bench_filename, 'ab') do |file|
  154
+              if new_file
  155
+                file.puts 'test,metric,measurement,runs,average,created_at,rails_version,ruby_engine,ruby_version,ruby_patchlevel,ruby_platform'
  156
+              end
  157
+
  158
+              file.puts [test_name,
  159
+                ActiveSupport::Testing::Performance::Util.metric_name(measure_mode),
  160
+                data, profile_options[:runs], data / profile_options[:runs],
  161
+                Time.now.utc.xmlschema,
  162
+                Rails::VERSION::STRING,
  163
+                defined?(RUBY_ENGINE) ? RUBY_ENGINE : 'ruby',
  164
+                RUBY_VERSION, RUBY_PATCHLEVEL, RUBY_PLATFORM].join(',')
  165
+            end
  166
+          end
  167
+
  168
+          def record_profile(test_name, data, measure_mode)
  169
+            printer_classes.each do |printer_class|
  170
+              fname = output_filename(test_name, printer, measure_mode)
  171
+
  172
+              FileUtils.mkdir_p(File.dirname(fname))
  173
+              File.open(fname, 'wb') do |file|
  174
+                printer_class.new(data).print(file, profile_printer_options)
  175
+              end
  176
+            end
  177
+          end
  178
+
  179
+          # The report filename is test_name + measure_mode + report_type
  180
+          def output_filename(test_name, printer, measure_mode)
  181
+            suffix =
  182
+              case printer
  183
+                when RubyProf::FlatPrinter; 'flat.txt'
  184
+                when RubyProf::GraphPrinter; 'graph.txt'
  185
+                when RubyProf::GraphHtmlPrinter; 'graph.html'
  186
+                when RubyProf::CallTreePrinter; 'tree.txt'
  187
+                else printer.to_s.downcase
  188
+              end
  189
+
  190
+            "#{profile_options[:output]}/#{test_name}_#{ActiveSupport::Testing::Performance::Util.metric_name(measure_mode)}_#{suffix}"
  191
+          end
  192
+
  193
+          def profile_printer_options
  194
+            profile_options.slice(:min_percent)
  195
+          end
  196
+      end
  197
+
  198
+      module Util
  199
+        extend self
  200
+
  201
+        def metric_name(measure_mode)
  202
+          case measure_mode
  203
+            when RubyProf::PROCESS_TIME; 'process_time'
  204
+            when RubyProf::WALL_TIME; 'wall_time'
  205
+            when RubyProf::MEMORY; 'memory'
  206
+            when RubyProf::ALLOCATIONS; 'allocations'
  207
+            else "measure#{measure_mode}"
  208
+          end
  209
+        end
  210
+
  211
+        def measure_modes(metrics)
  212
+          ruby_prof_consts(metrics.map { |m| m.to_s.upcase })
  213
+        end
  214
+
  215
+        def printer_classes(formats)
  216
+          ruby_prof_consts(formats.map { |f| "#{f.to_s.camelize}Printer" })
  217
+        end
  218
+
  219
+        private
  220
+          def ruby_prof_consts(names)
  221
+            names.map { |name| RubyProf.const_get(name) rescue nil }.compact
  222
+          end
  223
+      end
  224
+    end
  225
+  end
  226
+end
11  railties/helpers/performance_test.rb
... ...
@@ -0,0 +1,11 @@
  1
+ENV['RAILS_ENV'] ||= 'test'
  2
+require "#{File.dirname(__FILE__)}/../../config/environment"
  3
+require 'test/unit'
  4
+require 'action_controller/performance_test'
  5
+
  6
+# Profiling results for each test method are written to tmp/performance.
  7
+class BrowsingTest < ActionController::PerformanceTest
  8
+  def test_homepage
  9
+    get '/'
  10
+  end
  11
+end
2  railties/lib/rails_generator/generators/applications/app/app_generator.rb
@@ -51,6 +51,7 @@ def manifest
51 51
       m.template "helpers/application.rb",        "app/controllers/application.rb", :assigns => { :app_name => @app_name, :app_secret => md5.hexdigest }
52 52
       m.template "helpers/application_helper.rb", "app/helpers/application_helper.rb"
53 53
       m.template "helpers/test_helper.rb",        "test/test_helper.rb"
  54
+      m.template "helpers/performance_test.rb",   "test/performance/browsing_test.rb"
54 55
 
55 56
       # database.yml and routes.rb
56 57
       m.template "configs/databases/#{options[:db]}.yml", "config/database.yml", :assigns => {
@@ -155,6 +156,7 @@ def mysql_socket_location
155 156
     test/fixtures
156 157
     test/functional
157 158
     test/integration
  159
+    test/performance
158 160
     test/unit
159 161
     vendor
160 162
     vendor/plugins
15  railties/lib/tasks/testing.rake
@@ -103,6 +103,21 @@ namespace :test do
103 103
   end
104 104
   Rake::Task['test:integration'].comment = "Run the integration tests in test/integration"
105 105
 
  106
+  Rake::TestTask.new(:benchmark) do |t|
  107
+    t.libs << 'test'
  108
+    t.pattern = 'test/performance/**/*_test.rb'
  109
+    t.verbose = true
  110
+    t.options = '-- --benchmark'
  111
+  end
  112
+  Rake::Task['test:benchmark'].comment = 'Benchmark the performance tests'
  113
+
  114
+  Rake::TestTask.new(:profile) do |t|
  115
+    t.libs << 'test'
  116
+    t.pattern = 'test/performance/**/*_test.rb'
  117
+    t.verbose = true
  118
+  end
  119
+  Rake::Task['test:profile'].comment = 'Profile the performance tests'
  120
+
106 121
   Rake::TestTask.new(:plugins => :environment) do |t|
107 122
     t.libs << "test"
108 123
 

0 notes on commit eab7120

Please sign in to comment.
Something went wrong with that request. Please try again.