@@ -15,3 +15,4 @@ lib/coverage.html *.rbc _Yardoc .yardoc +tmp/* .gitignore @@ -17,6 +17,7 @@ Pathname.glob(ROOT + 'tasks/**/*.rb') { |t| require t } task :default => 'dm:spec' task :spec => 'dm:spec' task :rcov => 'dm:rcov' +task :perf => 'dm:perf:all' namespace :spec do task :unit => 'dm:spec:unit' @@ -66,8 +67,8 @@ namespace :dm do desc "Run integration specifications with rcov" run_spec('integration', integration_specs, true) end - - desc "Run comparison with ActiveRecord" + + desc "Run all comparisons with ActiveRecord" task :perf do sh ROOT + 'script/performance.rb' end Rakefile @@ -2,8 +2,13 @@ require File.join(File.dirname(__FILE__), '..', 'lib', 'dm-core') -require 'benchmark' require 'rubygems' +require 'ftools' + +# gem install somebee-rbench -s http://gems.github.com +# OR git clone git://github.com/somebee/rbench.git; rake install +# gem 'somebee-rbench', '>=0.2.0' +require 'rbench' gem 'faker', '>=0.3.1' require 'faker' @@ -34,6 +39,7 @@ log_dir.mkdir unless log_dir.directory? DataMapper::Logger.new(log_dir / 'dm.log', :debug) adapter = DataMapper.setup(:default, "mysql://root@localhost/data_mapper_1?socket=#{socket_file}") +sqlfile = File.join(File.dirname(__FILE__),'..','tmp','perf.sql') ActiveRecord::Base.logger = Logger.new(log_dir / 'ar.log') ActiveRecord::Base.logger.level = 0 @@ -52,7 +58,7 @@ class Exhibit property :id, Integer, :serial => true property :name, String property :zoo_id, Integer - property :notes, DM::Text, :lazy => true + property :notes, Text, :lazy => true property :created_on, Date # property :updated_at, DateTime end @@ -66,221 +72,159 @@ touch_attributes = lambda do |exhibits| end end -exhibits = [] - -# pre-compute the insert statements and fake data compilation, -# so the benchmarks below show the actual runtime for the execute -# method, minus the setup steps -10_000.times do - exhibits << [ -# 'INSERT INTO `exhibits` (`name`, `zoo_id`, `notes`, `created_on`, `updated_at`) VALUES (?, ?, ?, ?, ?)' - 'INSERT INTO `exhibits` (`name`, `zoo_id`, `notes`, `created_on`) VALUES (?, ?, ?, ?)', - Faker::Company.name, - rand(10).ceil, - Faker::Lorem.paragraphs.join($/), - Date.today, -# Time.now, - ] -end - -Benchmark.bmbm(60) do |x| - # reset the exhibits - x.report do - Exhibit.auto_migrate! - end - - x.report('DO.execute insert x10,000') do - 10_000.times { |i| adapter.execute(*exhibits.at(i)) } - end - - x.report('AR.id x10,000') do - ActiveRecord::Base::uncached do - 10_000.times { touch_attributes[ARExhibit.find(1)] } - end - end - - x.report('DM.id x10,000') do - 10_000.times { touch_attributes[Exhibit.get(1)] } - end - - x.report('AR.id x10,000 (cached)') do - ActiveRecord::Base::cache do - 10_000.times { touch_attributes[ARExhibit.find(1)] } - end - end - - x.report('DM.id x10,000 (cached)') do - repository(:default) do - 10_000.times { touch_attributes[Exhibit.get(1)] } - end - end - - x.report('AR.first x10,000') do - 10_000.times { touch_attributes[ARExhibit.find(:first)] } - end - - x.report('DM.first x10,000') do - 10_000.times { touch_attributes[Exhibit.first] } - end - - x.report('AR.all limit(100) x1,000') do - ActiveRecord::Base::uncached do - 1000.times { touch_attributes[ARExhibit.find(:all, :limit => 100)] } - end - end - - x.report('DM.all limit(100) x1,000') do - 1000.times { touch_attributes[Exhibit.all(:limit => 100)] } - end - - x.report('AR.all limit(100) x1,000 (cached)') do - ActiveRecord::Base::cache do - 1000.times { touch_attributes[ARExhibit.find(:all, :limit => 100)] } - end - end - - x.report('DM.all limit(100) x1,000 (cached)') do - repository(:default) do - 1000.times { touch_attributes[Exhibit.all(:limit => 100)] } - end - end - - x.report('AR.all limit(10,000) x10') do - ActiveRecord::Base::uncached do - 10.times { touch_attributes[ARExhibit.find(:all, :limit => 10_000)] } - end - end - - x.report('DM.all limit(10,000) x10') do - 10.times { touch_attributes[Exhibit.all(:limit => 10_000)] } - end - x.report('AR.all limit(10,000) x10 (cached)') do - ActiveRecord::Base::cache do - 10.times { touch_attributes[ARExhibit.find(:all, :limit => 10_000)] } +c = configuration_options + +if File.exists?(sqlfile) && configuration_options[:adapter] == 'mysql' + puts "Found data-file. Importing from #{sqlfile}" + `mysql -u #{c[:username]} #{"-p#{c[:password]}" unless c[:password].blank?} #{c[:database]} < #{sqlfile}` +else + + Exhibit.auto_migrate! + + exhibits = [] + # pre-compute the insert statements and fake data compilation, + # so the benchmarks below show the actual runtime for the execute + # method, minus the setup steps + 10_000.times do + exhibits << [ + 'INSERT INTO `exhibits` (`name`, `zoo_id`, `notes`, `created_on`) VALUES (?, ?, ?, ?)', + Faker::Company.name, + rand(10).ceil, + Faker::Lorem.paragraphs.join($/), + Date.today + ] + end + 10_000.times { |i| adapter.execute(*exhibits.at(i)) } + + if configuration_options[:adapter] == 'mysql' + answer = nil + until answer && answer[/^$|y|yes|n|no/] + print("Would you like to dump data into tmp/perf.sql (for faster setup)? [Yn]"); + STDOUT.flush + answer = gets end - end - - x.report('DM.all limit(10,000) x10 (cached)') do - repository(:default) do - 10.times { touch_attributes[Exhibit.all(:limit => 10_000)] } + + if answer[/^$|y|yes/] + File.makedirs(File.dirname(sqlfile)) + `mysqldump -u #{c[:username]} #{"-p#{c[:password]}" unless c[:password].blank?} #{c[:database]} exhibits > #{sqlfile}` + puts "File saved\n" end end + +end - # Static, just so AR and DM are on equal footing. +TIMES = ENV['x'] ? ENV['x'].to_i : 10_000 + +puts "You can specify how many times you want to run the benchmarks with rake:perf x=(number)" +puts "Some tasks will be run 10 and 1000 times less than (number)" +puts "Benchmarks will now run #{TIMES} times" + +RBench.run(TIMES) do + + column :times + column :dm, :title => "DM 0.9.2" + column :ar, :title => "AR 2.1" + column :diff, :compare => [:dm,:ar] + + report "#get specific (not cached)" do + dm { touch_attributes[Exhibit.get(1)] } + ActiveRecord::Base::uncached { ar { touch_attributes[ARExhibit.find(1)] } } + end + + report "#get specific (cached)" do + Exhibit.repository(:default) { dm { touch_attributes[Exhibit.get(1)] } } + ActiveRecord::Base::cache { ar { touch_attributes[ARExhibit.find(1)] } } + end + + report "#get first" do + dm { touch_attributes[Exhibit.first] } + ar { touch_attributes[ARExhibit.first] } + end + + report "#all limit(100) (not cached)", TIMES / 10 do + dm { touch_attributes[Exhibit.all(:limit => 100)] } + ar { touch_attributes[ARExhibit.find(:all, :limit => 100)] } + end + + report "#all limit(100) (cached)", TIMES / 10 do + Exhibit.repository(:default) { dm { touch_attributes[Exhibit.all(:limit => 100)] } } + ActiveRecord::Base::cache { ar { touch_attributes[ARExhibit.find(:all, :limit => 100)] } } + end + + report "#all limit(10,000) (not cached)", TIMES / 1000 do + dm { touch_attributes[Exhibit.all(:limit => 10_000)] } + ar { touch_attributes[ARExhibit.find(:all, :limit => 10_000)] } + end + + report "#all limit(10,000) (cached)", TIMES / 1000 do + Exhibit.repository(:default) { dm { touch_attributes[Exhibit.all(:limit => 10_000)] } } + ActiveRecord::Base::cache { ar { touch_attributes[ARExhibit.find(:all, :limit => 10_000)] } } + end + create_exhibit = { :name => Faker::Company.name, :zoo_id => rand(10).ceil, :notes => Faker::Lorem.paragraphs.join($/), - :created_on => Date.today, -# :updated_at => Time.now, + :created_on => Date.today } - - x.report('AR.create x10,000') do - 10_000.times { ARExhibit.create(create_exhibit) } - end - - x.report('DM.create x10,000') do - 10_000.times { Exhibit.create(create_exhibit) } + + report "#create" do + dm { Exhibit.create(create_exhibit) } + ar { ARExhibit.create(create_exhibit) } end - x.report('AR#update x10,000') do - 10_000.times { e = ARExhibit.find(1); e.name = 'bob'; e.save } + report "#update" do + dm { e = Exhibit.get(1); e.name = 'bob'; e.save } + ar { e = ARExhibit.find(1); e.name = 'bob'; e.save } end - - x.report('DM#update x10,000') do - 10_000.times { e = Exhibit.get(1); e.name = 'bob'; e.save } - end - - x.report('AR#destroy x10,000') do - # destroy records 1 to 10,000 - (1..10_000).each { |id| ARExhibit.find(id).destroy } - end - - x.report('DM#destroy x10,000') do - # destroy records 10,001 to 20,000 - (10_001..20_000).each { |id| Exhibit.get(id).destroy } + + report "#destroy" do + dm { Exhibit.first.destroy } + ar { ARExhibit.first.destroy } end + + summary "Total" + end -# connection = adapter.create_connection -# command = connection.create_command("DROP TABLE exhibits") -# command.execute_non_query rescue nil -# connection.close +connection = adapter.send(:create_connection) +command = connection.create_command("DROP TABLE exhibits") +command.execute_non_query rescue nil +connection.close __END__ -On an iMac Core2Duo 2.16GHz: - -I don't think AR is actually caching. - -~/src/dm-core > script/performance.rb -Text should not be declared inline. -Rehearsal ----------------------------------------------------------------------------------------------- -ActiveRecord:id x10_000 3.530000 0.320000 3.850000 ( 4.939399) -ActiveRecord:id(cached) x10_000 3.540000 0.320000 3.860000 ( 4.797453) -DataMapper:id x10_000 3.190000 0.280000 3.470000 ( 4.604910) -DataMapper:id(cached) x10_000 0.090000 0.000000 0.090000 ( 0.095606) -ActiveRecord:all limit(100) x1000 11.880000 0.060000 11.940000 ( 12.494454) -ActiveRecord:all limit(100) (cached) x1000 11.860000 0.060000 11.920000 ( 12.521918) -DataMapper:all limit(100) x1000 5.020000 0.040000 5.060000 ( 5.578594) -DataMapper:all limit(100) (cached) x1000 3.040000 0.040000 3.080000 ( 3.581179) -ActiveRecord:all limit(10,000) x10 12.040000 0.050000 12.090000 ( 12.243110) -ActiveRecord:all limit(10,000) (cached) x10 12.140000 0.040000 12.180000 ( 12.339861) -DataMapper:all limit(10,000) x10 5.740000 0.040000 5.780000 ( 5.788256) -DataMapper:all limit(10,000) (cached) x10 3.850000 0.020000 3.870000 ( 3.874642) -------------------------------------------------------------------------------------- total: 77.190000sec - - user system total real -ActiveRecord:id x10_000 3.480000 0.320000 3.800000 ( 4.701110) -ActiveRecord:id(cached) x10_000 3.480000 0.310000 3.790000 ( 4.701436) -DataMapper:id x10_000 3.170000 0.270000 3.440000 ( 4.375630) -DataMapper:id(cached) x10_000 0.100000 0.000000 0.100000 ( 0.097554) -ActiveRecord:all limit(100) x1000 11.410000 0.060000 11.470000 ( 11.993949) -ActiveRecord:all limit(100) (cached) x1000 11.410000 0.050000 11.460000 ( 11.998727) -DataMapper:all limit(100) x1000 5.060000 0.040000 5.100000 ( 5.608384) -DataMapper:all limit(100) (cached) x1000 3.020000 0.040000 3.060000 ( 3.554985) -ActiveRecord:all limit(10,000) x10 12.170000 0.040000 12.210000 ( 12.370468) -ActiveRecord:all limit(10,000) (cached) x10 12.180000 0.040000 12.220000 ( 12.371510) -DataMapper:all limit(10,000) x10 5.450000 0.020000 5.470000 ( 5.480993) -DataMapper:all limit(10,000) (cached) x10 3.130000 0.020000 3.150000 ( 3.160792) - -On a MacBook Air Core2Duo 1.6GHz (many performance optimizations since the run above) - -~/src/dm-core > script/performance.rb -Rehearsal ----------------------------------------------------------------------------------------------- -AR.id x10,000 11.290000 0.420000 11.710000 ( 14.845046) -DM.id x10,000 5.010000 0.480000 5.490000 ( 8.738212) -AR.id x10,000 (cached) 12.300000 0.520000 12.820000 ( 16.585108) -DM.id x10,000 (cached) 0.450000 0.000000 0.450000 ( 0.474010) -AR.all limit(100) x1,000 3.340000 0.080000 3.420000 ( 4.554041) -DM.all limit(100) x1,000 1.420000 0.050000 1.470000 ( 1.743724) -AR.all limit(100) x1,000 (cached) 3.450000 0.070000 3.520000 ( 3.937980) -DM.all limit(100) x1,000 (cached) 1.070000 0.050000 1.120000 ( 1.390889) -AR.all limit(10,000) x10 0.020000 0.000000 0.020000 ( 0.031253) -DM.all limit(10,000) x10 0.020000 0.000000 0.020000 ( 0.017896) -AR.all limit(10,000) x10 (cached) 0.030000 0.000000 0.030000 ( 0.032688) -DM.all limit(10,000) x10 (cached) 0.010000 0.000000 0.010000 ( 0.013167) -AR.create x10,000 21.390000 1.290000 22.680000 ( 28.519556) -DM.create x10,000 20.090000 0.630000 20.720000 ( 28.822219) -AR.update x10,000 25.450000 1.660000 27.110000 ( 33.040779) -DM.update x10,000 6.550000 0.780000 7.330000 ( 10.467941) ------------------------------------------------------------------------------------- total: 117.920000sec + | DM 0.9.2 | AR 2.1 | DIFF | +-------------------------------------------------------------------------------- +get specific (not cached) x10000 | 8.949 | 3.108 | 2.88x | +get specific (cached) x10000 | 0.223 | 3.090 | 0.07x | +get first x10000 | 6.987 | 2.756 | 2.54x | +#all limit(100) (not cached) x1000 | 6.625 | 7.734 | 0.86x | +#all limit(100) (cached) x1000 | 4.828 | 7.686 | 0.63x | +#all limit(10,000) (not cached) x10 | 6.252 | 9.145 | 0.68x | +#all limit(10,000) (cached) x10 | 5.001 | 9.368 | 0.53x | +#create x10000 | 38.884 | 21.757 | 1.79x | +#update x10000 | 9.102 | 6.790 | 1.34x | +#destroy x10000 | 30.391 | 21.502 | 1.41x | +================================================================================ +Total | 117.242 | 92.936 | 1.26x | + + +## PROBLEMS WITH THE RECENT DAYS COMMITS !!! ## + + | DM 0.9.2 | AR 2.1 | DIFF | +-------------------------------------------------------------------------------- +#get specific (not cached) x1000 | 12.490 | 0.309 | 40.43x | +#get specific (cached) x1000 | 3.980 | 0.305 | 13.03x | +#get first x1000 | 10.087 | 0.277 | 36.39x | +#all limit(100) (not cached) x100 | 59.252 | 0.780 | 75.92x | +#all limit(100) (cached) x100 | 41.236 | 0.774 | 53.24x | +#all limit(10,000) (not cached) x1 | 71.123 | 0.826 | 86.12x | +#all limit(10,000) (cached) x1 | 72.477 | 0.835 | 86.79x | +#create x1000 | 24.836 | 2.219 | 11.19x | +#update x1000 | 9.866 | 0.645 | 15.31x | +#destroy x1000 | 12.892 | 1.512 | 8.52x | +================================================================================ +Total 0 | 318.237 | 8.483 | 42.69x | - user system total real -AR.id x10,000 11.320000 0.420000 11.740000 ( 13.907577) -DM.id x10,000 4.510000 0.410000 4.920000 ( 6.611278) -AR.id x10,000 (cached) 11.680000 0.440000 12.120000 ( 14.792082) -DM.id x10,000 (cached) 0.380000 0.010000 0.390000 ( 0.384611) -AR.all limit(100) x1,000 25.010000 0.200000 25.210000 ( 25.954076) -DM.all limit(100) x1,000 10.370000 0.090000 10.460000 ( 11.558341) -AR.all limit(100) x1,000 (cached) 28.390000 0.350000 28.740000 ( 30.870503) -DM.all limit(100) x1,000 (cached) 7.760000 0.110000 7.870000 ( 9.211700) -AR.all limit(10,000) x10 27.850000 0.240000 28.090000 ( 28.510220) -DM.all limit(10,000) x10 10.970000 0.040000 11.010000 ( 11.042093) -AR.all limit(10,000) x10 (cached) 29.230000 0.170000 29.400000 ( 29.816547) -DM.all limit(10,000) x10 (cached) 7.470000 0.030000 7.500000 ( 7.520561) -AR.create x10,000 21.700000 1.400000 23.100000 ( 27.188424) -DM.create x10,000 24.380000 0.550000 24.930000 ( 29.439910) -AR.update x10,000 30.670000 1.910000 32.580000 ( 39.227956) -DM.update x10,000 8.290000 0.950000 9.240000 ( 12.912708) script/performance.rb cdd64b733f862b6cb73abc080742d6cc5e23cbc4 Sindre Aarsaether sindre@Sindres-MacPro.local http://github.com/dkubb/dm-core/commit/747f1e76e2ece665d38b6582b757585e74b98c3a 747f1e76e2ece665d38b6582b757585e74b98c3a 2008-06-16T07:58:49-07:00 2008-06-16T07:58:49-07:00 updated performance-scripts. will now cache db-dump for faster initializing if accepted, and display the results so that its easier to compare with AR. You can now also specify number of times to run with rake dm:perf x=(times). Depends on rbench: sudo gem install somebee-rbench -s http://gems.github.com 7e771413b876e356a00468d069d15e6a44cef64d Sindre Aarsaether sindre@Sindres-MacPro.local