Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Merge branch 'reload_all_records'

  • Loading branch information...
commit da76a47b230e6584bac0e4d74373955ee0ca9774 2 parents 7f844dd + cd56e56
@sdsykes sdsykes authored
View
56 lib/callsite.rb
@@ -5,8 +5,8 @@ class Callsite
# associations referenced at the callsite.
#
- Mtx = Mutex.new
-
+ Mtx = Mutex.new # mutex should perhaps be per-instance at the expense of a little memory
+
attr_accessor :klass,
:signature,
:columns,
@@ -20,16 +20,15 @@ def initialize( klass, signature )
# Flag a column as seen
#
def column!( column )
- Mtx.synchronize do
- columns << column
- end
+ columns && Mtx.synchronize { @columns << column }
end
# Flag an association as seen
+ # association should be an AssociationReflection object
#
- def association!( association )
- Mtx.synchronize do
- associations << association if preloadable_association?( association )
+ def association!(association, record_id)
+ if preloadable_association?(association.name)
+ associations.register(association, record_id)
end
end
@@ -40,25 +39,48 @@ def inspect
# Lazy init default columns
#
def default_columns
- @default_columns ||= setup_columns
+ @default_columns || Mtx.synchronize { @default_columns = setup_columns }
end
# Lazy init columns
#
def columns
- @columns ||= default_columns.dup
- end
+ @columns || default_columns && Mtx.synchronize { @columns = @default_columns.dup }
+ end
# Lazy init associations
#
def associations
- @associations ||= setup_associations
+ @associations || Mtx.synchronize { @associations = setup_associations }
+ end
+
+ def has_associations?
+ @associations
+ end
+
+ # Analyze previously collected information
+ # and reset ready for a new query
+ #
+ def reset
+ if has_associations?
+ associations.reset
+ end
+ end
+
+ def register_result_set(result_set)
+ if has_associations?
+ associations.register_result_set(result_set)
+ end
end
private
def associations_for_inspect
- associations.map{|a| ":#{a.to_s}" }.join(', ')
+ if has_associations?
+ associations.to_preload.map{|a| ":#{a.to_s}" }.join(', ')
+ else
+ ""
+ end
end
# Only register associations that isn't polymorphic or a collection
@@ -70,7 +92,7 @@ def preloadable_association?( association )
# Is the table a container for STI models ?
#
def inheritable?
- @klass.column_names.include?( inheritance_column )
+ @klass.columns_hash.has_key?( inheritance_column )
end
# Ensure that at least the primary key and optionally the inheritance
@@ -78,16 +100,16 @@ def inheritable?
#
def setup_columns
if inheritable?
- Set.new([primary_key, inheritance_column])
+ SimpleSet.new([primary_key, inheritance_column])
else
- primary_key.blank? ? Set.new : Set.new([primary_key])
+ primary_key.blank? ? SimpleSet.new : SimpleSet.new([primary_key])
end
end
# Start with no registered associations
#
def setup_associations
- Set.new
+ Optimizations::Associations::AssociationSet.new
end
# Memoize a string representation of the inheritance column
View
109 lib/optimizations/associations/association_set.rb
@@ -0,0 +1,109 @@
+module Scrooge
+ module Optimizations
+ module Associations
+
+ # Keeps track of how a result set is used to access associations
+ # Each callsite where associations are accessed will contain one of
+ # these objects.
+ # Each thread will collect data, and we check this data before each
+ # fetch from the database, adding any associations that are needed
+ # which are returned when to_preload is called.
+ #
+ # Note the the association set is only made by scrooge when an
+ # association is accessed, so the first time through the code
+ # data is not collected because we did not record the result set size.
+ #
+ class AssociationSet
+
+ Mtx = Mutex.new
+
+ def initialize
+ @associations = SimpleSet.new
+ @as_data_id = :"association_data_#{object_id}"
+ end
+
+ def register(association, record_id)
+ assoc_data.register(association, record_id)
+ end
+
+ def register_result_set(result_set)
+ assoc_data.register_result_set(result_set)
+ end
+
+ def reset
+ Mtx.synchronize do
+ @associations |= assoc_data.to_preload
+ end
+ assoc_data.reset
+ end
+
+ def to_preload
+ @associations.to_a
+ end
+
+ private
+
+ def assoc_data
+ Thread.current[@as_data_id] ||= AssociationData.new
+ end
+ end
+
+ class AssociationData
+ def initialize
+ reset
+ end
+
+ def reset
+ @associations = {}
+ @result_set_size = 0
+ end
+
+ def register(association, record_id)
+ if @result_set_size > 1
+ assoc = (@associations[association.name] ||= AssociationIdentity.new(association))
+ assoc.register(record_id)
+ end
+ end
+
+ def register_result_set(result_set)
+ @result_set_size = result_set.size
+ end
+
+ def to_preload
+ @associations.values.select { |association| preload_this_assoc?(association) }.map(&:name)
+ end
+
+ private
+
+ # Calculate the benefit of preloading an association
+ # There is no benefit if result set is just one record
+ # Otherwise we look at how many of the result set items were used
+ # to access the association - more than 25% and we preload
+ #
+ # TODO: more rules and analysis for different association types
+ #
+ def preload_this_assoc?(association)
+ if @result_set_size <= 1
+ false
+ else
+ association.accessed_via.size > @result_set_size / 4
+ end
+ end
+ end
+
+ class AssociationIdentity
+ attr_reader :name, :type, :accessed_via
+
+ def initialize(association)
+ @name = association.name
+ @type = association.macro
+ @accessed_via = []
+ end
+
+ def register(record_id)
+ @accessed_via << record_id
+ end
+ end
+ end
+ end
+end
View
18 lib/optimizations/associations/macro.rb
@@ -11,7 +11,7 @@ def install!
unless scrooge_installed?
ActiveRecord::Base.send( :extend, SingletonMethods )
ActiveRecord::Associations::AssociationProxy.send( :include, InstanceMethods )
- end
+ end
end
protected
@@ -34,9 +34,15 @@ def self.extended( base )
end
def preload_scrooge_associations(result_set, callsite_sig)
- scrooge_preloading_exclude do
- callsite_associations = scrooge_callsite(callsite_sig).associations.to_a
- preload_associations(result_set, callsite_associations) unless callsite_associations.empty?
+ if result_set.size > 1
+ scrooge_preloading_exclude do
+ if scrooge_callsite(callsite_sig).has_associations?
+ callsite_associations = scrooge_callsite(callsite_sig).associations.to_preload
+ unless callsite_associations.empty?
+ preload_associations(result_set, callsite_associations)
+ end
+ end
+ end
end
end
@@ -67,7 +73,7 @@ def self.included( base )
# do collections at the moment anyway
#
def load_target_with_scrooge
- scrooge_seen_association!(@reflection.name)
+ scrooge_seen_association!(@reflection)
load_target_without_scrooge
end
@@ -77,7 +83,7 @@ def load_target_with_scrooge
#
def scrooge_seen_association!( association )
if @owner.scrooged? && !@loaded
- @owner.class.scrooge_callsite(callsite_signature).association!(association)
+ @owner.class.scrooge_callsite(callsite_signature).association!(association, @owner.id)
end
end
View
59 lib/optimizations/columns/attributes_proxy.rb
@@ -31,42 +31,24 @@ class ScroogedAttributes < Hash
# Hash container for attributes with scrooge monitoring of attribute access
#
- attr_accessor :callsite_signature, :scrooge_columns, :fully_fetched, :klass, :updateable_result_set
+ attr_accessor :fully_fetched, :klass, :updateable_result_set
- def self.setup(record, scrooge_columns, klass, callsite_signature, updateable_result_set)
- hash = new.replace(record)
- hash.scrooge_columns = scrooge_columns.dup
+ def self.setup(record, klass, updateable_result_set)
+ hash = new.replace(klass.columns_hash.merge(record))
hash.fully_fetched = false
hash.klass = klass
- hash.callsite_signature = callsite_signature
hash.updateable_result_set = updateable_result_set
hash
end
- # Delegate Hash keys to all defined columns
- #
- def keys
- @klass.column_names
- end
-
- # Let #has_key? consider defined columns
- #
- def has_key?(attr_name)
- keys.include?(attr_name.to_s)
- end
-
- alias_method :include?, :has_key?
- alias_method :key?, :has_key?
- alias_method :member?, :has_key?
-
# Lazily augment and load missing attributes
#
def [](attr_name)
- attr_s = attr_name.to_s
- if interesting_for_scrooge?( attr_s )
- augment_callsite!( attr_s )
+ return nil unless has_key?(attr_name)
+ if !scrooge_columns.include?(attr_name)
+ augment_callsite!( attr_name )
fetch_remaining
- @scrooge_columns << attr_s
+ add_to_scrooge_columns(attr_name)
end
super
end
@@ -77,7 +59,7 @@ def fetch(*args, &block)
end
def []=(attr_name, value)
- @scrooge_columns << attr_name.to_s
+ add_to_scrooge_columns(attr_name)
super
end
@@ -111,14 +93,22 @@ def update(hash)
def fetch_remaining
unless @fully_fetched
- columns_to_fetch = @klass.column_names - @scrooge_columns.to_a
+ columns_to_fetch = @klass.column_names - scrooge_columns.to_a
unless columns_to_fetch.empty?
fetch_remaining!( columns_to_fetch )
end
@fully_fetched = true
end
end
-
+
+ def callsite_signature
+ @updateable_result_set.callsite_signature
+ end
+
+ def scrooge_columns
+ @scrooge_columns || @updateable_result_set.scrooge_columns
+ end
+
protected
def fetch_remaining!( columns_to_fetch )
@@ -126,12 +116,15 @@ def fetch_remaining!( columns_to_fetch )
@updateable_result_set.reload_columns!(columns_to_fetch)
end
- def interesting_for_scrooge?( attr_s )
- has_key?(attr_s) && !@scrooge_columns.include?(attr_s)
+ def augment_callsite!( attr_name )
+ @klass.scrooge_seen_column!(callsite_signature, attr_name)
end
- def augment_callsite!( attr_s )
- @klass.scrooge_seen_column!(callsite_signature, attr_s)
+ def add_to_scrooge_columns(attr_name)
+ unless frozen?
+ @scrooge_columns ||= @updateable_result_set.scrooge_columns.dup
+ @scrooge_columns << attr_name
+ end
end
def primary_key_name
@@ -139,7 +132,7 @@ def primary_key_name
end
def dup_self
- @scrooge_columns = @scrooge_columns.dup
+ @scrooge_columns = @scrooge_columns.dup if @scrooge_columns
self
end
end
View
34 lib/optimizations/columns/macro.rb
@@ -63,9 +63,9 @@ def scrooge_reload( p_keys, missing_columns )
#
def scope_with_scrooge?( sql )
sql =~ scrooge_select_regex &&
- column_names.include?(self.primary_key.to_s) &&
+ columns_hash.has_key?(self.primary_key.to_s) &&
sql !~ ScroogeRegexJoin
- end
+ end
private
@@ -73,24 +73,30 @@ def scope_with_scrooge?( sql )
#
def find_by_sql_with_scrooge(sql)
callsite_sig = callsite_signature( caller, callsite_sql( sql ) )
- callsite_columns = scrooge_callsite(callsite_sig).columns
- sql = sql.gsub(scrooge_select_regex, "SELECT #{scrooge_select_sql(callsite_columns)} FROM")
+ callsite = scrooge_callsite(callsite_sig)
+ callsite.reset
+ site_columns = callsite.columns
+
+ sql = sql.gsub(scrooge_select_regex, "SELECT #{scrooge_select_sql(site_columns)} FROM")
+
results = connection.select_all(sanitize_sql(sql), "#{name} Load Scrooged")
result_set = ResultSets::ResultArray.new
- updateable = ResultSets::UpdateableResultSet.new(result_set, self)
- results.inject(result_set) do |memo, record|
- memo << instantiate(ScroogedAttributes.setup(record, callsite_columns, self, callsite_sig, updateable))
- end
+ updateable = ResultSets::UpdateableResultSet.new(result_set, self, callsite_sig, site_columns.dup)
- if Associations::Macro.scrooge_installed?
- preload_scrooge_associations(result_set, callsite_sig)
+ results.collect! do |record|
+ instantiate(ScroogedAttributes.setup(record, self, updateable))
end
+ result_set.replace(results)
+
+ callsite.register_result_set(result_set)
+
+ preload_scrooge_associations(result_set, callsite_sig)
result_set
end
- def find_by_sql_without_scrooge( sql)
+ def find_by_sql_without_scrooge(sql)
connection.select_all(sanitize_sql(sql), "#{name} Load").collect! do |record|
instantiate( UnscroogedAttributes.setup(record) )
end
@@ -108,7 +114,7 @@ def scrooge_select_regex
def callsite_sql( sql )
sql.gsub(ScroogeRegexSanitize, ScroogeBlankString)
end
-
+
end
module InstanceMethods
@@ -211,7 +217,9 @@ def scrooge_fetch_remaining
# Dumped objects should not contain object_ids of old result sets
#
def scrooge_invalidate_updateable_result_set
- @attributes.updateable_result_set = ResultSets::UpdateableResultSet.new(nil, self) if scrooged?
+ if scrooged?
+ @attributes.updateable_result_set = ResultSets::UpdateableResultSet.new(nil, self, callsite_signature, @attributes.scrooge_columns)
+ end
end
# New objects should get an UnscroogedAttributes as their @attributes hash
View
7 lib/optimizations/result_sets/updateable_result_set.rb
@@ -7,13 +7,16 @@ class UpdateableResultSet
#
attr_accessor :updaters_attributes
+ attr_reader :callsite_signature, :scrooge_columns
- def initialize(result_set_array, klass)
+ def initialize(result_set_array, klass, callsite_signature, scrooge_columns)
if result_set_array
@result_set_object_id = result_set_array.object_id
@unique_id = result_set_array.unique_id ||= "#{Time.now.to_f}#{object_id}" # avoid recycled object ids
end
@klass = klass # expected class of items in the array
+ @callsite_signature = callsite_signature
+ @scrooge_columns = scrooge_columns
end
# Called by a ScroogedAttributes hash when it is asked for a column
@@ -83,7 +86,7 @@ def update_with(remaining_attributes)
remaining_attributes.each do |r_id, r_att|
old_attributes = current_attributes[r_id]
if old_attributes
- old_attributes.update(r_att.merge(old_attributes)) # must call update, do not use reverse_update
+ old_attributes.update(r_att)
end
end
end
View
34 lib/scrooge.rb
@@ -2,9 +2,11 @@
require 'set'
require 'callsite'
+require 'simple_set'
require 'optimizations/columns/attributes_proxy'
require 'optimizations/columns/macro'
require 'optimizations/associations/macro'
+require 'optimizations/associations/association_set'
require 'optimizations/result_sets/updateable_result_set'
require 'optimizations/result_sets/result_array'
@@ -13,9 +15,14 @@ class Base
@@scrooge_callsites = {}
ScroogeCallsiteSample = 0..10
+ ScroogeMutex = Mutex.new
+
+ # this can be set to help with testing
+ cattr_accessor :scrooge_ignore_call_stack
+ @@scrooge_ignore_call_stack = ($0 == "irb")
class << self
-
+
# Determine if a given SQL string is a candidate for callsite <=> columns
# optimization.
#
@@ -30,20 +37,23 @@ def find_by_sql(sql)
# Expose known callsites for this model
#
def scrooge_callsites
- @@scrooge_callsites[self.table_name] ||= {}
+ @@scrooge_callsites[table_name] || ScroogeMutex.synchronize { @@scrooge_callsites[table_name] = {} }
end
# Fetch or setup a callsite instance for a given signature
#
def scrooge_callsite( callsite_signature )
- @@scrooge_callsites[self.table_name] ||= {}
- @@scrooge_callsites[self.table_name][callsite_signature] ||= callsite( callsite_signature )
+ scrooge_callsites[callsite_signature] || ScroogeMutex.synchronize do
+ scrooge_callsites[callsite_signature] = callsite(callsite_signature)
+ end
end
- # Flush all known callsites.Mostly a test helper.
- #
+ # Flush all known callsites. Mostly a test helper.
+ #
def scrooge_flush_callsites!
- @@scrooge_callsites[self.table_name] = {}
+ ScroogeMutex.synchronize do
+ @@scrooge_callsites[table_name] = {}
+ end
end
private
@@ -51,7 +61,9 @@ def scrooge_flush_callsites!
# Removes a single callsite
#
def scrooge_unlink_callsite!( callsite_signature )
- @@scrooge_callsites.delete(callsite_signature)
+ ScroogeMutex.synchronize do
+ @@scrooge_callsites.delete(callsite_signature)
+ end
end
# Initialize a callsite
@@ -70,7 +82,11 @@ def attribute_with_table( attr_name )
# context information.
#
def callsite_signature( call_stack, supplementary )
- ( call_stack[ScroogeCallsiteSample] << supplementary ).hash
+ if @@scrooge_ignore_call_stack
+ supplementary.hash
+ else
+ ( call_stack[ScroogeCallsiteSample] << supplementary ).hash
+ end
end
end # class << self
View
91 lib/simple_set.rb
@@ -0,0 +1,91 @@
+module Scrooge
+
+ class SimpleSet < Hash
+
+ class << self
+ ##
+ # Creates a new set containing the given objects
+ #
+ # @return [SimpleSet] The new set
+ #
+ # @api public
+ def [](*ary)
+ new(ary)
+ end
+ end
+
+ ##
+ # Create a new SimpleSet containing the unique members of _arr_
+ #
+ # @param [Array] arr Initial set values.
+ #
+ # @return [Array] The array the Set was initialized with
+ #
+ # @api public
+ def initialize(arr = [])
+ Array(arr).each {|x| self[x] = true}
+ end
+
+ ##
+ # Add a value to the set, and return it
+ #
+ # @param [Object] value Value to add to set.
+ #
+ # @return [SimpleSet] Receiver
+ #
+ # @api public
+ def <<(value)
+ self[value] = true
+ self
+ end
+
+ ##
+ # Merge _arr_ with receiver, producing the union of receiver & _arr_
+ #
+ # s = Extlib::SimpleSet.new([:a, :b, :c])
+ # s.merge([:c, :d, :e, f]) #=> #<SimpleSet: {:e, :c, :f, :a, :d, :b}>
+ #
+ # @param [Array] arr Values to merge with set.
+ #
+ # @return [SimpleSet] The set after the Array was merged in.
+ #
+ # @api public
+ def merge(arr)
+ super(arr.inject({}) {|s,x| s[x] = true; s })
+ end
+ alias_method :|, :merge
+
+ ##
+ # Invokes block once for each item in the set. Creates an array
+ # containing the values returned by the block.
+ #
+ # s = Extlib::SimpleSet.new([1, 2, 3])
+ # s.collect {|s| s + 1} #=> [2, 3, 4]
+ #
+ # @return [Array] The values returned by the block
+ #
+ # @api public
+ def collect(&block)
+ keys.collect(&block)
+ end
+ alias_method :map, :collect
+
+ ##
+ # Get a human readable version of the set.
+ #
+ # s = SimpleSet.new([:a, :b, :c])
+ # s.inspect #=> "#<SimpleSet: {:c, :a, :b}>"
+ #
+ # @return [String] A human readable version of the set.
+ #
+ # @api public
+ def inspect
+ "#<SimpleSet: {#{keys.map {|x| x.inspect}.join(", ")}}>"
+ end
+
+ # def to_a
+ alias_method :to_a, :keys
+
+ end # SimpleSet
+
+end
View
16 test/callsite_test.rb
@@ -10,14 +10,14 @@ def setup
test "should initialize with a default set of columns" do
assert @callsite.columns.empty?
- assert_equal Scrooge::Callsite.new( MysqlUser, 123456 ).columns, Set["User"]
+ assert_equal Scrooge::Callsite.new( MysqlUser, 123456 ).columns, SimpleSet["User"]
Scrooge::Callsite.any_instance.stubs(:inheritable?).returns(true)
Scrooge::Callsite.any_instance.stubs(:inheritance_column).returns("inheritance")
- assert_equal Scrooge::Callsite.new( MysqlUser, 123456 ).columns, Set["User","inheritance"]
+ assert_equal Scrooge::Callsite.new( MysqlUser, 123456 ).columns, SimpleSet["User","inheritance"]
end
test "should be inspectable" do
- @callsite.association! :mysql_user
+ @callsite.association! :mysql_user, 123456
@callsite.column! :db
assert_equal @callsite.inspect, "<#MysqlTablePrivilege :select => '`tables_priv`.db', :include => [:mysql_user]>"
end
@@ -29,12 +29,12 @@ def setup
end
test "should flag only preloadable associations as seen" do
- assert_no_difference '@callsite.associations.size' do
- @callsite.association! :undefined
+ assert_no_difference '@callsite.associations.to_preload.size' do
+ @callsite.association! :undefined, 123456
end
- assert_difference '@callsite.associations.size', 2 do
- @callsite.association! :column_privilege
- @callsite.association! :mysql_user
+ assert_difference '@callsite.associations.to_preload.size', 2 do
+ @callsite.association! :column_privilege, 123456
+ @callsite.association! :mysql_user, 123456
end
end
View
4 test/optimizations/associations/macro_test.rb
@@ -7,14 +7,14 @@ class OptimizationsAssociationsMacroTest < ActiveSupport::TestCase
test "should be able to flag any associations instantiated from a record" do
@user = MysqlUser.find(:first)
@user.host
- assert_equal MysqlUser.scrooge_callsite( @user.callsite_signature ).associations, Set[:host]
+ assert_equal MysqlUser.scrooge_callsite( @user.callsite_signature ).associations.to_preload, [:host]
end
test "should only flag preloadable associations" do
Scrooge::Callsite.any_instance.expects(:association!).once
@user = MysqlUser.find(:first)
@user.host
- assert_equal MysqlUser.scrooge_callsite( @user.callsite_signature ).associations, Set.new
+ assert_equal [], MysqlUser.scrooge_callsite( @user.callsite_signature ).associations.to_preload
end
test "should be able to identify all preloadable associations for a given Model" do
View
2  test/scrooge_test.rb
@@ -51,7 +51,7 @@ class ScroogeTest < ActiveRecord::TestCase
end
test "should be able to generate a SQL select snippet from a given set" do
- assert_equal MysqlUser.scrooge_select_sql( Set['Password','User','Host'] ), "`user`.User,`user`.Password,`user`.Host"
+ assert_equal MysqlUser.scrooge_select_sql( SimpleSet['Password','User','Host'] ), "`user`.User,`user`.Password,`user`.Host"
end
test "should be able to augment an existing callsite when attributes is referenced that we haven't seen yet" do
Please sign in to comment.
Something went wrong with that request. Please try again.