Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Refactor DataFabric to support ActiveRecord 2.0, 2.1 and 2.2.

AR 2.0 and 2.1 are supported by the code in ar20.rb.
AR 2.2+ is supported by the code in ar22.rb.
TODO One test still failing.
  • Loading branch information...
commit 7d7a2acfaddc56fe016f6e61957b65075abba54f 1 parent fa5ef1b
@mperham mperham authored
View
4 CHANGELOG
@@ -1,5 +1,9 @@
DataFabric changelog
+v1.2.0 - 2008-12-25
+
+- Now supports ActiveRecord 2.2.
+
v1.1.0 - 2008-11-22
- Cache connections so we don't have to reconnect constantly. (Justin Balthrop, Geni)
View
6 Manifest
@@ -56,6 +56,8 @@ example/test/functional/accounts_controller_test.rb
example/test/integration/account_figments_test.rb
example/test/test_helper.rb
init.rb
+lib/data_fabric/ar20.rb
+lib/data_fabric/ar22.rb
lib/data_fabric/version.rb
lib/data_fabric.rb
Manifest
@@ -68,4 +70,8 @@ test/database_test.rb
test/shard_test.rb
test/test_helper.rb
test/thread_test.rb
+test/vr_austin_master.db
+test/vr_austin_slave.db
+test/vr_dallas_master.db
+test/vr_dallas_slave.db
TESTING.rdoc
View
2  README.rdoc
@@ -62,7 +62,7 @@ ActionController around_filter based on the user as follows:
== Warnings
* Sharded models should never be placed in the session store or you will get "Shard not set" errors when the session is persisted.
-* ActiveRecord's allow_concurrency = true is NOT supported in this version of data_fabric.
+* DataFabric does not support running with ActiveRecord's allow_concurrency = true in AR 2.0 and 2.1. allow_concurrency is gone in AR 2.2.
== Testing and Bug Reports
View
17 Rakefile
@@ -1,8 +1,6 @@
require 'rubygems'
require 'echoe'
-#gem 'rails', '=2.0.2'
-
require File.dirname(__FILE__) << "/lib/data_fabric/version"
Echoe.new 'data_fabric' do |p|
@@ -10,16 +8,23 @@ Echoe.new 'data_fabric' do |p|
p.author = "Mike Perham"
p.email = 'mperham@gmail.com'
p.project = 'fiveruns'
- p.summary = 'Sharding and replication support for ActiveRecord 2.0 and 2.1'
+ p.summary = 'Sharding and replication support for ActiveRecord 2.x'
p.url = "http://github.com/fiveruns/data_fabric"
-# p.dependencies = ['activerecord >=2.0.2']
p.development_dependencies = []
p.rubygems_version = nil
p.include_rakefile = true
+ p.test_pattern = 'test/*_test.rb'
end
task :test => [:pretest]
+desc "Test all versions of ActiveRecord installed locally"
+task :test_all do
+ Gem.source_index.search(Gem::Dependency.new('activerecord', '>=2.0')).each do |spec|
+ puts `rake test AR_VERSION=#{spec.version}`
+ end
+end
+
task :pretest do
setup(false)
end
@@ -28,10 +33,6 @@ task :create_db do
setup(true)
end
-task :changelog do
- `git log | grep -v git-svn-id > History.txt`
-end
-
def load_database_yml
filename = "test/database.yml"
if !File.exist?(filename)
View
139 lib/data_fabric.rb
@@ -45,7 +45,13 @@ module DataFabric
def self.init
logger = ActiveRecord::Base.logger unless logger
log { "Loading data_fabric #{DataFabric::Version::STRING} with ActiveRecord #{ActiveRecord::VERSION::STRING}" }
- ActiveRecord::Base.send(:include, self)
+
+ if ActiveRecord::VERSION::STRING < '2.2.0'
+ require 'data_fabric/ar20'
+ else
+ require 'data_fabric/ar22'
+ end
+ ActiveRecord::Base.send(:include, DataFabric::Extensions)
end
def self.activate_shard(shards, &block)
@@ -89,11 +95,6 @@ def self.shard_active_for?(group)
Thread.current[:shards] and Thread.current[:shards][group.to_s]
end
- def self.included(model)
- # Wire up ActiveRecord::Base
- model.extend ClassMethods
- end
-
def self.ensure_setup
Thread.current[:shards] = {} unless Thread.current[:shards]
end
@@ -102,130 +103,4 @@ def self.log(level=Logger::INFO, &block)
logger && logger.add(level, &block)
end
- # Class methods injected into ActiveRecord::Base
- module ClassMethods
- def data_fabric(options)
- proxy = DataFabric::ConnectionProxy.new(self, options)
- ActiveRecord::Base.active_connections[name] = proxy
-
- raise ArgumentError, "data_fabric does not support ActiveRecord's allow_concurrency = true" if allow_concurrency
- DataFabric.log { "Creating data_fabric proxy for class #{name}" }
- end
- end
-
- class StringProxy
- def initialize(&block)
- @proc = block
- end
- def to_s
- @proc.call
- end
- end
-
- class ConnectionProxy
- def initialize(model_class, options)
- @model_class = model_class
- @replicated = options[:replicated]
- @shard_group = options[:shard_by]
- @prefix = options[:prefix]
- @role = 'slave' if @replicated
-
- @model_class.send :include, ActiveRecordConnectionMethods if @replicated
- end
-
- delegate :insert, :update, :delete, :create_table, :rename_table, :drop_table, :add_column, :remove_column,
- :change_column, :change_column_default, :rename_column, :add_index, :remove_index, :initialize_schema_information,
- :dump_schema_information, :execute, :execute_ignore_duplicate, :to => :master
-
- def cache(&block)
- connection.cache(&block)
- end
-
- def transaction(start_db_transaction = true, &block)
- with_master { connection.transaction(start_db_transaction, &block) }
- end
-
- def method_missing(method, *args, &block)
- DataFabric.log(Logger::DEBUG) { "Calling #{method} on #{connection}" }
- connection.send(method, *args, &block)
- end
-
- def connection_name
- connection_name_builder.join('_')
- end
-
- def disconnect!
- if connected?
- connection.disconnect!
- cached_connections[connection_name] = nil
- end
- end
-
- def verify!(arg)
- connection.verify!(arg) if connected?
- end
-
- def with_master
- # Allow nesting of with_master.
- old_role = @role
- set_role('master')
- yield
- ensure
- set_role(old_role)
- end
-
- private
-
- def cached_connections
- @cached_connections ||= {}
- end
-
- def connection_name_builder
- @connection_name_builder ||= begin
- clauses = []
- clauses << @prefix if @prefix
- clauses << @shard_group if @shard_group
- clauses << StringProxy.new { DataFabric.active_shard(@shard_group) } if @shard_group
- clauses << RAILS_ENV
- clauses << StringProxy.new { @role } if @replicated
- clauses
- end
- end
-
- def connection
- name = connection_name
- if not connected?
- config = ActiveRecord::Base.configurations[name]
- raise ArgumentError, "Unknown database config: #{name}, have #{ActiveRecord::Base.configurations.inspect}" unless config
- DataFabric.log { "Connecting to #{name}" }
- @model_class.establish_connection(config)
- cached_connections[name] = @model_class.connection
- @model_class.active_connections[@model_class.name] = self
- end
- cached_connections[name].verify!(3600)
- cached_connections[name]
- end
-
- def connected?
- DataFabric.shard_active_for?(@shard_group) and cached_connections[connection_name]
- end
-
- def set_role(role)
- @role = role if @replicated
- end
-
- def master
- with_master { return connection }
- end
- end
-
- module ActiveRecordConnectionMethods
- def self.included(base)
- base.alias_method_chain :reload, :master
- end
-
- def reload_with_master(*args, &block)
- connection.with_master { reload_without_master }
- end
- end
end
View
135 lib/data_fabric/ar20.rb
@@ -0,0 +1,135 @@
+module DataFabric
+ module Extensions
+ def self.included(model)
+ # Wire up ActiveRecord::Base
+ model.extend ClassMethods
+ end
+
+ # Class methods injected into ActiveRecord::Base
+ module ClassMethods
+ def data_fabric(options)
+ proxy = DataFabric::ConnectionProxy.new(self, options)
+ ActiveRecord::Base.active_connections[name] = proxy
+
+ raise ArgumentError, "data_fabric does not support ActiveRecord's allow_concurrency = true" if allow_concurrency
+ DataFabric.log { "Creating data_fabric proxy for class #{name}" }
+ end
+ end
+ end
+
+ class ConnectionProxy
+ def initialize(model_class, options)
+ @model_class = model_class
+ @replicated = options[:replicated]
+ @shard_group = options[:shard_by]
+ @prefix = options[:prefix]
+ @role = 'slave' if @replicated
+
+ @model_class.send :include, ActiveRecordConnectionMethods if @replicated
+ end
+
+ delegate :insert, :update, :delete, :create_table, :rename_table, :drop_table, :add_column, :remove_column,
+ :change_column, :change_column_default, :rename_column, :add_index, :remove_index, :initialize_schema_information,
+ :dump_schema_information, :execute, :execute_ignore_duplicate, :to => :master
+
+ def cache(&block)
+ connection.cache(&block)
+ end
+
+ def transaction(start_db_transaction = true, &block)
+ with_master { connection.transaction(start_db_transaction, &block) }
+ end
+
+ def method_missing(method, *args, &block)
+ DataFabric.log(Logger::DEBUG) { "Calling #{method} on #{connection}" }
+ connection.send(method, *args, &block)
+ end
+
+ def connection_name
+ connection_name_builder.join('_')
+ end
+
+ def disconnect!
+ if connected?
+ connection.disconnect!
+ cached_connections[connection_name] = nil
+ end
+ end
+
+ def verify!(arg)
+ connection.verify!(arg) if connected?
+ end
+
+ def with_master
+ # Allow nesting of with_master.
+ old_role = @role
+ set_role('master')
+ yield
+ ensure
+ set_role(old_role)
+ end
+
+ private
+
+ def cached_connections
+ @cached_connections ||= {}
+ end
+
+ def connection_name_builder
+ @connection_name_builder ||= begin
+ clauses = []
+ clauses << @prefix if @prefix
+ clauses << @shard_group if @shard_group
+ clauses << StringProxy.new { DataFabric.active_shard(@shard_group) } if @shard_group
+ clauses << RAILS_ENV
+ clauses << StringProxy.new { @role } if @replicated
+ clauses
+ end
+ end
+
+ def connection
+ name = connection_name
+ if not connected?
+ config = ActiveRecord::Base.configurations[name]
+ raise ArgumentError, "Unknown database config: #{name}, have #{ActiveRecord::Base.configurations.inspect}" unless config
+ DataFabric.log { "Connecting to #{name}" }
+ @model_class.establish_connection(config)
+ cached_connections[name] = @model_class.connection
+ @model_class.active_connections[@model_class.name] = self
+ end
+ cached_connections[name].verify!(3600)
+ cached_connections[name]
+ end
+
+ def connected?
+ DataFabric.shard_active_for?(@shard_group) and cached_connections[connection_name]
+ end
+
+ def set_role(role)
+ @role = role if @replicated
+ end
+
+ def master
+ with_master { return connection }
+ end
+ end
+
+ module ActiveRecordConnectionMethods
+ def self.included(base)
+ base.alias_method_chain :reload, :master
+ end
+
+ def reload_with_master(*args, &block)
+ connection.with_master { reload_without_master }
+ end
+ end
+
+ class StringProxy
+ def initialize(&block)
+ @proc = block
+ end
+ def to_s
+ @proc.call
+ end
+ end
+end
View
169 lib/data_fabric/ar22.rb
@@ -0,0 +1,169 @@
+module DataFabric
+ module Extensions
+ def self.included(model)
+ # Wire up ActiveRecord::Base
+ model.extend ClassMethods
+ ConnectionProxy.shard_pools = {}
+ end
+
+ # Class methods injected into ActiveRecord::Base
+ module ClassMethods
+ def data_fabric(options)
+ DataFabric.log { "Creating data_fabric proxy for class #{name}" }
+ @proxy = DataFabric::ConnectionProxy.new(self, options)
+
+ class << self
+ def connection
+ @proxy
+ end
+
+ def connected?
+ @proxy.connected?
+ end
+
+ def remove_connection(klass)
+ raise "not implemented"
+ end
+
+ def connection_pool
+ raise "dynamic connection switching means you cannot get direct access to a pool"
+ end
+ end
+ end
+ end
+ end
+
+ class ConnectionProxy
+ cattr_accessor :shard_pools
+
+ def initialize(model_class, options)
+ @model_class = model_class
+ @replicated = options[:replicated]
+ @shard_group = options[:shard_by]
+ @prefix = options[:prefix]
+ set_role('slave') if @replicated
+
+ @model_class.send :include, ActiveRecordConnectionMethods if @replicated
+ end
+
+ delegate :insert, :update, :delete, :create_table, :rename_table, :drop_table, :add_column, :remove_column,
+ :change_column, :change_column_default, :rename_column, :add_index, :remove_index, :initialize_schema_information,
+ :dump_schema_information, :execute, :execute_ignore_duplicate, :to => :master
+
+ delegate :insert_many, :to => :master # ar-extensions bulk insert support
+
+ def cache(&block)
+ connection.cache(&block)
+ end
+
+ def transaction(start_db_transaction = true, &block)
+ with_master { connection.transaction(start_db_transaction, &block) }
+ end
+
+ def method_missing(method, *args, &block)
+ DataFabric.log(Logger::DEBUG) { "Calling #{method} on #{connection}" }
+ connection.send(method, *args, &block)
+ end
+
+ def connection_name
+ connection_name_builder.join('_')
+ end
+
+ def verify!(arg)
+ connection.verify!(arg) if connected?
+ end
+
+ def with_master
+ # Allow nesting of with_master.
+ old_role = current_role
+ set_role('master')
+ yield
+ ensure
+ set_role(old_role)
+ end
+
+ private
+
+ def current_pool
+ name = connection_name
+ self.class.shard_pools[name] ||= begin
+ config = ActiveRecord::Base.configurations[name]
+ raise ArgumentError, "Unknown database config: #{name}, have #{ActiveRecord::Base.configurations.inspect}" unless config
+ ActiveRecord::ConnectionAdapters::ConnectionPool.new(spec_for(config))
+ end
+ end
+
+ def spec_for(config)
+ # XXX This looks pretty fragile. Will break if AR changes how it initializes connections and adapters.
+ config = config.symbolize_keys
+ adapter_method = "#{config[:adapter]}_connection"
+ initialize_adapter(config[:adapter])
+ ActiveRecord::Base::ConnectionSpecification.new(config, adapter_method)
+ end
+
+ def initialize_adapter(adapter)
+ begin
+ require 'rubygems'
+ gem "activerecord-#{adapter}-adapter"
+ require "active_record/connection_adapters/#{adapter}_adapter"
+ rescue LoadError
+ begin
+ require "active_record/connection_adapters/#{adapter}_adapter"
+ rescue LoadError
+ raise "Please install the #{adapter} adapter: `gem install activerecord-#{adapter}-adapter` (#{$!})"
+ end
+ end
+ end
+
+ def connection_name_builder
+ @connection_name_builder ||= begin
+ clauses = []
+ clauses << @prefix if @prefix
+ clauses << @shard_group if @shard_group
+ clauses << StringProxy.new { DataFabric.active_shard(@shard_group) } if @shard_group
+ clauses << RAILS_ENV
+ clauses << StringProxy.new { current_role } if @replicated
+ clauses
+ end
+ end
+
+ def connection
+ current_pool.connection
+ end
+
+ def connected?
+ DataFabric.shard_active_for?(@shard_group) and cached_connections[connection_name]
+ end
+
+ def set_role(role)
+ Thread.current[:data_fabric_role] = role if @replicated
+ end
+
+ def current_role
+ Thread.current[:data_fabric_role]
+ end
+
+ def master
+ with_master { return connection }
+ end
+ end
+
+ module ActiveRecordConnectionMethods
+ def self.included(base)
+ base.alias_method_chain :reload, :master
+ end
+
+ def reload_with_master(*args, &block)
+ connection.with_master { reload_without_master }
+ end
+ end
+
+ class StringProxy
+ def initialize(&block)
+ @proc = block
+ end
+ def to_s
+ @proc.call
+ end
+ end
+end
View
2  lib/data_fabric/version.rb
@@ -1,5 +1,5 @@
module DataFabric
module Version
- STRING = "1.1.0"
+ STRING = "1.2.0"
end
end
View
8 test/connection_test.rb
@@ -36,7 +36,7 @@ def method_missing(name, *args)
class RawConnection
def method_missing(name, *args)
- puts "#{self.class.name} missing '#{name}': #{args.inspect}"
+ puts "#{self.class.name} missing '#{name}': #{args.inspect}"
end
end
@@ -96,7 +96,11 @@ def test_enchilada
private
def setup_configuration_for(clazz, name)
- flexmock(clazz).should_receive(:mysql_connection).and_return(AdapterMock.new(RawConnection.new))
+ if ar22?
+ flexmock(ActiveRecord::ConnectionAdapters::ConnectionPool).new_instances.should_receive(:new_connection).and_return(AdapterMock.new(RawConnection.new))
+ else
+ flexmock(klass).should_receive(:mysql_connection).and_return(AdapterMock.new(RawConnection.new))
+ end
ActiveRecord::Base.configurations ||= HashWithIndifferentAccess.new
ActiveRecord::Base.configurations[name] = HashWithIndifferentAccess.new({ :adapter => 'mysql', :database => name, :host => 'localhost'})
end
View
1  test/database_test.rb
@@ -10,6 +10,7 @@ class DatabaseTest < Test::Unit::TestCase
def setup
ActiveRecord::Base.configurations = load_database_yml
+ DataFabric::ConnectionProxy.shard_pools.clear
end
def test_live_burrito
View
63 test/test_helper.rb
@@ -1,34 +1,41 @@
-if !defined?(ROOT_PATH) # Don't evaluate this file twice.
- ENV['RAILS_ENV'] = 'test'
- RAILS_ENV = 'test'
- ROOT_PATH = File.expand_path(File.join(File.dirname(__FILE__), ".."))
- DATABASE_YML_PATH = File.join(ROOT_PATH, "test", "database.yml")
- Dir.chdir(ROOT_PATH)
+ENV['RAILS_ENV'] = 'test'
+RAILS_ENV = 'test'
+ROOT_PATH = File.expand_path(File.join(File.dirname(__FILE__), ".."))
+DATABASE_YML_PATH = File.join(ROOT_PATH, "test", "database.yml")
+Dir.chdir(ROOT_PATH)
- require 'rubygems'
- require 'test/unit'
+require 'rubygems'
+require 'test/unit'
- # Bootstrap AR
- gem 'activerecord', '=2.0.2'
- require 'active_record'
- require 'active_record/version'
- ActiveRecord::Base.logger = Logger.new(STDOUT)
- ActiveRecord::Base.logger.level = Logger::WARN
- ActiveRecord::Base.allow_concurrency = false
+version = ENV['AR_VERSION']
+if version
+ puts "Testing ActiveRecord #{version}"
+ gem 'activerecord', "=#{version}"
+end
+
+require 'active_record'
+require 'active_record/version'
+ActiveRecord::Base.logger = Logger.new(STDOUT)
+ActiveRecord::Base.logger.level = Logger::WARN
+ActiveRecord::Base.logger = Logger.new(STDOUT)
- # Bootstrap DF
- Dependencies.load_paths << File.join(File.dirname(__FILE__), '../lib')
- require 'init'
+# Bootstrap DF
+deps = defined?(ActiveSupport::Dependencies) ? ActiveSupport::Dependencies : Dependencies
+deps.load_paths << File.join(File.dirname(__FILE__), '../lib')
+require 'init'
- def load_database_yml
- filename = DATABASE_YML_PATH
- YAML::load(ERB.new(IO.read(filename)).result)
- end
+def load_database_yml
+ filename = DATABASE_YML_PATH
+ YAML::load(ERB.new(IO.read(filename)).result)
+end
+
+def ar22?
+ ActiveRecord::VERSION::STRING >= '2.2.0'
+end
- if !File.exist?(DATABASE_YML_PATH)
- STDERR.puts "\n*** ERROR ***:\n" <<
- "You must have a 'test/database.yml' file in order to run the unit tests. " <<
- "An example is provided in 'test/database.yml.example'.\n\n"
- exit 1
- end
+if !File.exist?(DATABASE_YML_PATH)
+ puts "\n*** ERROR ***:\n" <<
+ "You must have a 'test/database.yml' file in order to run the unit tests. " <<
+ "An example is provided in 'test/database.yml.example'.\n\n"
+ exit 1
end
View
30 test/thread_test.rb
@@ -4,20 +4,28 @@
class ThreadTest < Test::Unit::TestCase
MUTEX = Mutex.new
-
- def test_concurrency_not_allowed
- assert_raise ArgumentError do
- Object.class_eval %{
- class ThreadedEnchilada < ActiveRecord::Base
- self.allow_concurrency = true
- set_table_name :enchiladas
- data_fabric :prefix => 'fiveruns', :replicated => true, :shard_by => :city
- end
- }
+
+ if ActiveRecord::VERSION::STRING < '2.2.0'
+ def test_concurrency_not_allowed
+ assert_raise ArgumentError do
+ Object.class_eval %{
+ class ThreadedEnchilada < ActiveRecord::Base
+ self.allow_concurrency = true
+ set_table_name :enchiladas
+ data_fabric :prefix => 'fiveruns', :replicated => true, :shard_by => :city
+ end
+ }
+ end
end
end
- def xtest_class_and_instance_connections
+ def test_class_and_instance_connections
+ Object.class_eval %{
+ class ThreadedEnchilada < ActiveRecord::Base
+ set_table_name :enchiladas
+ data_fabric :prefix => 'fiveruns', :replicated => true, :shard_by => :city
+ end
+ }
ActiveRecord::Base.configurations = load_database_yml
cconn = ThreadedEnchilada.connection
Please sign in to comment.
Something went wrong with that request. Please try again.