Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Refactor DataFabric to support ActiveRecord 2.0, 2.1 and 2.2.

AR 2.0 and 2.1 are supported by the code in ar20.rb.
AR 2.2+ is supported by the code in ar22.rb.
TODO One test still failing.
  • Loading branch information...
commit 7d7a2acfaddc56fe016f6e61957b65075abba54f 1 parent fa5ef1b
@mperham mperham authored
View
4 CHANGELOG
@@ -1,5 +1,9 @@
DataFabric changelog
+v1.2.0 - 2008-12-25
+
+- Now supports ActiveRecord 2.2.
+
v1.1.0 - 2008-11-22
- Cache connections so we don't have to reconnect constantly. (Justin Balthrop, Geni)
View
6 Manifest
@@ -56,6 +56,8 @@ example/test/functional/accounts_controller_test.rb
example/test/integration/account_figments_test.rb
example/test/test_helper.rb
init.rb
+lib/data_fabric/ar20.rb
+lib/data_fabric/ar22.rb
lib/data_fabric/version.rb
lib/data_fabric.rb
Manifest
@@ -68,4 +70,8 @@ test/database_test.rb
test/shard_test.rb
test/test_helper.rb
test/thread_test.rb
+test/vr_austin_master.db
+test/vr_austin_slave.db
+test/vr_dallas_master.db
+test/vr_dallas_slave.db
TESTING.rdoc
View
2  README.rdoc
@@ -62,7 +62,7 @@ ActionController around_filter based on the user as follows:
== Warnings
* Sharded models should never be placed in the session store or you will get "Shard not set" errors when the session is persisted.
-* ActiveRecord's allow_concurrency = true is NOT supported in this version of data_fabric.
+* DataFabric does not support running with ActiveRecord's allow_concurrency = true in AR 2.0 and 2.1. allow_concurrency is gone in AR 2.2.
== Testing and Bug Reports
View
17 Rakefile
@@ -1,8 +1,6 @@
require 'rubygems'
require 'echoe'
-#gem 'rails', '=2.0.2'
-
require File.dirname(__FILE__) << "/lib/data_fabric/version"
Echoe.new 'data_fabric' do |p|
@@ -10,16 +8,23 @@ Echoe.new 'data_fabric' do |p|
p.author = "Mike Perham"
p.email = 'mperham@gmail.com'
p.project = 'fiveruns'
- p.summary = 'Sharding and replication support for ActiveRecord 2.0 and 2.1'
+ p.summary = 'Sharding and replication support for ActiveRecord 2.x'
p.url = "http://github.com/fiveruns/data_fabric"
-# p.dependencies = ['activerecord >=2.0.2']
p.development_dependencies = []
p.rubygems_version = nil
p.include_rakefile = true
+ p.test_pattern = 'test/*_test.rb'
end
task :test => [:pretest]
+desc "Test all versions of ActiveRecord installed locally"
+task :test_all do
+ Gem.source_index.search(Gem::Dependency.new('activerecord', '>=2.0')).each do |spec|
+ puts `rake test AR_VERSION=#{spec.version}`
+ end
+end
+
task :pretest do
setup(false)
end
@@ -28,10 +33,6 @@ task :create_db do
setup(true)
end
-task :changelog do
- `git log | grep -v git-svn-id > History.txt`
-end
-
def load_database_yml
filename = "test/database.yml"
if !File.exist?(filename)
View
139 lib/data_fabric.rb
@@ -45,7 +45,13 @@ module DataFabric
def self.init
logger = ActiveRecord::Base.logger unless logger
log { "Loading data_fabric #{DataFabric::Version::STRING} with ActiveRecord #{ActiveRecord::VERSION::STRING}" }
- ActiveRecord::Base.send(:include, self)
+
+ if ActiveRecord::VERSION::STRING < '2.2.0'
+ require 'data_fabric/ar20'
+ else
+ require 'data_fabric/ar22'
+ end
+ ActiveRecord::Base.send(:include, DataFabric::Extensions)
end
def self.activate_shard(shards, &block)
@@ -89,11 +95,6 @@ def self.shard_active_for?(group)
Thread.current[:shards] and Thread.current[:shards][group.to_s]
end
- def self.included(model)
- # Wire up ActiveRecord::Base
- model.extend ClassMethods
- end
-
def self.ensure_setup
Thread.current[:shards] = {} unless Thread.current[:shards]
end
@@ -102,130 +103,4 @@ def self.log(level=Logger::INFO, &block)
logger && logger.add(level, &block)
end
- # Class methods injected into ActiveRecord::Base
- module ClassMethods
- def data_fabric(options)
- proxy = DataFabric::ConnectionProxy.new(self, options)
- ActiveRecord::Base.active_connections[name] = proxy
-
- raise ArgumentError, "data_fabric does not support ActiveRecord's allow_concurrency = true" if allow_concurrency
- DataFabric.log { "Creating data_fabric proxy for class #{name}" }
- end
- end
-
- class StringProxy
- def initialize(&block)
- @proc = block
- end
- def to_s
- @proc.call
- end
- end
-
- class ConnectionProxy
- def initialize(model_class, options)
- @model_class = model_class
- @replicated = options[:replicated]
- @shard_group = options[:shard_by]
- @prefix = options[:prefix]
- @role = 'slave' if @replicated
-
- @model_class.send :include, ActiveRecordConnectionMethods if @replicated
- end
-
- delegate :insert, :update, :delete, :create_table, :rename_table, :drop_table, :add_column, :remove_column,
- :change_column, :change_column_default, :rename_column, :add_index, :remove_index, :initialize_schema_information,
- :dump_schema_information, :execute, :execute_ignore_duplicate, :to => :master
-
- def cache(&block)
- connection.cache(&block)
- end
-
- def transaction(start_db_transaction = true, &block)
- with_master { connection.transaction(start_db_transaction, &block) }
- end
-
- def method_missing(method, *args, &block)
- DataFabric.log(Logger::DEBUG) { "Calling #{method} on #{connection}" }
- connection.send(method, *args, &block)
- end
-
- def connection_name
- connection_name_builder.join('_')
- end
-
- def disconnect!
- if connected?
- connection.disconnect!
- cached_connections[connection_name] = nil
- end
- end
-
- def verify!(arg)
- connection.verify!(arg) if connected?
- end
-
- def with_master
- # Allow nesting of with_master.
- old_role = @role
- set_role('master')
- yield
- ensure
- set_role(old_role)
- end
-
- private
-
- def cached_connections
- @cached_connections ||= {}
- end
-
- def connection_name_builder
- @connection_name_builder ||= begin
- clauses = []
- clauses << @prefix if @prefix
- clauses << @shard_group if @shard_group
- clauses << StringProxy.new { DataFabric.active_shard(@shard_group) } if @shard_group
- clauses << RAILS_ENV
- clauses << StringProxy.new { @role } if @replicated
- clauses
- end
- end
-
- def connection
- name = connection_name
- if not connected?
- config = ActiveRecord::Base.configurations[name]
- raise ArgumentError, "Unknown database config: #{name}, have #{ActiveRecord::Base.configurations.inspect}" unless config
- DataFabric.log { "Connecting to #{name}" }
- @model_class.establish_connection(config)
- cached_connections[name] = @model_class.connection
- @model_class.active_connections[@model_class.name] = self
- end
- cached_connections[name].verify!(3600)
- cached_connections[name]
- end
-
- def connected?
- DataFabric.shard_active_for?(@shard_group) and cached_connections[connection_name]
- end
-
- def set_role(role)
- @role = role if @replicated
- end
-
- def master
- with_master { return connection }
- end
- end
-
- module ActiveRecordConnectionMethods
- def self.included(base)
- base.alias_method_chain :reload, :master
- end
-
- def reload_with_master(*args, &block)
- connection.with_master { reload_without_master }
- end
- end
end
View
135 lib/data_fabric/ar20.rb
@@ -0,0 +1,135 @@
+module DataFabric
+ module Extensions
+ def self.included(model)
+ # Wire up ActiveRecord::Base
+ model.extend ClassMethods
+ end
+
+ # Class methods injected into ActiveRecord::Base
+ module ClassMethods
+ def data_fabric(options)
+ proxy = DataFabric::ConnectionProxy.new(self, options)
+ ActiveRecord::Base.active_connections[name] = proxy
+
+ raise ArgumentError, "data_fabric does not support ActiveRecord's allow_concurrency = true" if allow_concurrency
+ DataFabric.log { "Creating data_fabric proxy for class #{name}" }
+ end
+ end
+ end
+
+ class ConnectionProxy
+ def initialize(model_class, options)
+ @model_class = model_class
+ @replicated = options[:replicated]
+ @shard_group = options[:shard_by]
+ @prefix = options[:prefix]
+ @role = 'slave' if @replicated
+
+ @model_class.send :include, ActiveRecordConnectionMethods if @replicated
+ end
+
+ delegate :insert, :update, :delete, :create_table, :rename_table, :drop_table, :add_column, :remove_column,
+ :change_column, :change_column_default, :rename_column, :add_index, :remove_index, :initialize_schema_information,
+ :dump_schema_information, :execute, :execute_ignore_duplicate, :to => :master
+
+ def cache(&block)
+ connection.cache(&block)
+ end
+
+ def transaction(start_db_transaction = true, &block)
+ with_master { connection.transaction(start_db_transaction, &block) }
+ end
+
+ def method_missing(method, *args, &block)
+ DataFabric.log(Logger::DEBUG) { "Calling #{method} on #{connection}" }
+ connection.send(method, *args, &block)
+ end
+
+ def connection_name
+ connection_name_builder.join('_')
+ end
+
+ def disconnect!
+ if connected?
+ connection.disconnect!
+ cached_connections[connection_name] = nil
+ end
+ end
+
+ def verify!(arg)
+ connection.verify!(arg) if connected?
+ end
+
+ def with_master
+ # Allow nesting of with_master.
+ old_role = @role
+ set_role('master')
+ yield
+ ensure
+ set_role(old_role)
+ end
+
+ private
+
+ def cached_connections
+ @cached_connections ||= {}
+ end
+
+ def connection_name_builder
+ @connection_name_builder ||= begin
+ clauses = []
+ clauses << @prefix if @prefix
+ clauses << @shard_group if @shard_group
+ clauses << StringProxy.new { DataFabric.active_shard(@shard_group) } if @shard_group
+ clauses << RAILS_ENV
+ clauses << StringProxy.new { @role } if @replicated
+ clauses
+ end
+ end
+
+ def connection
+ name = connection_name
+ if not connected?
+ config = ActiveRecord::Base.configurations[name]
+ raise ArgumentError, "Unknown database config: #{name}, have #{ActiveRecord::Base.configurations.inspect}" unless config
+ DataFabric.log { "Connecting to #{name}" }
+ @model_class.establish_connection(config)
+ cached_connections[name] = @model_class.connection
+ @model_class.active_connections[@model_class.name] = self
+ end
+ cached_connections[name].verify!(3600)
+ cached_connections[name]
+ end
+
+ def connected?
+ DataFabric.shard_active_for?(@shard_group) and cached_connections[connection_name]
+ end
+
+ def set_role(role)
+ @role = role if @replicated
+ end
+
+ def master
+ with_master { return connection }
+ end
+ end
+
+ module ActiveRecordConnectionMethods
+ def self.included(base)
+ base.alias_method_chain :reload, :master
+ end
+
+ def reload_with_master(*args, &block)
+ connection.with_master { reload_without_master }
+ end
+ end
+
+ class StringProxy
+ def initialize(&block)
+ @proc = block
+ end
+ def to_s
+ @proc.call
+ end
+ end
+end
View
169 lib/data_fabric/ar22.rb
@@ -0,0 +1,169 @@
+module DataFabric
+ module Extensions
+ def self.included(model)
+ # Wire up ActiveRecord::Base
+ model.extend ClassMethods
+ ConnectionProxy.shard_pools = {}
+ end
+
+ # Class methods injected into ActiveRecord::Base
+ module ClassMethods
+ def data_fabric(options)
+ DataFabric.log { "Creating data_fabric proxy for class #{name}" }
+ @proxy = DataFabric::ConnectionProxy.new(self, options)
+
+ class << self
+ def connection
+ @proxy
+ end
+
+ def connected?
+ @proxy.connected?
+ end
+
+ def remove_connection(klass)
+ raise "not implemented"
+ end
+
+ def connection_pool
+ raise "dynamic connection switching means you cannot get direct access to a pool"
+ end
+ end
+ end
+ end
+ end
+
+ class ConnectionProxy
+ cattr_accessor :shard_pools
+
+ def initialize(model_class, options)
+ @model_class = model_class
+ @replicated = options[:replicated]
+ @shard_group = options[:shard_by]
+ @prefix = options[:prefix]
+ set_role('slave') if @replicated
+
+ @model_class.send :include, ActiveRecordConnectionMethods if @replicated
+ end
+
+ delegate :insert, :update, :delete, :create_table, :rename_table, :drop_table, :add_column, :remove_column,
+ :change_column, :change_column_default, :rename_column, :add_index, :remove_index, :initialize_schema_information,
+ :dump_schema_information, :execute, :execute_ignore_duplicate, :to => :master
+
+ delegate :insert_many, :to => :master # ar-extensions bulk insert support
+
+ def cache(&block)
+ connection.cache(&block)
+ end
+
+ def transaction(start_db_transaction = true, &block)
+ with_master { connection.transaction(start_db_transaction, &block) }
+ end
+
+ def method_missing(method, *args, &block)
+ DataFabric.log(Logger::DEBUG) { "Calling #{method} on #{connection}" }
+ connection.send(method, *args, &block)
+ end
+
+ def connection_name
+ connection_name_builder.join('_')
+ end
+
+ def verify!(arg)
+ connection.verify!(arg) if connected?
+ end
+
+ def with_master
+ # Allow nesting of with_master.
+ old_role = current_role
+ set_role('master')
+ yield
+ ensure
+ set_role(old_role)
+ end
+
+ private
+
+ def current_pool
+ name = connection_name
+ self.class.shard_pools[name] ||= begin
+ config = ActiveRecord::Base.configurations[name]
+ raise ArgumentError, "Unknown database config: #{name}, have #{ActiveRecord::Base.configurations.inspect}" unless config
+ ActiveRecord::ConnectionAdapters::ConnectionPool.new(spec_for(config))
+ end
+ end
+
+ def spec_for(config)
+ # XXX This looks pretty fragile. Will break if AR changes how it initializes connections and adapters.
+ config = config.symbolize_keys
+ adapter_method = "#{config[:adapter]}_connection"
+ initialize_adapter(config[:adapter])
+ ActiveRecord::Base::ConnectionSpecification.new(config, adapter_method)
+ end
+
+ def initialize_adapter(adapter)
+ begin
+ require 'rubygems'
+ gem "activerecord-#{adapter}-adapter"
+ require "active_record/connection_adapters/#{adapter}_adapter"
+ rescue LoadError
+ begin
+ require "active_record/connection_adapters/#{adapter}_adapter"
+ rescue LoadError
+ raise "Please install the #{adapter} adapter: `gem install activerecord-#{adapter}-adapter` (#{$!})"
+ end
+ end
+ end
+
+ def connection_name_builder
+ @connection_name_builder ||= begin
+ clauses = []
+ clauses << @prefix if @prefix
+ clauses << @shard_group if @shard_group
+ clauses << StringProxy.new { DataFabric.active_shard(@shard_group) } if @shard_group
+ clauses << RAILS_ENV
+ clauses << StringProxy.new { current_role } if @replicated
+ clauses
+ end
+ end
+
+ def connection
+ current_pool.connection
+ end
+
+ def connected?
+ DataFabric.shard_active_for?(@shard_group) and cached_connections[connection_name]
+ end
+
+ def set_role(role)
+ Thread.current[:data_fabric_role] = role if @replicated
+ end
+
+ def current_role
+ Thread.current[:data_fabric_role]
+ end
+
+ def master
+ with_master { return connection }
+ end
+ end
+
+ module ActiveRecordConnectionMethods
+ def self.included(base)
+ base.alias_method_chain :reload, :master
+ end
+
+ def reload_with_master(*args, &block)
+ connection.with_master { reload_without_master }
+ end
+ end
+
+ class StringProxy
+ def initialize(&block)
+ @proc = block
+ end
+ def to_s
+ @proc.call
+ end
+ end
+end
View
2  lib/data_fabric/version.rb
@@ -1,5 +1,5 @@
module DataFabric
module Version
- STRING = "1.1.0"
+ STRING = "1.2.0"
end
end
View
8 test/connection_test.rb
@@ -36,7 +36,7 @@ def method_missing(name, *args)
class RawConnection
def method_missing(name, *args)
- puts "#{self.class.name} missing '#{name}': #{args.inspect}"
+ puts "#{self.class.name} missing '#{name}': #{args.inspect}"
end
end
@@ -96,7 +96,11 @@ def test_enchilada
private
def setup_configuration_for(clazz, name)
- flexmock(clazz).should_receive(:mysql_connection).and_return(AdapterMock.new(RawConnection.new))
+ if ar22?
+ flexmock(ActiveRecord::ConnectionAdapters::ConnectionPool).new_instances.should_receive(:new_connection).and_return(AdapterMock.new(RawConnection.new))
+ else
+ flexmock(klass).should_receive(:mysql_connection).and_return(AdapterMock.new(RawConnection.new))
+ end
ActiveRecord::Base.configurations ||= HashWithIndifferentAccess.new
ActiveRecord::Base.configurations[name] = HashWithIndifferentAccess.new({ :adapter => 'mysql', :database => name, :host => 'localhost'})
end
View
1  test/database_test.rb
@@ -10,6 +10,7 @@ class DatabaseTest < Test::Unit::TestCase
def setup
ActiveRecord::Base.configurations = load_database_yml
+ DataFabric::ConnectionProxy.shard_pools.clear
end
def test_live_burrito
View
63 test/test_helper.rb
@@ -1,34 +1,41 @@
-if !defined?(ROOT_PATH) # Don't evaluate this file twice.
- ENV['RAILS_ENV'] = 'test'
- RAILS_ENV = 'test'
- ROOT_PATH = File.expand_path(File.join(File.dirname(__FILE__), ".."))
- DATABASE_YML_PATH = File.join(ROOT_PATH, "test", "database.yml")
- Dir.chdir(ROOT_PATH)
+ENV['RAILS_ENV'] = 'test'
+RAILS_ENV = 'test'
+ROOT_PATH = File.expand_path(File.join(File.dirname(__FILE__), ".."))
+DATABASE_YML_PATH = File.join(ROOT_PATH, "test", "database.yml")
+Dir.chdir(ROOT_PATH)
- require 'rubygems'
- require 'test/unit'
+require 'rubygems'
+require 'test/unit'
- # Bootstrap AR
- gem 'activerecord', '=2.0.2'
- require 'active_record'
- require 'active_record/version'
- ActiveRecord::Base.logger = Logger.new(STDOUT)
- ActiveRecord::Base.logger.level = Logger::WARN
- ActiveRecord::Base.allow_concurrency = false
+version = ENV['AR_VERSION']
+if version
+ puts "Testing ActiveRecord #{version}"
+ gem 'activerecord', "=#{version}"
+end
+
+require 'active_record'
+require 'active_record/version'
+ActiveRecord::Base.logger = Logger.new(STDOUT)
+ActiveRecord::Base.logger.level = Logger::WARN
+ActiveRecord::Base.logger = Logger.new(STDOUT)
- # Bootstrap DF
- Dependencies.load_paths << File.join(File.dirname(__FILE__), '../lib')
- require 'init'
+# Bootstrap DF
+deps = defined?(ActiveSupport::Dependencies) ? ActiveSupport::Dependencies : Dependencies
+deps.load_paths << File.join(File.dirname(__FILE__), '../lib')
+require 'init'
- def load_database_yml
- filename = DATABASE_YML_PATH
- YAML::load(ERB.new(IO.read(filename)).result)
- end
+def load_database_yml
+ filename = DATABASE_YML_PATH
+ YAML::load(ERB.new(IO.read(filename)).result)
+end
+
+def ar22?
+ ActiveRecord::VERSION::STRING >= '2.2.0'
+end
- if !File.exist?(DATABASE_YML_PATH)
- STDERR.puts "\n*** ERROR ***:\n" <<
- "You must have a 'test/database.yml' file in order to run the unit tests. " <<
- "An example is provided in 'test/database.yml.example'.\n\n"
- exit 1
- end
+if !File.exist?(DATABASE_YML_PATH)
+ puts "\n*** ERROR ***:\n" <<
+ "You must have a 'test/database.yml' file in order to run the unit tests. " <<
+ "An example is provided in 'test/database.yml.example'.\n\n"
+ exit 1
end
View
30 test/thread_test.rb
@@ -4,20 +4,28 @@
class ThreadTest < Test::Unit::TestCase
MUTEX = Mutex.new
-
- def test_concurrency_not_allowed
- assert_raise ArgumentError do
- Object.class_eval %{
- class ThreadedEnchilada < ActiveRecord::Base
- self.allow_concurrency = true
- set_table_name :enchiladas
- data_fabric :prefix => 'fiveruns', :replicated => true, :shard_by => :city
- end
- }
+
+ if ActiveRecord::VERSION::STRING < '2.2.0'
+ def test_concurrency_not_allowed
+ assert_raise ArgumentError do
+ Object.class_eval %{
+ class ThreadedEnchilada < ActiveRecord::Base
+ self.allow_concurrency = true
+ set_table_name :enchiladas
+ data_fabric :prefix => 'fiveruns', :replicated => true, :shard_by => :city
+ end
+ }
+ end
end
end
- def xtest_class_and_instance_connections
+ def test_class_and_instance_connections
+ Object.class_eval %{
+ class ThreadedEnchilada < ActiveRecord::Base
+ set_table_name :enchiladas
+ data_fabric :prefix => 'fiveruns', :replicated => true, :shard_by => :city
+ end
+ }
ActiveRecord::Base.configurations = load_database_yml
cconn = ThreadedEnchilada.connection
Please sign in to comment.
Something went wrong with that request. Please try again.