Permalink
Browse files

Add AtomicSwitcher

The AtomicSwitcher provides atomic renaming of origin / destination
tables and will be used by default if mysql server version is not
affected by the famous rename bug (http://bugs.mysql.com/bug.php?id=39675).
  • Loading branch information...
2 parents 64d27ad + d9893a3 commit 198a96e2a738ecef99691095048d479331a0dd2e @grobie committed Mar 25, 2012
View
@@ -2,6 +2,7 @@
* Add option to specify custom index name
* Add mysql2 compatibility
+* Add AtomicSwitcher
# 1.0.3 (February 23, 2012)
View
@@ -21,22 +21,25 @@ module Lhm
# Alters a table with the changes described in the block
#
# @param [String, Symbol] table_name Name of the table
- # @param [Hash] chunk_options Optional options to alter the chunk behavior
- # @option chunk_options [Fixnum] :stride
+ # @param [Hash] options Optional options to alter the chunk / switch behavior
+ # @option options [Fixnum] :stride
# Size of a chunk (defaults to: 40,000)
- # @option chunk_options [Fixnum] :throttle
+ # @option options [Fixnum] :throttle
# Time to wait between chunks in milliseconds (defaults to: 100)
+ # @option options [Boolean] :atomic_switch
+ # Use atomic switch to rename tables (defaults to: true)
+ # If using a version of mysql affected by atomic switch bug, LHM forces user
+ # to set this option (see SqlHelper#supports_atomic_switch?)
# @yield [Migrator] Yielded Migrator object records the changes
# @return [Boolean] Returns true if the migration finishes
# @raise [Error] Raises Lhm::Error in case of a error and aborts the migration
- def self.change_table(table_name, chunk_options = {}, &block)
+ def self.change_table(table_name, options = {}, &block)
connection = ActiveRecord::Base.connection
origin = Table.parse(table_name, connection)
invoker = Invoker.new(origin, connection)
block.call(invoker.migrator)
- invoker.run(chunk_options)
+ invoker.run(options)
true
end
end
-
@@ -0,0 +1,45 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require 'lhm/command'
+require 'lhm/migration'
+require 'lhm/sql_helper'
+
+module Lhm
+ # Switches origin with destination table using an atomic rename.
+ class AtomicSwitcher
+ include Command
+ include SqlHelper
+
+ attr_reader :connection
+
+ def initialize(migration, connection = nil)
+ @migration = migration
+ @connection = connection
+ @origin = migration.origin
+ @destination = migration.destination
+ end
+
+ def statements
+ atomic_switch
+ end
+
+ def atomic_switch
+ [
+ "rename table `#{ @origin.name }` to `#{ @migration.archive_name }`, " +
+ "`#{ @destination.name }` to `#{ @origin.name }`"
+ ]
+ end
+
+ def validate
+ unless table?(@origin.name) && table?(@destination.name)
+ error "`#{ @origin.name }` and `#{ @destination.name }` must exist"
+ end
+ end
+
+ private
+ def execute
+ sql statements
+ end
+ end
+end
View
@@ -3,6 +3,7 @@
require 'lhm/chunker'
require 'lhm/entangler'
+require 'lhm/atomic_switcher'
require 'lhm/locked_switcher'
require 'lhm/migrator'
@@ -13,19 +14,35 @@ module Lhm
# Once the origin and destination tables have converged, origin is archived
# and replaced by destination.
class Invoker
- attr_reader :migrator
+ include SqlHelper
+
+ attr_reader :migrator, :connection
def initialize(origin, connection)
@connection = connection
@migrator = Migrator.new(origin, connection)
end
- def run(chunk_options = {})
+ def run(options = {})
+ if !options.include?(:atomic_switch)
+ if supports_atomic_switch?
+ options[:atomic_switch] = true
+ else
+ raise Error.new(
+ "Using mysql #{version_string}. You must explicitly set " +
+ "options[:atomic_switch] (re SqlHelper#supports_atomic_switch?)")
+ end
+ end
+
migration = @migrator.run
Entangler.new(migration, @connection).run do
- Chunker.new(migration, @connection, chunk_options).run
- LockedSwitcher.new(migration, @connection).run
+ Chunker.new(migration, @connection, options).run
+ if options[:atomic_switch]
+ AtomicSwitcher.new(migration, @connection).run
+ else
+ LockedSwitcher.new(migration, @connection).run
+ end
end
end
end
@@ -6,11 +6,7 @@
require 'lhm/sql_helper'
module Lhm
- # Switches origin with destination table with a write lock. Use this as
- # a safe alternative to rename, which can cause slave inconsistencies:
- #
- # http://bugs.mysql.com/bug.php?id=39675
- #
+ # Switches origin with destination table nonatomically using a locked write.
# LockedSwitcher adopts the Facebook strategy, with the following caveat:
#
# "Since alter table causes an implicit commit in innodb, innodb locks get
View
@@ -40,6 +40,10 @@ def update(statements)
error e.message
end
+ def version_string
+ connection.select_one("show variables like 'version'")["Value"]
+ end
+
private
def tagged(statement)
@@ -51,5 +55,31 @@ def column_definition(cols)
column.to_s.match(/`?([^\(]+)`?(\([^\)]+\))?/).captures
end
end
+
+ # Older versions of MySQL contain an atomic rename bug affecting bin
+ # log order. Affected versions extracted from bug report:
+ #
+ # http://bugs.mysql.com/bug.php?id=39675
+ #
+ # More Info: http://dev.mysql.com/doc/refman/5.5/en/metadata-locking.html
+ def supports_atomic_switch?
+ major, minor, tiny = version_string.split('.').map(&:to_i)
+
+ case major
+ when 4 then return false if minor and minor < 2
+ when 5
+ case minor
+ when 0 then return false if tiny and tiny < 52
+ when 1 then return false
+ when 4 then return false if tiny and tiny < 4
+ when 5 then return false if tiny and tiny < 3
+ end
+ when 6
+ case minor
+ when 0 then return false if tiny and tiny < 11
+ end
+ end
+ return true
+ end
end
end
@@ -0,0 +1,42 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
+
+require 'lhm/table'
+require 'lhm/migration'
+require 'lhm/atomic_switcher'
+
+describe Lhm::AtomicSwitcher do
+ include IntegrationHelper
+
+ before(:each) { connect_master! }
+
+ describe "switching" do
+ before(:each) do
+ @origin = table_create("origin")
+ @destination = table_create("destination")
+ @migration = Lhm::Migration.new(@origin, @destination)
+ end
+
+ it "rename origin to archive" do
+ switcher = Lhm::AtomicSwitcher.new(@migration, connection)
+ switcher.run
+
+ slave do
+ table_exists?(@origin).must_equal true
+ table_read(@migration.archive_name).columns.keys.must_include "origin"
+ end
+ end
+
+ it "rename destination to origin" do
+ switcher = Lhm::AtomicSwitcher.new(@migration, connection)
+ switcher.run
+
+ slave do
+ table_exists?(@destination).must_equal false
+ table_read(@origin.name).columns.keys.must_include "destination"
+ end
+ end
+ end
+end
@@ -16,7 +16,7 @@
end
it "should add a column" do
- Lhm.change_table(:users) do |t|
+ Lhm.change_table(:users, :atomic_switch => false) do |t|
t.add_column(:logins, "INT(12) DEFAULT '0'")
end
@@ -32,7 +32,7 @@
it "should copy all rows" do
23.times { |n| execute("insert into users set reference = '#{ n }'") }
- Lhm.change_table(:users) do |t|
+ Lhm.change_table(:users, :atomic_switch => false) do |t|
t.add_column(:logins, "INT(12) DEFAULT '0'")
end
@@ -42,7 +42,7 @@
end
it "should remove a column" do
- Lhm.change_table(:users) do |t|
+ Lhm.change_table(:users, :atomic_switch => false) do |t|
t.remove_column(:comment)
end
@@ -52,7 +52,7 @@
end
it "should add an index" do
- Lhm.change_table(:users) do |t|
+ Lhm.change_table(:users, :atomic_switch => false) do |t|
t.add_index([:comment, :created_at])
end
@@ -62,7 +62,7 @@
end
it "should add an index with a custom name" do
- Lhm.change_table(:users) do |t|
+ Lhm.change_table(:users, :atomic_switch => false) do |t|
t.add_index([:comment, :created_at], :my_index_name)
end
@@ -72,7 +72,7 @@
end
it "should add an index on a column with a reserved name" do
- Lhm.change_table(:users) do |t|
+ Lhm.change_table(:users, :atomic_switch => false) do |t|
t.add_index(:group)
end
@@ -82,7 +82,7 @@
end
it "should add a unqiue index" do
- Lhm.change_table(:users) do |t|
+ Lhm.change_table(:users, :atomic_switch => false) do |t|
t.add_unique_index(:comment)
end
@@ -92,7 +92,7 @@
end
it "should remove an index" do
- Lhm.change_table(:users) do |t|
+ Lhm.change_table(:users, :atomic_switch => false) do |t|
t.remove_index([:username, :created_at])
end
@@ -102,7 +102,7 @@
end
it "should remove an index with a custom name" do
- Lhm.change_table(:users) do |t|
+ Lhm.change_table(:users, :atomic_switch => false) do |t|
t.remove_index(:reference, :index_users_on_reference)
end
@@ -112,7 +112,7 @@
end
it "should apply a ddl statement" do
- Lhm.change_table(:users) do |t|
+ Lhm.change_table(:users, :atomic_switch => false) do |t|
t.ddl("alter table %s add column flag tinyint(1)" % t.name)
end
@@ -126,7 +126,7 @@
end
it "should change a column" do
- Lhm.change_table(:users) do |t|
+ Lhm.change_table(:users, :atomic_switch => false) do |t|
t.change_column(:comment, "varchar(20) DEFAULT 'none' NOT NULL")
end
@@ -142,7 +142,7 @@
it "should change the last column in a table" do
table_create(:small_table)
- Lhm.change_table(:small_table) do |t|
+ Lhm.change_table(:small_table, :atomic_switch => false) do |t|
t.change_column(:id, "int(5)")
end
@@ -166,7 +166,8 @@
end
end
- Lhm.change_table(:users, :stride => 10, :throttle => 97) do |t|
+ options = { :stride => 10, :throttle => 97, :atomic_switch => false }
+ Lhm.change_table(:users, options) do |t|
t.add_column(:parallel, "INT(10) DEFAULT '0'")
end
@@ -187,7 +188,8 @@
end
end
- Lhm.change_table(:users, :stride => 10, :throttle => 97) do |t|
+ options = { :stride => 10, :throttle => 97, :atomic_switch => false }
+ Lhm.change_table(:users, options) do |t|
t.add_column(:parallel, "INT(10) DEFAULT '0'")
end
@@ -0,0 +1,31 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
+
+require 'lhm/table'
+require 'lhm/migration'
+require 'lhm/atomic_switcher'
+
+describe Lhm::AtomicSwitcher do
+ include UnitHelper
+
+ before(:each) do
+ @start = Time.now
+ @origin = Lhm::Table.new("origin")
+ @destination = Lhm::Table.new("destination")
+ @migration = Lhm::Migration.new(@origin, @destination, @start)
+ @switcher = Lhm::AtomicSwitcher.new(@migration, nil)
+ end
+
+ describe "atomic switch" do
+ it "should perform a single atomic rename" do
+ @switcher.
+ statements.
+ must_equal([
+ "rename table `origin` to `#{ @migration.archive_name }`, " +
+ "`destination` to `origin`"
+ ])
+ end
+ end
+end
Oops, something went wrong.

0 comments on commit 198a96e

Please sign in to comment.