Permalink
Browse files

1) adds AtomicSwitcher class to use atomic rename to switch origin & …

…destination tables

2) adds option[:atomic_switch] boolean to specify use of AtomicSwitch or LockedSwitcher
3) warn user if options[:atomic_switch] is not set & their version is affected by old
mysql bug, and default options[:atomic_switch] to false
  • Loading branch information...
1 parent 9d97475 commit 9fe3eef33b4571659890fc243bc7ed06f264765b @tiegz tiegz committed Mar 16, 2012
View
@@ -21,20 +21,23 @@ module Lhm
# Alters a table with the changes described in the block
#
# @param [String, Symbol] table_name Name of the table
- # @param [Hash] chunk_options Optional options to alter the chunk behavior
- # @option chunk_options [Fixnum] :stride
+ # @param [Hash] options Optional options to alter the chunk behavior
+ # @option options [Fixnum] :stride
# Size of a chunk (defaults to: 40,000)
- # @option chunk_options [Fixnum] :throttle
+ # @option options [Fixnum] :throttle
# Time to wait between chunks in milliseconds (defaults to: 100)
+ # @option options [Boolean] :atomic_switch
+ # Use atomic switch to rename tables (defaults to: true)
+ # (see SqlHelper#supports_atomic_switch?)
# @yield [Migrator] Yielded Migrator object records the changes
# @return [Boolean] Returns true if the migration finishes
# @raise [Error] Raises Lhm::Error in case of a error and aborts the migration
- def self.change_table(table_name, chunk_options = {}, &block)
+ def self.change_table(table_name, options = {}, &block)
connection = ActiveRecord::Base.connection
origin = Table.parse(table_name, connection)
invoker = Invoker.new(origin, connection)
block.call(invoker.migrator)
- invoker.run(chunk_options)
+ invoker.run(options)
true
end
@@ -0,0 +1,45 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require 'lhm/command'
+require 'lhm/migration'
+require 'lhm/sql_helper'
+
+module Lhm
+ # Switches origin with destination table using an atomic rename.
+ class AtomicSwitcher
+ include Command
+ include SqlHelper
+
+ attr_reader :connection
+
+ def initialize(migration, connection = nil)
+ @migration = migration
+ @connection = connection
+ @origin = migration.origin
+ @destination = migration.destination
+ end
+
+ def statements
+ atomic_switch
+ end
+
+ def atomic_switch
+ [
+ "rename table `#{ @origin.name }` to `#{ @migration.archive_name }`, " +
+ "`#{ @destination.name }` to `#{ @origin.name }`"
+ ]
+ end
+
+ def validate
+ unless table?(@origin.name) && table?(@destination.name)
+ error "`#{ @origin.name }` and `#{ @destination.name }` must exist"
+ end
+ end
+
+ private
+ def execute
+ sql statements
+ end
+ end
+end
View
@@ -3,6 +3,7 @@
require 'lhm/chunker'
require 'lhm/entangler'
+require 'lhm/atomic_switcher'
require 'lhm/locked_switcher'
require 'lhm/migrator'
@@ -13,19 +14,30 @@ module Lhm
# Once the origin and destination tables have converged, origin is archived
# and replaced by destination.
class Invoker
- attr_reader :migrator
+ include SqlHelper
+
+ attr_reader :migrator, :connection
def initialize(origin, connection)
@connection = connection
@migrator = Migrator.new(origin, connection)
end
- def run(chunk_options = {})
+ def run(options = {})
+ unless options.include?(:atomic_switch)
+ options[:atomic_switch] = supports_atomic_switch?
+ atomic_switch_warning unless options[:atomic_switch]
+ end
+
migration = @migrator.run
Entangler.new(migration, @connection).run do
- Chunker.new(migration, @connection, chunk_options).run
- LockedSwitcher.new(migration, @connection).run
+ Chunker.new(migration, @connection, options).run
+ if options[:atomic_switch]
+ AtomicSwitcher.new(migration, @connection).run
+ else
+ LockedSwitcher.new(migration, @connection).run
+ end
end
end
end
@@ -6,11 +6,7 @@
require 'lhm/sql_helper'
module Lhm
- # Switches origin with destination table with a write lock. Use this as
- # a safe alternative to rename, which can cause slave inconsistencies:
- #
- # http://bugs.mysql.com/bug.php?id=39675
- #
+ # Switches origin with destination table nonatomically using a locked write.
# LockedSwitcher adopts the Facebook strategy, with the following caveat:
#
# "Since alter table causes an implicit commit in innodb, innodb locks get
@@ -37,6 +33,7 @@ def statements
uncommitted { switch }
end
+
def switch
[
"lock table `#{ @origin.name }` write, `#{ @destination.name }` write",
View
@@ -40,6 +40,10 @@ def update(statements)
error e.message
end
+ def version_string
+ @version_string ||= connection.execute("show variables like 'version';").first.last
+ end
+
private
def tagged(statement)
@@ -51,5 +55,42 @@ def column_definition(cols)
column.to_s.match(/`?([^\(]+)`?(\([^\)]+\))?/).captures
end
end
+
+ # Older versions of MySQL contain an atomic rename bug affecting bin
+ # log order. Affected versions extract from bug report:
+ #
+ # http://bugs.mysql.com/bug.php?id=39675
+ #
+ # More Info: http://dev.mysql.com/doc/refman/5.5/en/metadata-locking.html
+ def supports_atomic_switch?
+ major, minor, tiny = version_string.split('.').map(&:to_i)
+
+ case major
+ when 4 then return false if minor and minor < 2
+ when 5
+ case minor
+ when 0 then return false if tiny and tiny < 52
+ when 1 then return false
+ when 4 then return false if tiny and tiny < 4
+ when 5 then return false if tiny and tiny < 3
+ end
+ when 6
+ case minor
+ when 0 then return false if tiny and tiny < 11
+ end
+ end
+ return true
+ end
+
+ def atomic_switch_warning
+ puts "\n********************************** WARNING **************************"
+ puts "* This version of mysql (#{version_string}) might not support atomic table"
+ puts "* renames while writing to binlogs [http://bugs.mysql.com/bug.php?id=39675]."
+ puts "* Defaulting to a nonatomic locking switch. You may cancel and force the"
+ puts "* atomic switch by restarting with options[:atomic_switch] => true"
+ puts "*********************************************************************\n"
+ puts "Continuing in... "
+ 30.downto(1).each { |i| puts "#{i}... "; sleep 1 }
+ end
end
end
@@ -0,0 +1,42 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
+
+require 'lhm/table'
+require 'lhm/migration'
+require 'lhm/atomic_switcher'
+
+describe Lhm::AtomicSwitcher do
+ include IntegrationHelper
+
+ before(:each) { connect_master! }
+
+ describe "switching" do
+ before(:each) do
+ @origin = table_create("origin")
+ @destination = table_create("destination")
+ @migration = Lhm::Migration.new(@origin, @destination)
+ end
+
+ it "rename origin to archive" do
+ switcher = Lhm::AtomicSwitcher.new(@migration, connection)
+ switcher.run
+
+ slave do
+ table_exists?(@origin).must_equal true
+ table_read(@migration.archive_name).columns.keys.must_include "origin"
+ end
+ end
+
+ it "rename destination to origin" do
+ switcher = Lhm::AtomicSwitcher.new(@migration, connection)
+ switcher.run
+
+ slave do
+ table_exists?(@destination).must_equal false
+ table_read(@origin.name).columns.keys.must_include "destination"
+ end
+ end
+ end
+end
@@ -0,0 +1,30 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
+
+require 'lhm/table'
+require 'lhm/migration'
+require 'lhm/atomic_switcher'
+
+describe Lhm::AtomicSwitcher do
+ include UnitHelper
+
+ before(:each) do
+ @start = Time.now
+ @origin = Lhm::Table.new("origin")
+ @destination = Lhm::Table.new("destination")
+ @migration = Lhm::Migration.new(@origin, @destination, @start)
+ @switcher = Lhm::AtomicSwitcher.new(@migration, nil)
+ end
+
+ describe "atomic switch" do
+ it "should perform a single atomic rename" do
+ @switcher.
+ statements.
+ must_equal([
+ "rename table `origin` to `#{ @migration.archive_name }`, `destination` to `origin`"
+ ])
+ end
+ end
+end
@@ -15,7 +15,7 @@
@origin = Lhm::Table.new("origin")
@destination = Lhm::Table.new("destination")
@migration = Lhm::Migration.new(@origin, @destination, @start)
- @switcher = Lhm::LockedSwitcher.new(@migration)
+ @switcher = Lhm::LockedSwitcher.new(@migration, nil)
end
describe "uncommitted" do

0 comments on commit 9fe3eef

Please sign in to comment.