Add AtomicSwitcher

The AtomicSwitcher provides atomic renaming of origin / destination
tables and will be used by default if mysql server version is not
affected by the famous rename bug (http://bugs.mysql.com/bug.php?id=39675).

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 * Add option to specify custom index name
 * Add mysql2 compatibility
+* Add AtomicSwitcher
 
 # 1.0.3 (February 23, 2012)
 

lib/lhm.rb

@@ -21,22 +21,25 @@ module Lhm
   # Alters a table with the changes described in the block
   #
   # @param [String, Symbol] table_name Name of the table
-  # @param [Hash] chunk_options Optional options to alter the chunk behavior
-  # @option chunk_options [Fixnum] :stride
+  # @param [Hash] options Optional options to alter the chunk / switch behavior
+  # @option options [Fixnum] :stride
   #   Size of a chunk (defaults to: 40,000)
-  # @option chunk_options [Fixnum] :throttle
+  # @option options [Fixnum] :throttle
   #   Time to wait between chunks in milliseconds (defaults to: 100)
+  # @option options [Boolean] :atomic_switch
+  #   Use atomic switch to rename tables (defaults to: true)
+  #   If using a version of mysql affected by atomic switch bug, LHM forces user
+  #   to set this option (see SqlHelper#supports_atomic_switch?)
   # @yield [Migrator] Yielded Migrator object records the changes
   # @return [Boolean] Returns true if the migration finishes
   # @raise [Error] Raises Lhm::Error in case of a error and aborts the migration
-  def self.change_table(table_name, chunk_options = {}, &block)
+  def self.change_table(table_name, options = {}, &block)
     connection = ActiveRecord::Base.connection
     origin = Table.parse(table_name, connection)
     invoker = Invoker.new(origin, connection)
     block.call(invoker.migrator)
-    invoker.run(chunk_options)
+    invoker.run(options)
 
     true
   end
 end
-

lib/lhm/atomic_switcher.rb

@@ -0,0 +1,45 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require 'lhm/command'
+require 'lhm/migration'
+require 'lhm/sql_helper'
+
+module Lhm
+  # Switches origin with destination table using an atomic rename.
+  class AtomicSwitcher
+    include Command
+    include SqlHelper
+
+    attr_reader :connection
+
+    def initialize(migration, connection = nil)
+      @migration = migration
+      @connection = connection
+      @origin = migration.origin
+      @destination = migration.destination
+    end
+
+    def statements
+      atomic_switch
+    end
+
+    def atomic_switch
+      [
+        "rename table `#{ @origin.name }` to `#{ @migration.archive_name }`, " +
+        "`#{ @destination.name }` to `#{ @origin.name }`"
+      ]
+    end
+
+    def validate
+      unless table?(@origin.name) && table?(@destination.name)
+        error "`#{ @origin.name }` and `#{ @destination.name }` must exist"
+      end
+    end
+
+  private
+    def execute
+      sql statements
+    end
+  end
+end

lib/lhm/invoker.rb

@@ -3,6 +3,7 @@
 
 require 'lhm/chunker'
 require 'lhm/entangler'
+require 'lhm/atomic_switcher'
 require 'lhm/locked_switcher'
 require 'lhm/migrator'
 
@@ -13,19 +14,35 @@ module Lhm
   # Once the origin and destination tables have converged, origin is archived
   # and replaced by destination.
   class Invoker
-    attr_reader :migrator
+    include SqlHelper
+
+    attr_reader :migrator, :connection
 
     def initialize(origin, connection)
       @connection = connection
       @migrator = Migrator.new(origin, connection)
     end
 
-    def run(chunk_options = {})
+    def run(options = {})
+      if !options.include?(:atomic_switch)
+        if supports_atomic_switch?
+          options[:atomic_switch] = true
+        else
+          raise Error.new(
+            "Using mysql #{version_string}. You must explicitly set " +
+            "options[:atomic_switch] (re SqlHelper#supports_atomic_switch?)")
+        end
+      end
+
       migration = @migrator.run
 
       Entangler.new(migration, @connection).run do
-        Chunker.new(migration, @connection, chunk_options).run
-        LockedSwitcher.new(migration, @connection).run
+        Chunker.new(migration, @connection, options).run
+        if options[:atomic_switch]
+          AtomicSwitcher.new(migration, @connection).run
+        else
+          LockedSwitcher.new(migration, @connection).run
+        end
       end
     end
   end

lib/lhm/locked_switcher.rb

@@ -6,11 +6,7 @@
 require 'lhm/sql_helper'
 
 module Lhm
-  # Switches origin with destination table with a write lock. Use this as
-  # a safe alternative to rename, which can cause slave inconsistencies:
-  #
-  #   http://bugs.mysql.com/bug.php?id=39675
-  #
+  # Switches origin with destination table nonatomically using a locked write.
   # LockedSwitcher adopts the Facebook strategy, with the following caveat:
   #
   #   "Since alter table causes an implicit commit in innodb, innodb locks get

lib/lhm/sql_helper.rb

@@ -40,6 +40,10 @@ def update(statements)
       error e.message
     end
 
+    def version_string
+      connection.select_one("show variables like 'version'")["Value"]
+    end
+
   private
 
     def tagged(statement)
@@ -51,5 +55,31 @@ def column_definition(cols)
         column.to_s.match(/`?([^\(]+)`?(\([^\)]+\))?/).captures
       end
     end
+
+    # Older versions of MySQL contain an atomic rename bug affecting bin
+    # log order. Affected versions extracted from bug report:
+    #
+    #   http://bugs.mysql.com/bug.php?id=39675
+    #
+    # More Info: http://dev.mysql.com/doc/refman/5.5/en/metadata-locking.html
+    def supports_atomic_switch?
+      major, minor, tiny = version_string.split('.').map(&:to_i)
+
+      case major
+      when 4 then return false if minor and minor < 2
+      when 5
+        case minor
+        when 0 then return false if tiny and tiny < 52
+        when 1 then return false
+        when 4 then return false if tiny and tiny < 4
+        when 5 then return false if tiny and tiny < 3
+        end
+      when 6
+        case minor
+        when 0 then return false if tiny and tiny < 11
+        end
+      end
+      return true
+    end
   end
 end

spec/integration/atomic_switcher_spec.rb

@@ -0,0 +1,42 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
+
+require 'lhm/table'
+require 'lhm/migration'
+require 'lhm/atomic_switcher'
+
+describe Lhm::AtomicSwitcher do
+  include IntegrationHelper
+
+  before(:each) { connect_master! }
+
+  describe "switching" do
+    before(:each) do
+      @origin = table_create("origin")
+      @destination = table_create("destination")
+      @migration = Lhm::Migration.new(@origin, @destination)
+    end
+
+    it "rename origin to archive" do
+      switcher = Lhm::AtomicSwitcher.new(@migration, connection)
+      switcher.run
+
+      slave do
+        table_exists?(@origin).must_equal true
+        table_read(@migration.archive_name).columns.keys.must_include "origin"
+      end
+    end
+
+    it "rename destination to origin" do
+      switcher = Lhm::AtomicSwitcher.new(@migration, connection)
+      switcher.run
+
+      slave do
+        table_exists?(@destination).must_equal false
+        table_read(@origin.name).columns.keys.must_include "destination"
+      end
+    end
+  end
+end

spec/integration/lhm_spec.rb

@@ -16,7 +16,7 @@
     end
 
     it "should add a column" do
-      Lhm.change_table(:users) do |t|
+      Lhm.change_table(:users, :atomic_switch => false) do |t|
         t.add_column(:logins, "INT(12) DEFAULT '0'")
       end
 
@@ -32,7 +32,7 @@
     it "should copy all rows" do
       23.times { |n| execute("insert into users set reference = '#{ n }'") }
 
-      Lhm.change_table(:users) do |t|
+      Lhm.change_table(:users, :atomic_switch => false) do |t|
         t.add_column(:logins, "INT(12) DEFAULT '0'")
       end
 
@@ -42,7 +42,7 @@
     end
 
     it "should remove a column" do
-      Lhm.change_table(:users) do |t|
+      Lhm.change_table(:users, :atomic_switch => false) do |t|
         t.remove_column(:comment)
       end
 
@@ -52,7 +52,7 @@
     end
 
     it "should add an index" do
-      Lhm.change_table(:users) do |t|
+      Lhm.change_table(:users, :atomic_switch => false) do |t|
         t.add_index([:comment, :created_at])
       end
 
@@ -62,7 +62,7 @@
     end
 
     it "should add an index with a custom name" do
-      Lhm.change_table(:users) do |t|
+      Lhm.change_table(:users, :atomic_switch => false) do |t|
         t.add_index([:comment, :created_at], :my_index_name)
       end
 
@@ -72,7 +72,7 @@
     end
 
     it "should add an index on a column with a reserved name" do
-      Lhm.change_table(:users) do |t|
+      Lhm.change_table(:users, :atomic_switch => false) do |t|
         t.add_index(:group)
       end
 
@@ -82,7 +82,7 @@
     end
 
     it "should add a unqiue index" do
-      Lhm.change_table(:users) do |t|
+      Lhm.change_table(:users, :atomic_switch => false) do |t|
         t.add_unique_index(:comment)
       end
 
@@ -92,7 +92,7 @@
     end
 
     it "should remove an index" do
-      Lhm.change_table(:users) do |t|
+      Lhm.change_table(:users, :atomic_switch => false) do |t|
         t.remove_index([:username, :created_at])
       end
 
@@ -102,7 +102,7 @@
     end
 
     it "should remove an index with a custom name" do
-      Lhm.change_table(:users) do |t|
+      Lhm.change_table(:users, :atomic_switch => false) do |t|
         t.remove_index(:reference, :index_users_on_reference)
       end
 
@@ -112,7 +112,7 @@
     end
 
     it "should apply a ddl statement" do
-      Lhm.change_table(:users) do |t|
+      Lhm.change_table(:users, :atomic_switch => false) do |t|
         t.ddl("alter table %s add column flag tinyint(1)" % t.name)
       end
 
@@ -126,7 +126,7 @@
     end
 
     it "should change a column" do
-      Lhm.change_table(:users) do |t|
+      Lhm.change_table(:users, :atomic_switch => false) do |t|
         t.change_column(:comment, "varchar(20) DEFAULT 'none' NOT NULL")
       end
 
@@ -142,7 +142,7 @@
     it "should change the last column in a table" do
       table_create(:small_table)
 
-      Lhm.change_table(:small_table) do |t|
+      Lhm.change_table(:small_table, :atomic_switch => false) do |t|
         t.change_column(:id, "int(5)")
       end
 
@@ -166,7 +166,8 @@
           end
         end
 
-        Lhm.change_table(:users, :stride => 10, :throttle => 97) do |t|
+        options = { :stride => 10, :throttle => 97, :atomic_switch => false }
+        Lhm.change_table(:users, options) do |t|
           t.add_column(:parallel, "INT(10) DEFAULT '0'")
         end
 
@@ -187,7 +188,8 @@
           end
         end
 
-        Lhm.change_table(:users, :stride => 10, :throttle => 97) do |t|
+        options = { :stride => 10, :throttle => 97, :atomic_switch => false }
+        Lhm.change_table(:users, options) do |t|
           t.add_column(:parallel, "INT(10) DEFAULT '0'")
         end
 

spec/unit/atomic_switcher_spec.rb

@@ -0,0 +1,31 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
+
+require 'lhm/table'
+require 'lhm/migration'
+require 'lhm/atomic_switcher'
+
+describe Lhm::AtomicSwitcher do
+  include UnitHelper
+
+  before(:each) do
+    @start       = Time.now
+    @origin      = Lhm::Table.new("origin")
+    @destination = Lhm::Table.new("destination")
+    @migration   = Lhm::Migration.new(@origin, @destination, @start)
+    @switcher    = Lhm::AtomicSwitcher.new(@migration, nil)
+  end
+
+  describe "atomic switch" do
+    it "should perform a single atomic rename" do
+      @switcher.
+        statements.
+        must_equal([
+          "rename table `origin` to `#{ @migration.archive_name }`, " +
+          "`destination` to `origin`"
+        ])
+    end
+  end
+end