Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

1) adds AtomicSwitcher class to use atomic rename to switch origin & …

…destination tables

2) adds option[:atomic_switch] boolean to specify use of AtomicSwitch or LockedSwitcher
3) warn user if options[:atomic_switch] is not set & their version is affected by old
mysql bug, and default options[:atomic_switch] to false
  • Loading branch information...
commit 9fe3eef33b4571659890fc243bc7ed06f264765b 1 parent 9d97475
@tiegz tiegz authored
View
13 lib/lhm.rb
@@ -21,20 +21,23 @@ module Lhm
# Alters a table with the changes described in the block
#
# @param [String, Symbol] table_name Name of the table
- # @param [Hash] chunk_options Optional options to alter the chunk behavior
- # @option chunk_options [Fixnum] :stride
+ # @param [Hash] options Optional options to alter the chunk behavior
+ # @option options [Fixnum] :stride
# Size of a chunk (defaults to: 40,000)
- # @option chunk_options [Fixnum] :throttle
+ # @option options [Fixnum] :throttle
# Time to wait between chunks in milliseconds (defaults to: 100)
+ # @option options [Boolean] :atomic_switch
+ # Use atomic switch to rename tables (defaults to: true)
+ # (see SqlHelper#supports_atomic_switch?)
# @yield [Migrator] Yielded Migrator object records the changes
# @return [Boolean] Returns true if the migration finishes
# @raise [Error] Raises Lhm::Error in case of a error and aborts the migration
- def self.change_table(table_name, chunk_options = {}, &block)
+ def self.change_table(table_name, options = {}, &block)
connection = ActiveRecord::Base.connection
origin = Table.parse(table_name, connection)
invoker = Invoker.new(origin, connection)
block.call(invoker.migrator)
- invoker.run(chunk_options)
+ invoker.run(options)
true
end
View
45 lib/lhm/atomic_switcher.rb
@@ -0,0 +1,45 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require 'lhm/command'
+require 'lhm/migration'
+require 'lhm/sql_helper'
+
+module Lhm
+ # Switches origin with destination table using an atomic rename.
+ class AtomicSwitcher
+ include Command
+ include SqlHelper
+
+ attr_reader :connection
+
+ def initialize(migration, connection = nil)
+ @migration = migration
+ @connection = connection
+ @origin = migration.origin
+ @destination = migration.destination
+ end
+
+ def statements
+ atomic_switch
+ end
+
+ def atomic_switch
+ [
+ "rename table `#{ @origin.name }` to `#{ @migration.archive_name }`, " +
+ "`#{ @destination.name }` to `#{ @origin.name }`"
+ ]
+ end
+
+ def validate
+ unless table?(@origin.name) && table?(@destination.name)
+ error "`#{ @origin.name }` and `#{ @destination.name }` must exist"
+ end
+ end
+
+ private
+ def execute
+ sql statements
+ end
+ end
+end
View
20 lib/lhm/invoker.rb
@@ -3,6 +3,7 @@
require 'lhm/chunker'
require 'lhm/entangler'
+require 'lhm/atomic_switcher'
require 'lhm/locked_switcher'
require 'lhm/migrator'
@@ -13,19 +14,30 @@ module Lhm
# Once the origin and destination tables have converged, origin is archived
# and replaced by destination.
class Invoker
- attr_reader :migrator
+ include SqlHelper
+
+ attr_reader :migrator, :connection
def initialize(origin, connection)
@connection = connection
@migrator = Migrator.new(origin, connection)
end
- def run(chunk_options = {})
+ def run(options = {})
+ unless options.include?(:atomic_switch)
+ options[:atomic_switch] = supports_atomic_switch?
+ atomic_switch_warning unless options[:atomic_switch]
+ end
+
migration = @migrator.run
Entangler.new(migration, @connection).run do
- Chunker.new(migration, @connection, chunk_options).run
- LockedSwitcher.new(migration, @connection).run
+ Chunker.new(migration, @connection, options).run
+ if options[:atomic_switch]
+ AtomicSwitcher.new(migration, @connection).run
+ else
+ LockedSwitcher.new(migration, @connection).run
+ end
end
end
end
View
7 lib/lhm/locked_switcher.rb
@@ -6,11 +6,7 @@
require 'lhm/sql_helper'
module Lhm
- # Switches origin with destination table with a write lock. Use this as
- # a safe alternative to rename, which can cause slave inconsistencies:
- #
- # http://bugs.mysql.com/bug.php?id=39675
- #
+ # Switches origin with destination table nonatomically using a locked write.
# LockedSwitcher adopts the Facebook strategy, with the following caveat:
#
# "Since alter table causes an implicit commit in innodb, innodb locks get
@@ -37,6 +33,7 @@ def statements
uncommitted { switch }
end
+
def switch
[
"lock table `#{ @origin.name }` write, `#{ @destination.name }` write",
View
41 lib/lhm/sql_helper.rb
@@ -40,6 +40,10 @@ def update(statements)
error e.message
end
+ def version_string
+ @version_string ||= connection.execute("show variables like 'version';").first.last
+ end
+
private
def tagged(statement)
@@ -51,5 +55,42 @@ def column_definition(cols)
column.to_s.match(/`?([^\(]+)`?(\([^\)]+\))?/).captures
end
end
+
+ # Older versions of MySQL contain an atomic rename bug affecting bin
+ # log order. Affected versions extract from bug report:
+ #
+ # http://bugs.mysql.com/bug.php?id=39675
+ #
+ # More Info: http://dev.mysql.com/doc/refman/5.5/en/metadata-locking.html
+ def supports_atomic_switch?
+ major, minor, tiny = version_string.split('.').map(&:to_i)
+
+ case major
+ when 4 then return false if minor and minor < 2
+ when 5
+ case minor
+ when 0 then return false if tiny and tiny < 52
+ when 1 then return false
+ when 4 then return false if tiny and tiny < 4
+ when 5 then return false if tiny and tiny < 3
+ end
+ when 6
+ case minor
+ when 0 then return false if tiny and tiny < 11
+ end
+ end
+ return true
+ end
+
+ def atomic_switch_warning
+ puts "\n********************************** WARNING **************************"
+ puts "* This version of mysql (#{version_string}) might not support atomic table"
+ puts "* renames while writing to binlogs [http://bugs.mysql.com/bug.php?id=39675]."
+ puts "* Defaulting to a nonatomic locking switch. You may cancel and force the"
+ puts "* atomic switch by restarting with options[:atomic_switch] => true"
+ puts "*********************************************************************\n"
+ puts "Continuing in... "
+ 30.downto(1).each { |i| puts "#{i}... "; sleep 1 }
+ end
end
end
View
42 spec/integration/atomic_switcher_spec.rb
@@ -0,0 +1,42 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
+
+require 'lhm/table'
+require 'lhm/migration'
+require 'lhm/atomic_switcher'
+
+describe Lhm::AtomicSwitcher do
+ include IntegrationHelper
+
+ before(:each) { connect_master! }
+
+ describe "switching" do
+ before(:each) do
+ @origin = table_create("origin")
+ @destination = table_create("destination")
+ @migration = Lhm::Migration.new(@origin, @destination)
+ end
+
+ it "rename origin to archive" do
+ switcher = Lhm::AtomicSwitcher.new(@migration, connection)
+ switcher.run
+
+ slave do
+ table_exists?(@origin).must_equal true
+ table_read(@migration.archive_name).columns.keys.must_include "origin"
+ end
+ end
+
+ it "rename destination to origin" do
+ switcher = Lhm::AtomicSwitcher.new(@migration, connection)
+ switcher.run
+
+ slave do
+ table_exists?(@destination).must_equal false
+ table_read(@origin.name).columns.keys.must_include "destination"
+ end
+ end
+ end
+end
View
30 spec/unit/atomic_switcher_spec.rb
@@ -0,0 +1,30 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
+
+require 'lhm/table'
+require 'lhm/migration'
+require 'lhm/atomic_switcher'
+
+describe Lhm::AtomicSwitcher do
+ include UnitHelper
+
+ before(:each) do
+ @start = Time.now
+ @origin = Lhm::Table.new("origin")
+ @destination = Lhm::Table.new("destination")
+ @migration = Lhm::Migration.new(@origin, @destination, @start)
+ @switcher = Lhm::AtomicSwitcher.new(@migration, nil)
+ end
+
+ describe "atomic switch" do
+ it "should perform a single atomic rename" do
+ @switcher.
+ statements.
+ must_equal([
+ "rename table `origin` to `#{ @migration.archive_name }`, `destination` to `origin`"
+ ])
+ end
+ end
+end
View
2  spec/unit/locked_switcher_spec.rb
@@ -15,7 +15,7 @@
@origin = Lhm::Table.new("origin")
@destination = Lhm::Table.new("destination")
@migration = Lhm::Migration.new(@origin, @destination, @start)
- @switcher = Lhm::LockedSwitcher.new(@migration)
+ @switcher = Lhm::LockedSwitcher.new(@migration, nil)
end
describe "uncommitted" do
Please sign in to comment.
Something went wrong with that request. Please try again.