Browse files

explicity ensure that we still have the lock

  • Loading branch information...
1 parent c665187 commit 3f05b0cb380221d3bcaca1d152c09759ecd3cdf9 @ryanlecompte committed Aug 22, 2012
Showing with 22 additions and 3 deletions.
  1. +22 −3 lib/redis_failover/node_manager.rb
View
25 lib/redis_failover/node_manager.rb
@@ -13,6 +13,12 @@ class NodeManager
TIMEOUT = 5
# Default client failover ACK timeout.
DEFAULT_ACK_TIMEOUT = 10
+ # ZK Errors that the Node Manager cares about.
+ ZK_ERRORS = [
+ ZK::Exceptions::LockAssertionFailedError,
+ ZK::Exceptions::InterruptedSession,
+ ZKDisconnectedError
+ ].freeze
# Creates a new instance.
#
@@ -47,15 +53,15 @@ def start
@leader = false
setup_zk
logger.info('Waiting to become master Node Manager ...')
- @zk.with_lock(@lock_path) do
+ with_lock do
@leader = true
logger.info('Acquired master Node Manager lock')
discover_nodes
initialize_path
spawn_watchers
handle_state_reports
end
- rescue ZK::Exceptions::InterruptedSession, ZKDisconnectedError => ex
+ rescue *ZK_ERRORS => ex
logger.error("ZK error while attempting to manage nodes: #{ex.inspect}")
logger.error(ex.backtrace.join("\n"))
shutdown
@@ -78,6 +84,7 @@ def shutdown
@queue << nil
@watchers.each(&:shutdown) if @watchers
@zk.close! if @zk
+ @zk_lock = nil
end
private
@@ -110,6 +117,9 @@ def setup_zk
# Handles periodic state reports from {RedisFailover::NodeWatcher} instances.
def handle_state_reports
while state_report = @queue.pop
+ # Ensure that we still have the master lock.
+ @zk_lock.assert!
+
begin
node, state = state_report
case state
@@ -123,7 +133,7 @@ def handle_state_reports
# flush current state
write_state
- rescue ZK::Exceptions::InterruptedSession, ZKDisconnectedError
+ rescue *ZK_ERRORS
# fail hard if this is a ZK connection-related error
raise
rescue => ex
@@ -357,6 +367,15 @@ def write_state
@zk.set(@znode, encode(current_nodes))
end
+ # Executes a block wrapped in a ZK exclusive lock.
+ def with_lock
+ @zk_lock = @zk.locker(@lock_path)
+ @zk_lock.lock(true)
+ yield
+ ensure
+ @zk_lock.unlock!
+ end
+
# Schedules a manual failover to a redis node.
def schedule_manual_failover
return unless @leader

0 comments on commit 3f05b0c

Please sign in to comment.