Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

improved shutdown handling for node manager

  • Loading branch information...
commit 68b4d7d14ef88975a559042e81214088733d4466 1 parent 7bb5f00
@ryanlecompte authored
View
1  Changes.md
@@ -1,6 +1,7 @@
0.9.5
-----------
- Introduce a safer master node discovery process for the Node Manager (#34)
+- Improved shutdown process for Node Manager
0.9.4
-----------
View
28 lib/redis_failover/node_manager.rb
@@ -51,10 +51,11 @@ def start
with_lock do
@leader = true
logger.info('Acquired master Node Manager lock')
- discover_nodes
- initialize_path
- spawn_watchers
- handle_state_reports
+ if discover_nodes
+ initialize_path
+ spawn_watchers
+ handle_state_reports
+ end
end
rescue *ZK_ERRORS => ex
logger.error("ZK error while attempting to manage nodes: #{ex.inspect}")
@@ -74,19 +75,17 @@ def notify_state(node, state)
# Performs a reset of the manager.
def reset
@leader = false
- @queue.clear
- @queue << nil
@watchers.each(&:shutdown) if @watchers
- sleep(TIMEOUT)
+ @queue.clear
@zk.close! if @zk
@zk_lock = nil
end
# Initiates a graceful shutdown.
def shutdown
+ logger.info('Shutting down ...')
@mutex.synchronize do
@shutdown = true
- reset
end
end
@@ -232,9 +231,10 @@ def promote_new_master(node = nil)
end
# Discovers the current master and slave nodes.
+ # @return [Boolean] true if nodes successfully discovered, false otherwise
def discover_nodes
@mutex.synchronize do
- return unless running?
+ return false unless running?
nodes = @options[:nodes].map { |opts| Node.new(opts) }.uniq
if @master = find_existing_master
logger.info("Using master #{@master} from existing znode config.")
@@ -244,9 +244,9 @@ def discover_nodes
@slaves = nodes - [@master]
logger.info("Managing master (#{@master}) and slaves " +
"(#{@slaves.map(&:to_s).join(', ')})")
-
# ensure that slaves are correctly pointing to this master
redirect_slaves_to(@master)
+ true
end
rescue NodeUnavailableError, NoMasterError, MultipleMastersError => ex
msg = <<-MSG.gsub(/\s+/, ' ')
@@ -391,11 +391,13 @@ def write_state
# Executes a block wrapped in a ZK exclusive lock.
def with_lock
@zk_lock = @zk.locker(@lock_path)
- until @zk_lock.lock
- return unless running?
+ while running? && !@zk_lock.lock
sleep(TIMEOUT)
end
- yield
+
+ if running?
+ yield
+ end
ensure
@zk_lock.unlock! if @zk_lock
end
View
4 lib/redis_failover/node_watcher.rb
@@ -35,8 +35,8 @@ def shutdown
@done = true
@node.wakeup
@monitor_thread.join if @monitor_thread
- rescue
- # best effort
+ rescue => ex
+ logger.warn("Failed to gracefully shutdown watcher for #{@node}")
end
private
View
16 lib/redis_failover/runner.rb
@@ -8,22 +8,20 @@ class Runner
# Node Manager is gracefully stopped
def self.run(options)
options = CLI.parse(options)
- @node_manager = NodeManager.new(options)
- trap_signals
- @node_manager_thread = Thread.new { @node_manager.start }
- @node_manager_thread.join
+ node_manager = NodeManager.new(options)
+ trap_signals(node_manager)
+ node_manager.start
end
# Traps shutdown signals.
- def self.trap_signals
+ # @param [NodeManager] node_manager the node manager
+ def self.trap_signals(node_manager)
[:INT, :TERM].each do |signal|
trap(signal) do
- Util.logger.info('Shutting down ...')
- @node_manager.shutdown
- @node_manager_thread.join
- exit(0)
+ node_manager.shutdown
end
end
end
+ private_class_method :trap_signals
end
end
View
2  spec/support/node_manager_stub.rb
@@ -6,7 +6,7 @@ class NodeManagerStub < NodeManager
def discover_nodes
# only discover nodes once in testing
- return if @nodes_discovered
+ return true if @nodes_discovered
master = Node.new(:host => 'master')
slave = Node.new(:host => 'slave')
Please sign in to comment.
Something went wrong with that request. Please try again.