Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Ruby-478 handle zombie processes in cluster framework, restore nearly…

… all connect tests
  • Loading branch information...
commit ca1503e99e4ff2cfcb7c98799adb2f1c67497f53 1 parent 66f7ab0
@gjmurakami-10gen gjmurakami-10gen authored
View
2  Gemfile
@@ -18,7 +18,7 @@ group :development, :test do
gem "ci_reporter"
gem "ruby-prof" unless RUBY_PLATFORM =~ /java/
gem "rake-compiler"
- # posix-spawn: XCode 4.4 - brew install apple-gcc42 && export CC=/usr/local/bin/gcc-4.2 && bundle install
+ # posix-spawn: XCode 4.4 - brew install apple-gcc42; export CC=/usr/local/bin/gcc-4.2 && bundle install
gem "posix-spawn" if RUBY_PLATFORM =~ /java/
# Java
View
4 README.md
@@ -345,6 +345,10 @@ To run a single test from its subdirectory, add -I.. since we no longer modify L
$ ruby -I.. -I../../lib bson_test.rb
+To fix the following error on Mac OS X - "/.../lib/bson_ext/cbson.bundle: [BUG] Segmentation fault":
+
+ $ rake compile
+
# Release Notes
See HISTORY.
View
227 test/replica_set/connect_test.rb
@@ -4,6 +4,11 @@ class ConnectTest < Test::Unit::TestCase
def setup
ensure_cluster(:rs)
+ @conn = nil
+ end
+
+ def teardown
+ @conn.close if @conn
end
def self.shutdown
@@ -12,7 +17,7 @@ def self.shutdown
end
# To reset after (test) failure
- # $ killall mongod; rm -fr rs
+ # $ rm -fr data
def step_down_command
# Adding force=true to avoid 'no secondaries within 10 seconds of my optime' errors
@@ -25,16 +30,16 @@ def step_down_command
# TODO: test connect timeout.
def test_connect_with_deprecated_multi
- host = @rs.servers.first.host
+ #replica_host_ports = @rs.replicas.collect{|replica| [replica.host, replica.port]}
+ host = @rs.replicas.first.host
@conn = Connection.multi([
# guaranteed to have one data-holding member
- [host, @rs.servers[0].port],
- [host, @rs.servers[1].port],
- [host, @rs.servers[2].port],
+ [host, @rs.replicas[0].port],
+ [host, @rs.replicas[1].port],
+ [host, @rs.replicas[2].port],
], :name => @rs.repl_set_name)
assert @conn.is_a?(ReplSetConnection)
assert @conn.connected?
- @conn.close
end
def test_connect_bad_name
@@ -50,63 +55,199 @@ def test_connect_with_first_secondary_node_terminated
@conn = ReplSetConnection.new @rs.repl_set_seeds
end
assert @conn.connected?
- @conn.close
end
- #def test_connect_with_last_secondary_node_terminated
- # @rs.secondaries.last.stop
- #
- # rescue_connection_failure do
- # @conn = ReplSetConnection.new @rs.repl_set_seeds
- # end
- # assert @conn.connected?
- # @conn.close
- #end
+ def test_connect_with_last_secondary_node_terminated
+ @rs.secondaries.last.stop
+
+ rescue_connection_failure do
+ @conn = ReplSetConnection.new @rs.repl_set_seeds
+ end
+ assert @conn.connected?
+ end
+
+ def test_connect_with_primary_stepped_down
+ @conn = ReplSetConnection.new @rs.repl_set_seeds
+ @conn[MONGO_TEST_DB]['bar'].save({:a => 1}, {:safe => {:w => 3}})
+ assert @conn[MONGO_TEST_DB]['bar'].find_one
+
+ primary = Mongo::Connection.new(@conn.primary_pool.host, @conn.primary_pool.port)
+ assert_raise Mongo::ConnectionFailure do
+ primary['admin'].command(step_down_command)
+ end
+ assert @conn.connected?
+
+ rescue_connection_failure do
+ @conn[MONGO_TEST_DB]['bar'].find_one
+ end
+ end
+
+ def test_connect_with_primary_killed
+ @conn = ReplSetConnection.new @rs.repl_set_seeds
+ assert @conn.connected?
+ @conn[MONGO_TEST_DB]['bar'].save({:a => 1}, {:safe => {:w => 3}})
+ assert @conn[MONGO_TEST_DB]['bar'].find_one
+
+ @rs.primary.kill(Signal.list['KILL'])
+
+ rescue_connection_failure do
+ @conn[MONGO_TEST_DB]['bar'].find_one
+ end
+ end
- #def test_connect_with_primary_stepped_down
+ def test_save_with_primary_stepped_down
+ @conn = ReplSetConnection.new @rs.repl_set_seeds
+ assert @conn.connected?
+
+ primary = Mongo::Connection.new(@conn.primary_pool.host, @conn.primary_pool.port)
+ assert_raise Mongo::ConnectionFailure do
+ primary['admin'].command(step_down_command)
+ end
+
+ rescue_connection_failure do
+ @conn[MONGO_TEST_DB]['bar'].save({:a => 1}, {:safe => {:w => 3}})
+ end
+ end
+
+ #def test_connect_with_first_node_removed
# @conn = ReplSetConnection.new @rs.repl_set_seeds
# @conn[MONGO_TEST_DB]['bar'].save({:a => 1}, {:safe => {:w => 3}})
- # assert @conn[MONGO_TEST_DB]['bar'].find_one
#
- # primary = Mongo::Connection.new(@conn.primary_pool.host, @conn.primary_pool.port)
+ # old_primary = [@conn.primary_pool.host, @conn.primary_pool.port]
+ # old_primary_conn = Mongo::Connection.new(*old_primary)
# assert_raise Mongo::ConnectionFailure do
- # primary['admin'].command(step_down_command)
+ # old_primary_conn['admin'].command(step_down_command)
# end
- # assert @conn.connected?
#
+ # # Wait for new primary
# rescue_connection_failure do
- # @conn[MONGO_TEST_DB]['bar'].find_one
+ # sleep 1 until @rs.get_node_with_state(1)
# end
- # @conn.close
- #end
-
- #def test_connect_with_primary_killed
- # @conn = ReplSetConnection.new @rs.repl_set_seeds
- # assert @conn.connected?
- # @conn[MONGO_TEST_DB]['bar'].save({:a => 1}, {:safe => {:w => 3}})
- # assert @conn[MONGO_TEST_DB]['bar'].find_one
#
- # @rs.primary.kill(Signal.list['KILL'])
+ # new_primary = @rs.get_all_host_pairs_with_state(1).first
+ # new_primary_conn = Mongo::Connection.new(*new_primary)
+ #
+ # config = nil
#
+ # # Remove old primary from replset
# rescue_connection_failure do
- # @conn[MONGO_TEST_DB]['bar'].find_one
+ # config = @conn['local']['system.replset'].find_one
# end
- # @conn.close
- #end
-
- #def test_save_with_primary_stepped_down
- # @conn = ReplSetConnection.new @rs.repl_set_seeds
- # assert @conn.connected?
#
- # primary = Mongo::Connection.new(@conn.primary_pool.host, @conn.primary_pool.port)
- # assert_raise Mongo::ConnectionFailure do
- # primary['admin'].command(step_down_command)
+ # old_member = config['members'].select {|m| m['host'] == old_primary.join(':')}.first
+ # config['members'].reject! {|m| m['host'] == old_primary.join(':')}
+ # config['version'] += 1
+ #
+ # begin
+ # new_primary_conn['admin'].command({'replSetReconfig' => config})
+ # rescue Mongo::ConnectionFailure
# end
#
+ # # Wait for the dust to settle
# rescue_connection_failure do
- # @conn[MONGO_TEST_DB]['bar'].save({:a => 1}, {:safe => {:w => 3}})
+ # assert @conn[MONGO_TEST_DB]['bar'].find_one
+ # end
+ #
+ # # Make sure a new connection skips the old primary
+ # @new_conn = ReplSetConnection.new @rs.repl_set_seeds
+ # @new_conn.connect
+ # new_nodes = [@new_conn.primary] + @new_conn.secondaries
+ # assert !(new_nodes).include?(old_primary)
+ #
+ # # Add the old primary back
+ # config['members'] << old_member
+ # config['version'] += 1
+ #
+ # begin
+ # new_primary_conn['admin'].command({'replSetReconfig' => config})
+ # rescue Mongo::ConnectionFailure
+ # end
+ #end
+
+ #def test_connect_with_hung_first_node
+ # hung_node = nil
+ # begin
+ # hung_node = IO.popen('nc -lk 127.0.0.1 29999 >/dev/null 2>&1')
+ #
+ # @conn = ReplSetConnection.new(['localhost:29999'] + @rs.repl_set_seeds,
+ # :connect_timeout => 2)
+ # @conn.connect
+ # assert ['localhost:29999'] != @conn.primary
+ # assert !@conn.secondaries.include?('localhost:29999')
+ # ensure
+ # Process.kill("KILL", hung_node.pid) if hung_node
# end
- # @conn.close
#end
+ def test_connect_with_connection_string
+ @conn = Connection.from_uri("mongodb://#{@rs.replicas[0].host_port},#{@rs.replicas[1].host_port}?replicaset=#{@rs.repl_set_name}")
+ assert @conn.is_a?(ReplSetConnection)
+ assert @conn.connected?
+ end
+
+ def test_connect_with_connection_string_in_env_var
+ ENV['MONGODB_URI'] = "mongodb://#{@rs.replicas[0].host_port},#{@rs.replicas[1].host_port}?replicaset=#{@rs.repl_set_name}"
+ @conn = ReplSetConnection.new
+ assert @conn.is_a?(ReplSetConnection)
+ assert_equal 2, @conn.seeds.length
+ assert_equal @rs.replicas[0].host, @conn.seeds[0][0]
+ assert_equal @rs.replicas[1].host, @conn.seeds[1][0]
+ assert_equal @rs.replicas[0].port, @conn.seeds[0][1]
+ assert_equal @rs.replicas[1].port, @conn.seeds[1][1]
+ assert_equal @rs.repl_set_name, @conn.replica_set_name
+ assert @conn.connected?
+ end
+
+ def test_connect_with_connection_string_in_implicit_mongodb_uri
+ ENV['MONGODB_URI'] = "mongodb://#{@rs.replicas[0].host_port},#{@rs.replicas[1].host_port}?replicaset=#{@rs.repl_set_name}"
+ @conn = Connection.from_uri
+ assert @conn.is_a?(ReplSetConnection)
+ assert_equal 2, @conn.seeds.length
+ assert_equal @rs.replicas[0].host, @conn.seeds[0][0]
+ assert_equal @rs.replicas[1].host, @conn.seeds[1][0]
+ assert_equal @rs.replicas[0].port, @conn.seeds[0][1]
+ assert_equal @rs.replicas[1].port, @conn.seeds[1][1]
+ assert_equal @rs.repl_set_name, @conn.replica_set_name
+ assert @conn.connected?
+ end
+
+ def test_connect_with_new_seed_format
+ @conn = ReplSetConnection.new @rs.repl_set_seeds
+ assert @conn.connected?
+ end
+
+ def test_connect_with_old_seed_format
+ silently do
+ @conn = ReplSetConnection.new(@rs.replicas[0].host_port_a, @rs.replicas[1].host_port_a)
+ end
+ assert @conn.connected?
+ end
+
+ def test_connect_with_full_connection_string
+ @conn = Connection.from_uri("mongodb://#{@rs.replicas[0].host_port},#{@rs.replicas[1].host_port}?replicaset=#{@rs.repl_set_name};safe=true;w=2;fsync=true;slaveok=true")
+ assert @conn.is_a?(ReplSetConnection)
+ assert @conn.connected?
+ assert_equal 2, @conn.safe[:w]
+ assert @conn.safe[:fsync]
+ assert @conn.read_pool
+ end
+
+ def test_connect_with_full_connection_string_in_env_var
+ ENV['MONGODB_URI'] = "mongodb://#{@rs.replicas[0].host_port},#{@rs.replicas[1].host_port}?replicaset=#{@rs.repl_set_name};safe=true;w=2;fsync=true;slaveok=true"
+ @conn = ReplSetConnection.new
+ assert @conn.is_a?(ReplSetConnection)
+ assert @conn.connected?
+ assert_equal 2, @conn.safe[:w]
+ assert @conn.safe[:fsync]
+ assert @conn.read_pool
+ end
+
+ def test_connect_options_override_env_var
+ ENV['MONGODB_URI'] = "mongodb://#{@rs.replicas[0].host_port},#{@rs.replicas[1].host_port}?replicaset=#{@rs.repl_set_name};safe=true;w=2;fsync=true;slaveok=true"
+ @conn = ReplSetConnection.new({:safe => false})
+ assert @conn.is_a?(ReplSetConnection)
+ assert @conn.connected?
+ assert_equal @conn.safe, false
+ end
+
end
View
44 test/tools/mongo_config.rb
@@ -24,10 +24,10 @@ def debug(level, arg)
#
module Mongo
class Config
- DEFAULT_BASE_OPTS = { :host => 'localhost', :dbpath => 'data' }
- DEFAULT_REPLICA_SET = DEFAULT_BASE_OPTS.merge( :replicas => 3 )
+ DEFAULT_BASE_OPTS = { :host => 'localhost', :dbpath => 'data', :logpath => 'data/log'}
+ DEFAULT_REPLICA_SET = DEFAULT_BASE_OPTS.merge( :replicas => 3, :arbiters => 0 )
DEFAULT_SHARDED_SIMPLE = DEFAULT_BASE_OPTS.merge( :shards => 2, :configs => 1, :routers => 2 )
- DEFAULT_SHARDED_REPLICA = DEFAULT_SHARDED_SIMPLE.merge( :replicas => 3 )
+ DEFAULT_SHARDED_REPLICA = DEFAULT_SHARDED_SIMPLE.merge( :replicas => 3, :arbiters => 0)
SERVER_PRELUDE_KEYS = [:host, :command]
SHARDING_OPT_KEYS = [:shards, :configs, :routers]
@@ -127,7 +127,15 @@ def initialize(cmd = nil)
@cmd = cmd
end
+ def clear_zombie
+ if @pid
+ pid = Process.wait(@pid, Process::WNOHANG)
+ @pid = nil if pid && pid > 0
+ end
+ end
+
def start(verifies = 0)
+ clear_zombie
return @pid if running?
begin
if RUBY_PLATFORM == 'java'
@@ -141,6 +149,7 @@ def start(verifies = 0)
end
#@pid = Process.spawn(@cmd, [:in, :out, :err] => :close)
end
+ sleep 1 # relinquish the processor so the child runs
verify(verifies) if verifies > 0
@pid
end
@@ -193,6 +202,10 @@ def initialize(cmd = nil, host = nil, port = nil)
def host_port
[@host, @port].join(':')
end
+
+ def host_port_a # for old format
+ [@host, @port]
+ end
end
class DbServer < Server
@@ -203,15 +216,15 @@ def initialize(config)
[dbpath, File.dirname(@config[:logpath])].compact.each{|dir| FileUtils.mkdir_p(dir) unless File.directory?(dir) }
command = @config[:command] || 'mongod'
params = @config.reject{|k,v| SERVER_PRELUDE_KEYS.include?(k)}
- arguments = params.collect do |arg, value|
+ arguments = params.sort{|a, b| a[0].to_s <=> b[0].to_s}.collect do |arg, value| # sort block is needed for 1.8.7 which lacks Symbol#<=>
argument = '--' + arg.to_s
- if FLAGS.member?(arg) && value == true
- [argument]
+ if FLAGS.member?(arg)
+ [value && argument]
else
[argument, value]
end
end
- cmd = [command, arguments].flatten.join(' ')
+ cmd = [command, arguments].flatten.compact.join(' ')
super(cmd, @config[:host], @config[:port])
end
@@ -222,7 +235,7 @@ def start(verifies = DEFAULT_VERIFIES)
def verify(verifies = 60)
verifies.times do |i|
- #puts "DbServer.verify - port: #{@port} iteration: #{i}"
+ #puts "DbServer.verify - port: #{@port} iteration: #{i} @pid:#{@pid.inspect} kill:#{Process.kill(0, @pid).inspect} running?:#{running?.inspect} cmd:#{cmd}"
begin
raise Mongo::ConnectionFailure unless running?
Mongo::Connection.new(@host, @port).close
@@ -311,8 +324,9 @@ def repl_set_name
end
def member_names_by_state(state)
+ states = Array(state)
status = repl_set_get_status
- status['members'].find_all{|member| member['state'] == state }.collect{|member| member['name']}
+ status['members'].find_all{|member| states.index(member['state']) }.collect{|member| member['name']}
end
def primary_name
@@ -323,6 +337,10 @@ def secondary_names
member_names_by_state(2)
end
+ def replica_names
+ member_names_by_state([1,2])
+ end
+
def arbiter_names
member_names_by_state(7)
end
@@ -331,8 +349,8 @@ def members_by_name(names)
names.collect do |name|
host, port = name.split(':')
port = port.to_i
- @servers[:replicas].find{|server| server.host == host && server.port == port}
- end
+ servers.find{|server| server.host == host && server.port == port}
+ end.compact
end
def primary
@@ -343,6 +361,10 @@ def secondaries
members_by_name(secondary_names)
end
+ def replicas
+ members_by_name(replica_names)
+ end
+
def arbiters
members_by_name(arbiter_names)
end
View
81 test/tools/mongo_config_test.rb
@@ -1,8 +1,15 @@
require 'test_helper'
-require 'tools/mongo_config'
class MongoConfig < Test::Unit::TestCase
+ def startup
+ @sys_proc = nil
+ end
+
+ def shutdown
+ @sys_proc.stop if @sys_proc && @sys_proc.running?
+ end
+
test "config defaults" do
[ Mongo::Config::DEFAULT_BASE_OPTS,
Mongo::Config::DEFAULT_REPLICA_SET,
@@ -20,34 +27,46 @@ class MongoConfig < Test::Unit::TestCase
test "SysProc start" do
cmd = "true"
- sys_proc = Mongo::Config::SysProc.new(cmd)
- assert_equal(cmd, sys_proc.cmd)
- assert_nil(sys_proc.pid)
- assert_not_nil(sys_proc.start(0))
- assert_not_nil(sys_proc.pid)
+ @sys_proc = Mongo::Config::SysProc.new(cmd)
+ assert_equal(cmd, @sys_proc.cmd)
+ assert_nil(@sys_proc.pid)
+ start_and_assert_running?(@sys_proc)
end
test "SysProc wait" do
- sys_proc = Mongo::Config::SysProc.new("true")
- assert_not_nil(sys_proc.start(0))
- assert(sys_proc.running?)
- sys_proc.wait
- assert(!sys_proc.running?)
+ @sys_proc = Mongo::Config::SysProc.new("true")
+ start_and_assert_running?(@sys_proc)
+ assert(@sys_proc.running?)
+ @sys_proc.wait
+ assert(!@sys_proc.running?)
end
test "SysProc kill" do
- sys_proc = Mongo::Config::SysProc.new("true")
- assert_not_nil(sys_proc.start(0))
- sys_proc.kill
- sys_proc.wait
- assert(!sys_proc.running?)
+ @sys_proc = Mongo::Config::SysProc.new("true")
+ start_and_assert_running?(@sys_proc)
+ @sys_proc.kill
+ @sys_proc.wait
+ assert(!@sys_proc.running?)
end
test "SysProc stop" do
- sys_proc = Mongo::Config::SysProc.new("true")
- assert_not_nil(sys_proc.start(0))
- sys_proc.stop
- assert(!sys_proc.running?)
+ @sys_proc = Mongo::Config::SysProc.new("true")
+ start_and_assert_running?(@sys_proc)
+ @sys_proc.stop
+ assert(!@sys_proc.running?)
+ end
+
+ test "SysProc zombie respawn" do
+ @sys_proc = Mongo::Config::SysProc.new("true")
+ start_and_assert_running?(@sys_proc)
+ prev_pid = @sys_proc.pid
+ @sys_proc.kill
+ # don't wait, leaving a zombie
+ assert(@sys_proc.running?)
+ start_and_assert_running?(@sys_proc)
+ assert(prev_pid && @sys_proc.pid && prev_pid != @sys_proc.pid, 'SysProc#start should spawn a new process after a zombie')
+ @sys_proc.stop
+ assert(!@sys_proc.running?)
end
test "Server" do
@@ -61,7 +80,7 @@ class MongoConfig < Test::Unit::TestCase
config = Mongo::Config::DEFAULT_BASE_OPTS
server = Mongo::Config::DbServer.new(config)
assert_equal(config, server.config)
- assert_equal("mongod --logpath data/log --dbpath data", server.cmd)
+ assert_equal("mongod --dbpath data --logpath data/log", server.cmd)
assert_equal(config[:host], server.host)
assert_equal(config[:port], server.port)
end
@@ -73,28 +92,28 @@ def cluster_test(opts)
manager = Mongo::Config::ClusterManager.new(config)
assert_equal(config, manager.config)
manager.start
- manager.servers.each{|s| p s}
+ #assert_not_nil(Mongo::Connection.new(manager.servers.first.host, manager.servers.first.port)) # servers is [] for DEFAULT_BASE_OPTS
manager.stop
- manager.servers.each{|s| assert_equal(false, s.running?)}
+ manager.servers.each{|s| assert(!s.running?)}
manager.clobber
end
test "cluster manager base" do
- #cluster_test(Mongo::Config::DEFAULT_BASE_OPTS)
+ cluster_test(Mongo::Config::DEFAULT_BASE_OPTS)
end
test "cluster manager replica set" do
- #cluster_test(Mongo::Config::DEFAULT_REPLICA_SET)
+ cluster_test(Mongo::Config::DEFAULT_REPLICA_SET)
end
test "cluster manager sharded simple" do
- #manager = Mongo::Config::ClusterManager.new(Mongo::Config.cluster(Mongo::Config::DEFAULT_SHARDED_SIMPLE)).start
opts = Mongo::Config::DEFAULT_SHARDED_SIMPLE
#debug 1, opts.inspect
config = Mongo::Config.cluster(opts)
#debug 1, config.inspect
manager = Mongo::Config::ClusterManager.new(config)
assert_equal(config, manager.config)
+ assert_no_match(/nojournal/, manager.servers.first.cmd, '--nojournal option should not be specified')
manager.start
#debug 1, manager.ismaster
#debug 1, manager.mongos_discover
@@ -102,7 +121,15 @@ def cluster_test(opts)
end
test "cluster manager sharded replica" do
- #cluster_test(Mongo::Config::DEFAULT_SHARDED_REPLICA)
+ #cluster_test(Mongo::Config::DEFAULT_SHARDED_REPLICA) # not yet supported by ClusterManager
+ end
+
+ private
+
+ def start_and_assert_running?(sys_proc)
+ assert_not_nil(sys_proc.start(0))
+ assert_not_nil(sys_proc.pid)
+ assert(sys_proc.running?)
end
end
Please sign in to comment.
Something went wrong with that request. Please try again.