From 2addb924a29453e2defe9fd6965cc1158b1f2da8 Mon Sep 17 00:00:00 2001 From: Wandenberg Date: Fri, 5 Sep 2014 10:01:13 -0300 Subject: [PATCH 1/4] add support to operation timeout, stopping long queries --- Gemfile | 1 + lib/moped/connection.rb | 22 +++++++++++++++++----- lib/moped/errors.rb | 3 +++ lib/moped/session.rb | 5 +++++ spec/moped/query_spec.rb | 36 ++++++++++++++++++++++++++++++++++++ spec/spec_helper.rb | 30 ++++++++++++++++++++++++++++++ 6 files changed, 92 insertions(+), 5 deletions(-) diff --git a/Gemfile b/Gemfile index bf798ad..b3603fa 100644 --- a/Gemfile +++ b/Gemfile @@ -1,6 +1,7 @@ source "https://rubygems.org" group :test do + gem "popen4" gem "rspec", "~> 2.14.1" if ENV["CI"] gem "coveralls", :require => false diff --git a/lib/moped/connection.rb b/lib/moped/connection.rb index 0041867..80cad54 100644 --- a/lib/moped/connection.rb +++ b/lib/moped/connection.rb @@ -191,12 +191,24 @@ def write(operations) # # @since 1.2.9 def read_data(socket, length) - data = socket.read(length) - unless data - raise Errors::ConnectionFailure.new( - "Attempted to read #{length} bytes from the socket but nothing was returned." - ) + # Block on data to read for op_timeout seconds + begin + op_timeout = @options[:op_timeout] || timeout + ready = IO.select([socket], nil, [socket], op_timeout) + unless ready + raise Errors::OperationTimeout.new("Took more than #{op_timeout} seconds to receive data.") + end + rescue IOError => e + raise Errors::ConnectionFailure + end + + # Read data from socket + begin + data = socket.read(length) + rescue SystemCallError, IOError => e + raise Errors::ConnectionFailure.new("Attempted to read #{length} bytes from the socket but an error happend #{e.message}.") end + if data.length < length data << read_data(socket, length - data.length) end diff --git a/lib/moped/errors.rb b/lib/moped/errors.rb index c4c27b7..b62689e 100644 --- a/lib/moped/errors.rb +++ b/lib/moped/errors.rb @@ -20,6 +20,9 @@ class PoolTimeout < RuntimeError; end # Generic error class for exceptions related to connection failures. class ConnectionFailure < StandardError; end + # Generic error class for exceptions related to read timeout failures. + class OperationTimeout < StandardError; end + # Raised when a database name is invalid. class InvalidDatabaseName < StandardError; end diff --git a/lib/moped/session.rb b/lib/moped/session.rb index 1ada2b4..3d8f928 100644 --- a/lib/moped/session.rb +++ b/lib/moped/session.rb @@ -235,6 +235,11 @@ def logout # @since 2.0.0 option(:timeout).allow(Optionable.any(Numeric)) + # Setup validation of allowed timeout options. (Any numeric) + # + # @since 2.0.0 + option(:op_timeout).allow(Optionable.any(Numeric)) + # Pass an object that responds to instrument as an instrumenter. # # @since 2.0.0 diff --git a/spec/moped/query_spec.rb b/spec/moped/query_spec.rb index 534259c..a79a011 100644 --- a/spec/moped/query_spec.rb +++ b/spec/moped/query_spec.rb @@ -1018,6 +1018,42 @@ end end + context "with test commands enabled" do + + let(:session) do + Moped::Session.new([ "127.0.0.1:#{port}" ], database: "moped_test") + end + + let(:users) do + session.with(safe: true)[:users] + end + + describe "when a query take too long" do + let(:port) { 31100 } + + before do + start_mongo_server(port, "--setParameter enableTestCommands=1") + Process.detach(spawn("echo 'db.adminCommand({sleep: 1, w: true, secs: 10})' | mongo localhost:#{port}", :out => "/dev/null", :err => "/dev/null")) + sleep(1) # to sleep command on mongodb begins work + end + + after do + stop_mongo_server(port) + end + + it "raises a operation timeout exception" do + time = Benchmark.realtime do + expect { + Timeout::timeout(7) do + users.find("age" => { "$gte" => 65 }).first + end + }.to raise_exception("Took more than 5 seconds to receive data.") + end + expect(time).to be < 5.5 + end + end + end + context "with a remote connection", mongohq: :auth do before(:all) do diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 341a488..b74c21e 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -12,6 +12,10 @@ $:.unshift((Pathname(__FILE__).dirname.parent + "lib").to_s) +require "benchmark" +require "fileutils" +require "tmpdir" +require "popen4" require "moped" require "support/examples" require "support/mongohq" @@ -36,7 +40,33 @@ Moped::Connection::Manager.instance_variable_set(:@pools, {}) end + config.after(:suite) do + stop_mongo_server(31100) + end + unless Support::MongoHQ.replica_set_configured? || Support::MongoHQ.auth_node_configured? $stderr.puts Support::MongoHQ.message end end + + +def start_mongo_server(port, extra_options=nil) + stop_mongo_server(port) + dbpath = File.join(Dir.tmpdir, port.to_s) + FileUtils.mkdir_p(dbpath) + POpen4::popen4("mongod --oplogSize 40 --noprealloc --smallfiles --port #{port} --dbpath #{dbpath} --logpath #{dbpath}/log --pidfilepath #{dbpath}/pid --fork #{extra_options}") do |stdout, stderr, stdin, pid| + error_message = stderr.read.strip unless stderr.eof + raise StandardError.new error_message unless error_message.nil? + end + + while `echo 'db.runCommand({ping:1}).ok' | mongo --quiet --port #{port}`.chomp != "1" + sleep 0.1 + end +end + +def stop_mongo_server(port) + dbpath = File.join(Dir.tmpdir, port.to_s) + pidfile = File.join(dbpath, "pid") + `kill #{File.read(pidfile).chomp}` if File.exists?(pidfile) + FileUtils.rm_rf(dbpath) +end From 94478059445b7b974eff4f0b2c9b8a1b059e4789 Mon Sep 17 00:00:00 2001 From: Wandenberg Date: Fri, 5 Sep 2014 10:09:43 -0300 Subject: [PATCH 2/4] move with_retry method to a different module --- lib/moped/read_preference/selectable.rb | 37 ++------------------- lib/moped/retryable.rb | 43 +++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 35 deletions(-) create mode 100644 lib/moped/retryable.rb diff --git a/lib/moped/read_preference/selectable.rb b/lib/moped/read_preference/selectable.rb index 73686de..148f58d 100644 --- a/lib/moped/read_preference/selectable.rb +++ b/lib/moped/read_preference/selectable.rb @@ -1,4 +1,5 @@ # encoding: utf-8 +require "moped/retryable" module Moped module ReadPreference @@ -7,6 +8,7 @@ module ReadPreference # # @since 2.0.0 module Selectable + include Retryable # @!attribute tags # @return [ Array ] The tag sets. @@ -39,41 +41,6 @@ def query_options(options) options[:flags] |= [ :slave_ok ] options end - - private - - # Execute the provided block on the cluster and retry if the execution - # fails. - # - # @api private - # - # @example Execute with retry. - # preference.with_retry(cluster) do - # cluster.with_primary do |node| - # node.refresh - # end - # end - # - # @param [ Cluster ] cluster The cluster. - # @param [ Integer ] retries The number of times to retry. - # - # @return [ Object ] The result of the block. - # - # @since 2.0.0 - def with_retry(cluster, retries = cluster.max_retries, &block) - begin - block.call - rescue Errors::ConnectionFailure => e - if retries > 0 - Loggable.warn(" MOPED:", "Retrying connection attempt #{retries} more time(s).", "n/a") - sleep(cluster.retry_interval) - cluster.refresh - with_retry(cluster, retries - 1, &block) - else - raise e - end - end - end end end end diff --git a/lib/moped/retryable.rb b/lib/moped/retryable.rb new file mode 100644 index 0000000..27a6f92 --- /dev/null +++ b/lib/moped/retryable.rb @@ -0,0 +1,43 @@ +# encoding: utf-8 +module Moped + # Provides the shared behaviour for retry failed operations. + # + # @since 2.0.0 + module Retryable + + private + + # Execute the provided block on the cluster and retry if the execution + # fails. + # + # @api private + # + # @example Execute with retry. + # preference.with_retry(cluster) do + # cluster.with_primary do |node| + # node.refresh + # end + # end + # + # @param [ Cluster ] cluster The cluster. + # @param [ Integer ] retries The number of times to retry. + # + # @return [ Object ] The result of the block. + # + # @since 2.0.0 + def with_retry(cluster, retries = cluster.max_retries, &block) + begin + block.call + rescue Errors::ConnectionFailure => e + if retries > 0 + Loggable.warn(" MOPED:", "Retrying connection attempt #{retries} more time(s).", "n/a") + sleep(cluster.retry_interval) + cluster.refresh + with_retry(cluster, retries - 1, &block) + else + raise e + end + end + end + end +end From d6ea5f63982c2e0df2cbc7dcb25797daf42ec425 Mon Sep 17 00:00:00 2001 From: Wandenberg Date: Fri, 5 Sep 2014 10:12:27 -0300 Subject: [PATCH 3/4] retry the operation if the failure was due to a cluster reconfiguration and the node is not a master anymore --- lib/moped/collection.rb | 10 ++- lib/moped/query.rb | 55 ++++++++------- lib/moped/retryable.rb | 4 +- spec/moped/cluster_spec.rb | 140 +++++++++++++++++++++++++++++++++++++ spec/spec_helper.rb | 3 + 5 files changed, 184 insertions(+), 28 deletions(-) diff --git a/lib/moped/collection.rb b/lib/moped/collection.rb index d68f8da..5aa8f43 100644 --- a/lib/moped/collection.rb +++ b/lib/moped/collection.rb @@ -1,5 +1,6 @@ # encoding: utf-8 require "moped/query" +require "moped/retryable" module Moped @@ -8,6 +9,7 @@ module Moped # @since 1.0.0 class Collection include Readable + include Retryable # @!attribute database # @return [ Database ] The database for the collection. @@ -120,9 +122,11 @@ def initialize(database, name) # # @since 1.0.0 def insert(documents, flags = nil) - docs = documents.is_a?(Array) ? documents : [ documents ] - cluster.with_primary do |node| - node.insert(database.name, name, docs, write_concern, flags: flags || []) + with_retry(cluster) do + docs = documents.is_a?(Array) ? documents : [ documents ] + cluster.with_primary do |node| + node.insert(database.name, name, docs, write_concern, flags: flags || []) + end end end diff --git a/lib/moped/query.rb b/lib/moped/query.rb index f308010..de6af6e 100644 --- a/lib/moped/query.rb +++ b/lib/moped/query.rb @@ -21,6 +21,7 @@ module Moped # people.find.count # => 1 class Query include Enumerable + include Retryable # @attribute [r] collection The collection to execute the query on. # @attribute [r] operation The query operation. @@ -321,14 +322,16 @@ def modify(change, options = {}) # # @since 1.0.0 def remove - cluster.with_primary do |node| - node.remove( - operation.database, - operation.collection, - operation.basic_selector, - write_concern, - flags: [ :remove_first ] - ) + with_retry(cluster) do + cluster.with_primary do |node| + node.remove( + operation.database, + operation.collection, + operation.basic_selector, + write_concern, + flags: [ :remove_first ] + ) + end end end @@ -341,13 +344,15 @@ def remove # # @since 1.0.0 def remove_all - cluster.with_primary do |node| - node.remove( - operation.database, - operation.collection, - operation.basic_selector, - write_concern - ) + with_retry(cluster) do + cluster.with_primary do |node| + node.remove( + operation.database, + operation.collection, + operation.basic_selector, + write_concern + ) + end end end @@ -423,15 +428,17 @@ def tailable # # @since 1.0.0 def update(change, flags = nil) - cluster.with_primary do |node| - node.update( - operation.database, - operation.collection, - operation.selector["$query"] || operation.selector, - change, - write_concern, - flags: flags - ) + with_retry(cluster) do + cluster.with_primary do |node| + node.update( + operation.database, + operation.collection, + operation.selector["$query"] || operation.selector, + change, + write_concern, + flags: flags + ) + end end end diff --git a/lib/moped/retryable.rb b/lib/moped/retryable.rb index 27a6f92..33f7252 100644 --- a/lib/moped/retryable.rb +++ b/lib/moped/retryable.rb @@ -28,7 +28,9 @@ module Retryable def with_retry(cluster, retries = cluster.max_retries, &block) begin block.call - rescue Errors::ConnectionFailure => e + rescue Errors::ConnectionFailure, Errors::PotentialReconfiguration => e + raise e if e.is_a?(Errors::PotentialReconfiguration) && !e.message.include?("not master") + if retries > 0 Loggable.warn(" MOPED:", "Retrying connection attempt #{retries} more time(s).", "n/a") sleep(cluster.retry_interval) diff --git a/spec/moped/cluster_spec.rb b/spec/moped/cluster_spec.rb index f51111c..7f0f985 100644 --- a/spec/moped/cluster_spec.rb +++ b/spec/moped/cluster_spec.rb @@ -524,3 +524,143 @@ end end end + +describe Moped::Cluster, "after a reconfiguration" do + let(:options) do + { + max_retries: 30, + retry_interval: 1, + timeout: 5, + database: 'test_db', + read: :primary, + write: {w: 'majority'} + } + end + + let(:replica_set_name) { 'dev' } + + let(:session) do + Moped::Session.new([ "127.0.0.1:31100", "127.0.0.1:31101", "127.0.0.1:31102" ], options) + end + + def step_down_servers + file = Tempfile.new('step_down') + begin + user_data = with_authentication? ? ", 'admin', 'admin_pwd'" : "" + file.puts %{ + function stepDown(dbs) { + for (i in dbs) { + dbs[i].adminCommand({replSetFreeze:5}); + try { dbs[i].adminCommand({replSetStepDown:5}); } catch(e) { print(e) }; + } + }; + + var db1 = connect('localhost:31100/admin'#{user_data}); + var db2 = connect('localhost:31101/admin'#{user_data}); + var db3 = connect('localhost:31102/admin'#{user_data}); + + var dbs = [db1, db2, db3]; + stepDown(dbs); + + while (db1.adminCommand({ismaster:1}).ismaster || db2.adminCommand({ismaster:1}).ismaster || db2.adminCommand({ismaster:1}).ismaster) { + stepDown(dbs); + } + } + file.close + + system "mongo --nodb #{file.path} 2>&1 > /dev/null" + + ensure + file.close + file.unlink # deletes the temp file + end + end + + shared_examples_for "recover the session" do + it "should execute commands normally before the stepDown" do + time = Benchmark.realtime do + session[:foo].find().remove_all() + session[:foo].find().to_a.count.should eql(0) + session[:foo].insert({ name: "bar 1" }) + session[:foo].find().to_a.count.should eql(1) + expect { + session[:foo].insert({ name: "bar 1" }) + }.to raise_exception + end + time.should be < 2 + end + + it "should recover and execute a find" do + session[:foo].find().remove_all() + session[:foo].insert({ name: "bar 1" }) + step_down_servers + time = Benchmark.realtime do + session[:foo].find().to_a.count.should eql(1) + end + time.should be > 5 + time.should be < 29 + end + + it "should recover and execute an insert" do + session[:foo].find().remove_all() + session[:foo].insert({ name: "bar 1" }) + step_down_servers + time = Benchmark.realtime do + session[:foo].insert({ name: "bar 2" }) + session[:foo].find().to_a.count.should eql(2) + end + time.should be > 5 + time.should be < 29 + + session[:foo].insert({ name: "bar 3" }) + session[:foo].find().to_a.count.should eql(3) + end + + it "should recover and try an insert which hit a constraint" do + session[:foo].find().remove_all() + session[:foo].insert({ name: "bar 1" }) + step_down_servers + time = Benchmark.realtime do + expect { + session[:foo].insert({ name: "bar 1" }) + }.to raise_exception + end + time.should be > 5 + time.should be < 29 + + session[:foo].find().to_a.count.should eql(1) + + session[:foo].insert({ name: "bar 2" }) + session[:foo].find().to_a.count.should eql(2) + end + end + + describe "with authentication off" do + before do + rs_initiated = `echo 'db.isMaster().me' | mongo --quiet --port 31100 2>/dev/null`.chomp.end_with?("31100") + unless rs_initiated + start_mongo_server(31100, "--replSet #{replica_set_name}") + start_mongo_server(31101, "--replSet #{replica_set_name}") + start_mongo_server(31102, "--replSet #{replica_set_name}") + + `echo "rs.initiate({_id : '#{replica_set_name}', 'members' : [{_id:0, host:'localhost:31100'},{_id:1, host:'localhost:31101'},{_id:2, host:'localhost:31102'}]})" | mongo --port 31100` + rs_up = false + while !rs_up + status = `echo 'rs.status().members[0].stateStr + "|" + rs.status().members[1].stateStr + "|" + rs.status().members[2].stateStr' | mongo --quiet --port 31100`.chomp.split("|") + rs_up = status.all?{|st| st == "PRIMARY" || st == "SECONDARY"} + end + + master = `echo 'db.isMaster().primary' | mongo --quiet --port 31100`.chomp + + `echo " + use test_db; + db.foo.ensureIndex({name:1}, {unique:1}); + " | mongo #{master}` + end + end + + let(:with_authentication?) { false } + + it_should_behave_like "recover the session" + end +end \ No newline at end of file diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index b74c21e..b45d19d 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -15,6 +15,7 @@ require "benchmark" require "fileutils" require "tmpdir" +require "tempfile" require "popen4" require "moped" require "support/examples" @@ -42,6 +43,8 @@ config.after(:suite) do stop_mongo_server(31100) + stop_mongo_server(31101) + stop_mongo_server(31102) end unless Support::MongoHQ.replica_set_configured? || Support::MongoHQ.auth_node_configured? From bff8f65440a4e3e669e0baa279a218050104933d Mon Sep 17 00:00:00 2001 From: Wandenberg Date: Fri, 5 Sep 2014 10:23:23 -0300 Subject: [PATCH 4/4] retry operation and refresh authentication if the failure was related with authorization. after a cluster reconfiguration with auth enabled as example --- lib/moped/authenticatable.rb | 6 ++++ lib/moped/cluster.rb | 4 +++ lib/moped/node.rb | 6 ++++ lib/moped/retryable.rb | 4 ++- spec/moped/cluster_spec.rb | 60 ++++++++++++++++++++++++++++++++++++ 5 files changed, 79 insertions(+), 1 deletion(-) diff --git a/lib/moped/authenticatable.rb b/lib/moped/authenticatable.rb index 88103b7..24695c9 100644 --- a/lib/moped/authenticatable.rb +++ b/lib/moped/authenticatable.rb @@ -94,5 +94,11 @@ def logout(database) end credentials.delete(database) end + + def refresh_authentication + credentials.each do |database, (username, password)| + login(database, username, password) + end + end end end diff --git a/lib/moped/cluster.rb b/lib/moped/cluster.rb index 2bb17cb..c9e8459 100644 --- a/lib/moped/cluster.rb +++ b/lib/moped/cluster.rb @@ -195,6 +195,10 @@ def refresh(nodes_to_refresh = seeds) refreshed_nodes end + def refresh_authentication(nodes_to_refresh = seeds) + nodes_to_refresh.each(&:refresh_authentication) + end + # Get the interval in which the node list should be refreshed. # # @example Get the refresh interval, in seconds. diff --git a/lib/moped/node.rb b/lib/moped/node.rb index 0ede953..1f7d208 100644 --- a/lib/moped/node.rb +++ b/lib/moped/node.rb @@ -443,6 +443,12 @@ def refresh end end + def refresh_authentication + connection do |conn| + conn.refresh_authentication + end + end + # Execute a remove command for the provided selector. # # @example Remove documents. diff --git a/lib/moped/retryable.rb b/lib/moped/retryable.rb index 33f7252..707d31e 100644 --- a/lib/moped/retryable.rb +++ b/lib/moped/retryable.rb @@ -29,12 +29,14 @@ def with_retry(cluster, retries = cluster.max_retries, &block) begin block.call rescue Errors::ConnectionFailure, Errors::PotentialReconfiguration => e - raise e if e.is_a?(Errors::PotentialReconfiguration) && !e.message.include?("not master") + raise e if e.is_a?(Errors::PotentialReconfiguration) && + !(e.message.include?("not master") || e.message.include?("not authorized") || e.message.include?("unauthorized")) if retries > 0 Loggable.warn(" MOPED:", "Retrying connection attempt #{retries} more time(s).", "n/a") sleep(cluster.retry_interval) cluster.refresh + cluster.refresh_authentication if e.message.include?("not authorized") || e.message.include?("unauthorized") with_retry(cluster, retries - 1, &block) else raise e diff --git a/spec/moped/cluster_spec.rb b/spec/moped/cluster_spec.rb index 7f0f985..29a140f 100644 --- a/spec/moped/cluster_spec.rb +++ b/spec/moped/cluster_spec.rb @@ -663,4 +663,64 @@ def step_down_servers it_should_behave_like "recover the session" end + + describe "with authentication on" do + before do + rs_initiated = `echo 'db.isMaster().me' | mongo --quiet --port 31100 -u admin -p admin_pwd --authenticationDatabase admin 2>/dev/null`.chomp.end_with?("31100") + unless rs_initiated + keyfile = File.join(Dir.tmpdir, "31000", "keyfile") + FileUtils.mkdir_p(File.dirname(keyfile)) + File.open(keyfile, "w") do |f| f.puts "SyrfEmAevWPEbgRZoZx9qZcZtJAAfd269da+kzi0H/7OuowGLxM3yGGUHhD379qP +nw4X8TT2T6ecx6aqJgxG+biJYVOpNK3HHU9Dp5q6Jd0bWGHGGbgFHV32/z2FFiti +EFLimW/vfn2DcJwTW29nQWhz2wN+xfMuwA6hVxFczlQlz5hIY0+a+bQChKw8wDZk +rW1OjTQ//csqPbVA8fwB49ghLGp+o84VujhRxLJ+0sbs8dKoIgmVlX2kLeHGQSf0 +KmF9b8kAWRLwLneOR3ESovXpEoK0qpQb2ym6BNqP32JKyPA6Svb/smVONhjUI71f +/zQ2ETX7ylpxIzw2SMv/zOWcVHBqIbdP9Llrxb3X0EsB6J8PeI8qLjpS94FyEddw +ACMcAxbP+6BaLjXyJ2WsrEeqThAyUC3uF5YN/oQ9XiATqP7pDOTrmfn8LvryyzcB +ByrLRTPOicBaG7y13ATcCbBdrYH3BE4EeLkTUZOg7VzvRnATvDpt0wOkSnbqXow8 +GQ6iMUgd2XvUCuknQLD6gWyoUyHiPADKrLsgnd3Qo9BPxYJ9VWSKB4phK3N7Bic+ +BwxlcpDFzGI285GR4IjcJbRRjjywHq5XHOxrJfN+QrZ/6wy6yu2+4NTPj+BPC5iX +/dNllTEyn7V+pr6FiRv8rv8RcxJgf3nfn/Xz0t2zW2olcalEFxwKKmR20pZxPnSv +Kr6sVHEzh0mtA21LoK5G8bztXsgFgWU7hh9z8UUo7KQQnDfyPb6k4xroeeQtWBNo +TZF1pI5joLytNSEtT+BYA5wQSYm4WCbhG+j7ipcPIJw6Un4ZtAZs0aixDfVE0zo0 +w2FWrYH2dmmCMbz7cEXeqvQiHh9IU/hkTrKGY95STszGGFFjhtS2TbHAn2rRoFI0 +VwNxMJCC+9ZijTWBeGyQOuEupuI4C9IzA5Gz72048tpZ0qMJ9mOiH3lZFtNTg/5P +28Td2xzaujtXjRnP3aZ9z2lKytlr +" + end + + File.chmod(0600, keyfile) + + start_mongo_server(31100, "--replSet #{replica_set_name} --keyFile #{keyfile} --auth") + start_mongo_server(31101, "--replSet #{replica_set_name} --keyFile #{keyfile} --auth") + start_mongo_server(31102, "--replSet #{replica_set_name} --keyFile #{keyfile} --auth") + + `echo "rs.initiate({_id : '#{replica_set_name}', 'members' : [{_id:0, host:'localhost:31100'},{_id:1, host:'localhost:31101'},{_id:2, host:'localhost:31102'}]})" | mongo --port 31100` + rs_up = false + while !rs_up + status = `echo 'rs.status().members[0].stateStr + "|" + rs.status().members[1].stateStr + "|" + rs.status().members[2].stateStr' | mongo --quiet --port 31100`.chomp.split("|") + rs_up = status.all?{|st| st == "PRIMARY" || st == "SECONDARY"} + end + + master = `echo 'db.isMaster().primary' | mongo --quiet --port 31100`.chomp + + `echo " + use admin; + db.addUser('admin', 'admin_pwd'); + " | mongo #{master}` + + `echo " + use test_db; + db.addUser('common', 'common_pwd'); + db.foo.ensureIndex({name:1}, {unique:1}); + " | mongo #{master} -u admin -p admin_pwd --authenticationDatabase admin` + end + + session.login('common', 'common_pwd') + end + + let(:with_authentication?) { true } + + it_should_behave_like "recover the session" + end end \ No newline at end of file