Merge branch 'cornsilk_14_8_acu162901_rightlink_rc3_fixes'

Gemfile.lock

@@ -137,7 +137,7 @@ GEM
       mime-types (>= 1.16)
       net-http-persistent
       netrc (~> 0.7.7)
-    right_agent (2.1.2)
+    right_agent (2.1.4)
       eventmachine (>= 0.12.10, < 2.0)
       faye-websocket (= 0.7.0)
       ffi
@@ -147,7 +147,7 @@ GEM
       rest-client (= 1.7.0.1)
       right_amqp (~> 0.7)
       right_support (>= 2.4.1, < 3.0)
-    right_agent (2.1.2-x86-mingw32)
+    right_agent (2.1.4-x86-mingw32)
       eventmachine (>= 0.12.10, < 2.0)
       faye-websocket (= 0.7.0)
       ffi

actors/instance_setup.rb

@@ -50,6 +50,9 @@ class InstanceSetup
   # How long to wait for tag query before proceeding to boot sequence
   INITIAL_TAG_QUERY_TIMEOUT = 20
 
+  # Maximum time before re-enroll after being triggered
+  MAX_REENROLL_DELAY = 60
+
   # Boot if and only if instance state is 'booting'
   # Prime timer for shutdown on unsuccessful boot ('suicide' functionality)
   #
@@ -100,7 +103,6 @@ def report_state
   end
 
   # Handle status update from agent by adjusting offline mode or re-enrolling
-  # Ignore router disconnects since they do not prevent sending requests
   #
   # === Parameters
   # type(Symbol):: Type of client: :auth, :api, :router, :broker
@@ -109,17 +111,24 @@ def report_state
   # === Return
   # true:: Always return true
   def update_status(type, state)
-    if [:auth, :api, :broker].include?(type)
+    if [:auth, :api, :router, :broker].include?(type)
       case state
       when :connected
-        RightScale::Sender.instance.disable_offline_mode
+        if @agent.mode == :http
+          # Require connectivity to both API and router to be considered fully connected
+          status = @agent.client.status
+          if status[:api] == :connected && status[:router] == :connected
+            RightScale::Sender.instance.disable_offline_mode
+          end
+        else
+          RightScale::Sender.instance.disable_offline_mode
+        end
       when :disconnected
         RightScale::Sender.instance.enable_offline_mode
       when :failed
         RightScale::Log.error("RightNet connectivity failure for #{type}, need to re-enroll")
-        RightScale::ReenrollManager.vote
-        RightScale::ReenrollManager.vote
-        RightScale::ReenrollManager.vote
+        # Randomize when re-enroll to prevent possibly having multiple instances do so simultaneously
+        EM.add_timer(rand(MAX_REENROLL_DELAY)) { RightScale::ReenrollManager.reenroll! }
       when :authorized, :unauthorized, :expired, :closing
       else
         RightScale::Log.error("Unrecognized state for #{type}: #{state}")

spec/actors/instance_setup_spec.rb

@@ -714,22 +714,56 @@ def mock_vm_assign_device(planned_volumes, volumes)
       flexmock(RightScale::Sender).should_receive(:instance).and_return(@sender)
     end
 
-    [:auth, :api, :broker].each do |type|
+    [:auth, :api, :router, :broker].each do |type|
       context "from #{type}" do
-        it 'should go online when become connected' do
-          @sender.should_receive(:disable_offline_mode).once
-          @setup.update_status(type, :connected)
+        if type == :broker
+          it 'should go online when become connected' do
+            @sender.should_receive(:disable_offline_mode).once
+            @setup.update_status(type, :connected)
+          end
+        else
+          before(:each) do
+            @status = {:auth => :connected, :api => :connected, :router => :connected}
+            @client = flexmock("client")
+            @client.should_receive(:status).and_return(@status).by_default
+            flexmock(@agent).should_receive(:client).and_return(@client).by_default
+            flexmock(@agent).should_receive(:mode).and_return(:http).by_default
+          end
+
+          it 'should go online if :api and :router are also connected' do
+            @client.should_receive(:status).and_return(@status).once
+            @sender.should_receive(:disable_offline_mode).once
+            @sender.should_receive(:enable_offline_mode).never
+            @setup.update_status(type, :connected)
+          end
+
+          it 'should not go online if :api is not connected' do
+            @client.should_receive(:status).and_return(@status.merge(:api => :disconnected)).once
+            @sender.should_receive(:disable_offline_mode).never
+            @sender.should_receive(:enable_offline_mode).never
+            @setup.update_status(type, :connected)
+          end
+
+          it 'should not go online if :router is not connected' do
+            @client.should_receive(:status).and_return(@status.merge(:router => :disconnected)).once
+            @sender.should_receive(:disable_offline_mode).never
+            @sender.should_receive(:enable_offline_mode).never
+            @setup.update_status(type, :connected)
+          end
         end
 
         it 'should go offline when become disconnected' do
           @sender.should_receive(:enable_offline_mode).once
+          @sender.should_receive(:disable_offline_mode).never
           @setup.update_status(type, :disconnected)
         end
 
         it 'should re-enroll when connection fails' do
           flexmock(RightScale::Log).should_receive(:error).with("RightNet connectivity failure for #{type}, need to re-enroll").once
           @sender.should_receive(:enable_offline_mode).never
-          @reenroller.should_receive(:vote).times(3)
+          flexmock(@setup).should_receive(:rand).with(60).and_return(15).once
+          flexmock(EM).should_receive(:add_timer).with(15, Proc).and_yield.once
+          @reenroller.should_receive(:reenroll!).once
           @setup.update_status(type, :failed)
         end
 
@@ -742,10 +776,5 @@ def mock_vm_assign_device(planned_volumes, volumes)
         end
       end
     end
-
-    it 'should not go offline when router becomes disconnected' do
-      @sender.should_receive(:enable_offline_mode).never
-      @setup.update_status(:router, :disconnected)
-    end
   end
 end