Merge "Fix the redis warning: Could not cleanup instance, the reasons…

…: ["ERR max number of clients reached"]" into services-r10

redis/lib/redis_service/redis_node.rb

@@ -49,6 +49,25 @@ def listening?
     def running?
       VCAP.process_running? pid
     end
+
+    def kill(sig=:SIGTERM)
+      @wait_thread = Process.detach(pid)
+      Process.kill(sig, pid) if running?
+    end
+
+    def wait_killed(timeout=5, interval=0.2)
+      begin
+        Timeout::timeout(timeout) do
+          @wait_thread.join if @wait_thread
+          while running? do
+            sleep interval
+          end
+        end
+      rescue Timeout::Error
+        return false
+      end
+      true
+    end
   end
 
   def initialize(options)

redis/lib/redis_service/util.rb

@@ -185,10 +185,18 @@ def stop_redis_server(instance)
             begin
               redis.shutdown(@shutdown_command_name)
             rescue RuntimeError => e
-              # It could be a disabled instance
-              if @disable_password
-                redis = ::Redis.new({:port => instance.port, :password => @disable_password})
-                redis.shutdown(@shutdown_command_name)
+              if e.message == "ERR max number of clients reached"
+                # The max clients limitation could be reached, try to kill the process
+                  instance.kill
+                  instance.wait_killed ?
+                    @logger.debug("Redis server pid: #{instance.pid} terminated") :
+                    @logger.error("Timeout to terminate Redis server pid: #{instance.pid}")
+              else
+                # It could be a disabled instance
+                if @disable_password
+                  redis = ::Redis.new({:port => instance.port, :password => @disable_password})
+                  redis.shutdown(@shutdown_command_name)
+                end
               end
             end
           end

redis/spec/node_spec.rb

@@ -461,6 +461,18 @@ class Node
       Redis.new({:port => @credentials["port"], :password => @credentials["password"]}).info
       @node.unprovision(@credentials["name"])
     end
+
+    it "should unprovision successfully when reach the maximum number of clients" do
+      @credentials = @node.provision(:free)
+      sleep 1
+      redis = []
+      # Create max_clients connections
+      for i in 1..@options[:max_clients]
+        redis[i] = Redis.new({:port => @credentials["port"], :password => @credentials["password"]})
+        redis[i].info
+      end
+      @node.unprovision(@credentials["name"]).should == {}
+    end
   end
 
   describe "Node.timeout" do