Merge pull request #761 from divino/Issue-703

Fix #703

deploy/default.properties

@@ -260,3 +260,9 @@ http.retry-count=3
 http.open-timeout=5
 http.read-timeout=300
 http.retry-delay=15
+
+#
+# Verified restart config
+#
+verify_retry_max=5
+verify_retry_interval=10

deploy/lib/server_config.rb

@@ -520,7 +520,7 @@ def execute_query(query, properties = {})
     return r
   end
 
-  def restart
+  def restart_basic
     @ml_username = @properties['ml.bootstrap-user'] || @properties['ml.user']
     if @ml_username == @properties['ml.bootstrap-user']
       @ml_password = @properties['ml.bootstrap-password']
@@ -556,6 +556,43 @@ def restart
     return true
   end
 
+  # implemented verified restart
+  def restart
+    verify = find_arg(['--verify'])
+    if verify==='false'
+      restart_basic
+    else
+      # defaults to verified restart
+      old_timestamp = go(%Q{http://#{@properties["ml.server"]}:8001/admin/v1/timestamp}, "get").body
+      restart_basic
+      retry_count = 0
+      retry_max = @properties["ml.verify_retry_max"].to_i
+      retry_interval = @properties["ml.verify_retry_interval"].to_i
+      new_timestamp = old_timestamp
+      while retry_count < retry_max do
+        begin
+          new_timestamp = go(%Q{http://#{@properties["ml.server"]}:8001/admin/v1/timestamp}, "get").body
+        rescue
+          logger.info 'Verifying restart ...'
+          logger.debug 'Retry attempt ' + retry_count.to_s + ' failed'
+        end
+        if new_timestamp != old_timestamp
+          # indicates that restart is confirmed successful
+          break
+        end
+        logger.debug "Verifying restart..."
+        sleep retry_interval
+        retry_count += 1
+      end
+      if new_timestamp == old_timestamp
+        logger.warn "Could not verify restart"
+      else
+        logger.info 'Verified restart.'
+        logger.debug "Verified restart new #{new_timestamp} old #{old_timestamp}"
+      end
+    end
+  end
+
   def merge
     what = ARGV.shift
     raise HelpException.new("merge", "Missing WHAT") unless what

deploy/lib/xquery/setup.xqy

@@ -1175,9 +1175,9 @@ declare function setup:do-restart($group-name as xs:string?) as item()*
         "Restarting hosts to make configuration changes take effect"),
 
       if ($group-id) then
-        fn:concat("Group ", $group-name, " restarted")
+        fn:concat("Invoked group ", $group-name, " restart")
       else
-        fn:concat("Cluster restarted")
+        fn:concat("Invoked cluster restart")
     )
   }
   catch ($ex)