Add ES_STOP_TIMEOUT/ES_KILL_TIMEOUT variables to packages

This variable allows to configure the waiting time after a TERM signal has
been sent. After that waiting time a KILL signal is sent to ensure the
service is stopped.

In case of a bigger installation the default values might be to slow, so it
now is configurable.

Also, we opted out from sending a KILL signal by default. This has to be
explicitely done by using the ES_KILL_TIMEOUT variable.

Original work done by @tmclaugh at the PR elastic#3719

Closes elastic#3719
Closes elastic#3972

docs/reference/setup/as-a-service.asciidoc

@@ -26,6 +26,8 @@ Each package features a configuration file, which allows you to set the followin
 `CONF_FILE`::             Path to configuration file, defaults to `/etc/elasticsearch/elasticsearch.yml`
 `ES_JAVA_OPTS`::          Any additional java options you may want to apply. This may be useful, if you need to set the `node.name` property, but do not want to change the `elasticsearch.yml` configuration file, because it is distributed via a provisioning system like puppet or chef. Example: `ES_JAVA_OPTS="-Des.node.name=search-01"`
 `RESTART_ON_UPGRADE`::    Configure restart on package upgrade, defaults to `false`. This means you will have to restart your elasticsearch instance after installing a package manually. The reason for this is to ensure, that upgrades in a cluster do not result in a continuous shard reallocation resulting in high network traffic and reducing the response times of your cluster.
+`ES_STOP_TIMEOUT`::       Configure the seconds to wait after a TERM signal was sent to check if the elasticsearch process is stopped and decide whether stopping was successful or a failure
+`ES_KILL_TIMEOUT`::       Configure the seconds to wait after a KILL signal was sent to make sure the elasticsearch process is stopped. If this setting is configured, a process first stopped by a TERM signal, then waits for the `ES_STOP_TIMEOUT`, and then sends a KILL signal and waits for the `ES_KILL_TIMEOUT` interval.
 
 [float]
 ==== Debian/Ubuntu

src/deb/default/elasticsearch

@@ -42,3 +42,10 @@
 
 # Configure restart on package upgrade (true, every other setting will lead to not restarting)
 #RESTART_ON_UPGRADE=true
+
+# Shutdown delay in seconds, the process is tried to be killed with TERM
+#ES_STOP_TIMEOUT=3
+
+# Shutdown delay in seconds, until the process is tried to be killed with KILL signal, after trying TERM
+# Disabled by default, so you have to kill -9 manually
+#ES_KILL_TIMEOUT=5

src/deb/init.d/elasticsearch

@@ -93,6 +93,12 @@ CONF_FILE=$CONF_DIR/elasticsearch.yml
 # Maximum number of VMA (Virtual Memory Areas) a process can own
 MAX_MAP_COUNT=262144
 
+# Timeout in seconds to wait after sending KILL signal
+#ES_KILL_TIMEOUT=
+
+# Timeout in seconds to wait after sending TERM signal
+ES_STOP_TIMEOUT=3
+
 # End of variables that can be overwritten in $DEFAULT
 
 # overwrite settings from default file
@@ -168,12 +174,17 @@ case "$1" in
 	log_daemon_msg "Stopping $DESC"
 
 	if [ -f "$PID_FILE" ]; then 
+        timeout="TERM/$ES_STOP_TIMEOUT"
+        if [ -n "$ES_KILL_TIMEOUT" ] ; then
+            timeout="$timeout/KILL/$ES_KILL_TIMEOUT"
+        fi
 		start-stop-daemon --stop --pidfile "$PID_FILE" \
 			--user "$ES_USER" \
-			--retry=TERM/20/KILL/5 >/dev/null
-		if [ $? -eq 1 ]; then
+			--retry=$timeout >/dev/null
+		RC=$?
+		if [ $RC -eq 1 ]; then
 			log_progress_msg "$DESC is not running but pid file exists, cleaning up"
-		elif [ $? -eq 3 ]; then
+		elif [ $RC -eq 3 -o $RC -eq 2 ]; then
 			PID="`cat $PID_FILE`"
 			log_failure_msg "Failed to stop $DESC (pid $PID)"
 			exit 1

src/rpm/init.d/elasticsearch

@@ -94,10 +94,24 @@ start() {
 
 stop() {
     echo -n $"Stopping $prog: "
-    # stop it here, often "killproc $prog"
-    killproc -p $pidfile -d 20 $prog
-    retval=$?
-    echo
+    pid=`pidofproc -p "$pidfile" $prog`
+    # dont send kill -9 by default, but allow to configure it
+    # if kill timeout is set, send -15, wait STOP_TIMEOUT, send -9, wait KILL_TIMEOUT (otherwise omit the last two steps)
+    TIMEOUT=$ES_STOP_TIMEOUT
+    if [ -n "$ES_KILL_TIMEOUT" ] ; then
+        TIMEOUT=$ES_KILL_TIMEOUT
+        killproc -p $pidfile -d $ES_STOP_TIMEOUT $prog
+        retval=$?
+    else
+        kill -15 $pid
+        retval=$?
+        if checkpid $pid; then
+            sleep $TIMEOUT
+            checkpid $pid
+            retval=$?
+        fi
+        [ "$retval" -eq 0 ] && failure $"$base shutdown" || success $"$base shutdown"
+    fi
     [ $retval -eq 0 ] && rm -f $lockfile
     return $retval
 }

src/rpm/sysconfig/elasticsearch

@@ -44,3 +44,10 @@ ES_USER=elasticsearch
 
 # Configure restart on package upgrade (true, every other setting will lead to not restarting)
 #RESTART_ON_UPGRADE=true
+
+# Shutdown delay in seconds, the process is tried to be killed with TERM
+ES_STOP_TIMEOUT=3
+
+# Shutdown delay in seconds, until the process is tried to be killed with KILL signal, after trying TERM
+# Disabled by default, so you have to kill -9 manually
+#ES_KILL_TIMEOUT=5

src/rpm/systemd/elasticsearch.service

@@ -14,7 +14,9 @@ LimitNOFILE=65535
 # See MAX_LOCKED_MEMORY in sysconfig, use "infinity" when MAX_LOCKED_MEMORY=unlimited and using bootstrap.mlockall: true
 #LimitMEMLOCK=infinity
 # Shutdown delay in seconds, before process is tried to be killed with KILL (if configured)
-TimeoutStopSec=20
+TimeoutStopSec=$ES_STOP_TIMEOUT
+# Should a KILL signal be sent, if a TERM signal fails?
+SendSIGKILL=false
 
 [Install]
 WantedBy=multi-user.target