Add ES_STOP_TIMEOUT configuration variable to packages

docs/reference/setup/as-a-service.asciidoc

@@ -26,6 +26,7 @@ Each package features a configuration file, which allows you to set the followin
 `CONF_FILE`::             Path to configuration file, defaults to `/etc/elasticsearch/elasticsearch.yml`
 `ES_JAVA_OPTS`::          Any additional java options you may want to apply. This may be useful, if you need to set the `node.name` property, but do not want to change the `elasticsearch.yml` configuration file, because it is distributed via a provisioning system like puppet or chef. Example: `ES_JAVA_OPTS="-Des.node.name=search-01"`
 `RESTART_ON_UPGRADE`::    Configure restart on package upgrade, defaults to `false`. This means you will have to restart your elasticsearch instance after installing a package manually. The reason for this is to ensure, that upgrades in a cluster do not result in a continuous shard reallocation resulting in high network traffic and reducing the response times of your cluster.
+`ES_STOP_TIMEOUT`::       Configure the seconds to wait after a TERM signal was sent to check if the elasticsearch process is stopped and decide whether stopping was successful or a failure.
 
 [float]
 ==== Debian/Ubuntu

src/deb/default/elasticsearch

@@ -42,3 +42,7 @@
 
 # Configure restart on package upgrade (true, every other setting will lead to not restarting)
 #RESTART_ON_UPGRADE=true
+
+# Shutdown delay in seconds, the process is tried to be killed with TERM
+#ES_STOP_TIMEOUT=20
+

src/deb/init.d/elasticsearch

@@ -93,6 +93,9 @@ CONF_FILE=$CONF_DIR/elasticsearch.yml
 # Maximum number of VMA (Virtual Memory Areas) a process can own
 MAX_MAP_COUNT=262144
 
+# Timeout in seconds to wait after sending TERM signal
+ES_STOP_TIMEOUT=20
+
 # End of variables that can be overwritten in $DEFAULT
 
 # overwrite settings from default file
@@ -170,10 +173,11 @@ case "$1" in
 	if [ -f "$PID_FILE" ]; then 
 		start-stop-daemon --stop --pidfile "$PID_FILE" \
 			--user "$ES_USER" \
-			--retry=TERM/20/KILL/5 >/dev/null
-		if [ $? -eq 1 ]; then
+			--retry=$ES_STOP_TIMEOUT >/dev/null
+		RC=$?
+		if [ $RC -eq 1 ]; then
 			log_progress_msg "$DESC is not running but pid file exists, cleaning up"
-		elif [ $? -eq 3 ]; then
+		elif [ $RC -eq 3 -o $RC -eq 2 ]; then
 			PID="`cat $PID_FILE`"
 			log_failure_msg "Failed to stop $DESC (pid $PID)"
 			exit 1

src/rpm/init.d/elasticsearch

@@ -94,10 +94,10 @@ start() {
 
 stop() {
     echo -n $"Stopping $prog: "
-    # stop it here, often "killproc $prog"
-    killproc -p $pidfile -d 20 $prog
-    retval=$?
-    echo
+    pid=`pidofproc -p "$pidfile" $prog`
+    # dont send kill -9 by default, but allow to configure it
+    # if kill timeout is set, send -15, wait STOP_TIMEOUT, send -9, wait KILL_TIMEOUT (otherwise omit the last two steps)
+    killproc -p $pidfile -d $ES_STOP_TIMEOUT $prog
     [ $retval -eq 0 ] && rm -f $lockfile
     return $retval
 }

src/rpm/sysconfig/elasticsearch

@@ -44,3 +44,7 @@ ES_USER=elasticsearch
 
 # Configure restart on package upgrade (true, every other setting will lead to not restarting)
 #RESTART_ON_UPGRADE=true
+
+# Shutdown delay in seconds, the process is tried to be killed with TERM
+ES_STOP_TIMEOUT=20
+

src/rpm/systemd/elasticsearch.service

@@ -14,7 +14,9 @@ LimitNOFILE=65535
 # See MAX_LOCKED_MEMORY in sysconfig, use "infinity" when MAX_LOCKED_MEMORY=unlimited and using bootstrap.mlockall: true
 #LimitMEMLOCK=infinity
 # Shutdown delay in seconds, before process is tried to be killed with KILL (if configured)
-TimeoutStopSec=20
+TimeoutStopSec=$ES_STOP_TIMEOUT
+# Should a KILL signal be sent, if a TERM signal fails?
+SendSIGKILL=false
 
 [Install]
 WantedBy=multi-user.target