Check whether the pid exists before killing.

agent/base

@@ -138,3 +138,9 @@ function verify_tool_group {
 	fi
 	return 0
 }
+
+if [[ "${_PBENCH_BENCH_TESTS}" == 1 ]] ;then
+    # for unit tests, we use a mock kill
+    # disable the built-in
+    enable -n kill
+fi

agent/bench-scripts/test-bin/kill

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+cmd="$0 $*"
+echo "$cmd" >> $_testlog
+
+# make pid 123463 unkillable, except on KILL
+pid=""
+case $1 in
+    -s)
+        signal=$2
+        pid=$3
+        ;;
+    *)
+        signal=NONE
+        pid=$1
+        shift
+        ;;
+esac
+
+echo $signal/$pid >> $_testlog
+if [ "$signal" == KILL ] ;then
+
+    echo "$cmd" exited with 0 >> $_testlog
+    exit 0
+fi
+
+if [ "$pid" == 123463 ] ;then
+    echo "$cmd" exited with 1 >> $_testlog
+    exit 1
+fi
+
+echo "$cmd" exited with 0 >> $_testlog
+exit 0

agent/bench-scripts/test-bin/pidof

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+echo "$0 $*" >> $_testlog
+
+case $1 in
+    -x)
+        shift
+        ;;
+esac
+case $1 in
+    turbostat)
+        # test-05
+        # multiple pids to trigger the warning
+        # 123463 is unkillable 
+        echo 123463 123464 123465 123466 123467
+        ;;
+    kvmstat)
+        # test-06
+        # single unkillable pid
+        echo 123463
+        ;;
+    vmstat|iostat)
+        # test-07 and test-09
+        # list of killable pids
+        # no effect for iostat: does not go through safe_kill currently.
+        echo 123456 123457 123458 123459
+        ;;
+    numastat|sar)
+        # test-09 and test-10
+        # killable pid
+        # no effect on sar: does not go through safe_kill currently.
+        echo 123456
+        ;;
+    *)
+        echo 123400
+        ;;
+esac
+
+exit 0

agent/tool-scripts/kvm-spinlock

@@ -1,4 +1,5 @@
 #!/bin/bash
+# -*- mode: shell-script; indent-tabs-mode: t; sh-basic-offset: 8; sh-indentation: 8; sh-indent-for-case-alt: + -*-
 
 script_path=`dirname $0`
 script_name=`basename $0`
@@ -235,6 +236,63 @@ case "$tool" in
 	tool_cmd="$script_path/datalog/$tool-datalog $interval $username $password $tool_stdout_file"
 	;;
 esac
+
+function safe_kill() {
+	# safer kill: check for strange situations and deal with them.
+	# If there is a pid, try to kill it.  If the kill succeeds,
+	# return success, but if it fails, try to see if the pid is
+	# still running: if so, try to kill it with a KILL and return
+	# failure.  Eat any error message.
+	local tool=$1
+	local pid=$2
+	local signal=${3:-TERM}
+	local pidlist p pid_to_kill len rc
+
+	if [ -z "$pid" ] ;then
+		# should not happen
+		return 1
+	fi
+
+	# check that the pid corresponds to the tool
+	pidlist=$(pidof -x $tool)
+	typeset -i len=0
+	for p in $pidlist ;do
+		len=$len+1
+		if [[ $p == $pid ]] ;then
+			pid_to_kill=$pid
+		fi
+	done
+	# hack: individual tools will have a different number of pids running.
+	# we might just delete the warning if it gets too cumbersome.
+	if [ $len -ge 4 -o $len -ge 3 -a $tool!="turbostat" ] ;then
+		warn_log "Too many pids for $tool: $pidlist -- maybe old tools running? Use pbench-kill-tools."
+	fi
+
+	rc=0
+	if [ ! -z "$pid_to_kill" ] ;then
+		kill -s $signal $pid_to_kill 2>/dev/null
+		rc=$?
+	fi
+
+	[[ $rc == 0 ]] && return 0
+
+	# check if the process is still running.
+	pidlist=$(pidof -x $tool)
+	for p in $pidlist ;do
+		if [[ $p == $pid ]] ;then
+			# why is the pid still there?
+			kill -s KILL $pid 2>/dev/null
+			return 2
+		# else
+			# why are there other tool processes running?
+			# should we kill them?
+		fi
+	done
+	return 0
+}
+
+# return code
+rc=0
 case "$mode" in
 	install)
 	case "$tool" in
@@ -294,29 +352,35 @@ case "$mode" in
 	debug_log "$script_name: running $tool_cmd"
 	"$tool_cmd_file" 2>"$tool_stderr_file" & echo $! >"$tool_pid_file"
 	wait
+	rc=0
 	popd >/dev/null
 	;;
 	stop)
 	debug_log "stopping $script_name"
 	pid=`cat $tool_pid_file`
-	if [ "$tool" == "kvmtrace" ]; then
-		if [ -z "$sleep_cmd" ]; then
-			# kill the perf cmd
-			kill -s SIGINT $pid
-		else
-			# kill the sleep process if it still exists
-			sleep_pid=`ps --ppid $pid | pgrep sleep`
-			if [ ! -z "$sleep_pid" ]; then
-				kill $sleep_pid
+	if [ ! -z "$pid" ] ;then
+		if [ "$tool" == "kvmtrace" ]; then
+			if [ -z "$sleep_cmd" ]; then
+				# kill the perf cmd
+				safe_kill $tool $pid INT
+			else
+				# kill the sleep process if it still exists
+				sleep_pid=`ps --ppid $pid | pgrep sleep`
+				if [ ! -z "$sleep_pid" ]; then
+					safe_kill $tool $sleep_pid
+				fi
 			fi
+			# wait for the trace-cmd pid to complete
+			while [ -d /proc/$pid ]; do
+				debug_log "waiting for PID $pid to die"
+				sleep 0.5
+			done
+			rc=0
+		else
+			safe_kill $tool $pid
+			rc=$?
+			/bin/rm -f "$tool_pid_file"
 		fi
-		# wait for the trace-cmd pid to complete
-		while [ -d /proc/$pid ]; do
-			debug_log "waiting for PID $pid to die"
-			sleep 0.5
-		done
-	else
-		kill $pid && /bin/rm -f "$tool_pid_file"
 	fi
 	;;
 	postprocess)
@@ -328,6 +392,7 @@ case "$mode" in
 		done
 		pushd $tool_output_dir >/dev/null
 		blkparse $devs >blkparse-stdout.txt 2>blkparse-stderr.txt
+		rc=$?
 		bzip2 blkparse-stdout.txt &
 		for dev in $devs; do
 			bzip2 $dev.blktrace.* &
@@ -344,6 +409,7 @@ case "$mode" in
 		fi
 		ssh $vm cat /proc/kallsyms >kallsyms-guest.txt
 		$script_path/postprocess/kvmtrace-postprocess .
+		rc=$?
 		bzip2 --fast kvmtrace-report-stdout.txt &
 		bzip2 --fast trace.dat &
 		bzip2 --fast kallsyms-guest.txt &
@@ -352,8 +418,11 @@ case "$mode" in
 		*)
 		debug_log "postprocessing $script_name"
 		$script_path/postprocess/$script_name-postprocess "$tool_output_dir"
+		rc=$?
 		;;
 	esac
 	popd >/dev/null
 	;;
 esac
+
+exit $rc

agent/util-scripts/gold/pbench-stop-tools/test-05.txt

@@ -0,0 +1,30 @@
++++ Running test-05 pbench-stop-tools
+[warn][1900-01-01T00:00:00.000000] Too many pids for turbostat: 123463 123464 123465 123466 123467 -- maybe old tools running? Use pbench-kill-tools.
+--- Finished test-05 pbench-stop-tools (status=0}
++++ pbench tree state
+/var/tmp/pbench-test-utils/pbench
+/var/tmp/pbench-test-utils/pbench/pbench.log
+/var/tmp/pbench-test-utils/pbench/tmp
+/var/tmp/pbench-test-utils/pbench/tmp/tools-default
+/var/tmp/pbench-test-utils/pbench/tools-default
+/var/tmp/pbench-test-utils/pbench/tools-default/turbostat
+/var/tmp/pbench-test-utils/pbench/tools-default/turbostat:
+--interval=3
+--- pbench tree state
++++ pbench.log file contents
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools]started: --dir=/var/tmp/pbench-test-utils/pbench/tmp
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools] /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/turbostat --stop --iteration=1 --group=default --dir=/var/tmp/pbench-test-utils/pbench/tmp --interval=3
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] stopping turbostat
+/var/tmp/pbench-test-utils/pbench/pbench.log:[warn][1900-01-01T00:00:00.000000] Too many pids for turbostat: 123463 123464 123465 123466 123467 -- maybe old tools running? Use pbench-kill-tools.
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools]completed: 
+--- pbench.log file contents
++++ test-execution.log file contents
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pidof -x turbostat
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill -s TERM 123463
+/var/tmp/pbench-test-utils/test-execution.log:TERM/123463
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill -s TERM 123463 exited with 1
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pidof -x turbostat
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill -s KILL 123463
+/var/tmp/pbench-test-utils/test-execution.log:KILL/123463
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill -s KILL 123463 exited with 0
+--- test-execution.log file contents

agent/util-scripts/gold/pbench-stop-tools/test-06.txt

@@ -0,0 +1,28 @@
++++ Running test-06 pbench-stop-tools
+--- Finished test-06 pbench-stop-tools (status=0}
++++ pbench tree state
+/var/tmp/pbench-test-utils/pbench
+/var/tmp/pbench-test-utils/pbench/pbench.log
+/var/tmp/pbench-test-utils/pbench/tmp
+/var/tmp/pbench-test-utils/pbench/tmp/tools-default
+/var/tmp/pbench-test-utils/pbench/tools-default
+/var/tmp/pbench-test-utils/pbench/tools-default/kvmstat
+/var/tmp/pbench-test-utils/pbench/tools-default/kvmstat:
+--interval="30"
+--- pbench tree state
++++ pbench.log file contents
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools]started: --dir=/var/tmp/pbench-test-utils/pbench/tmp
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools] /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/kvmstat --stop --iteration=1 --group=default --dir=/var/tmp/pbench-test-utils/pbench/tmp --interval="30"
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] stopping kvmstat
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools]completed: 
+--- pbench.log file contents
++++ test-execution.log file contents
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pidof -x kvmstat
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill -s TERM 123463
+/var/tmp/pbench-test-utils/test-execution.log:TERM/123463
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill -s TERM 123463 exited with 1
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pidof -x kvmstat
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill -s KILL 123463
+/var/tmp/pbench-test-utils/test-execution.log:KILL/123463
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill -s KILL 123463 exited with 0
+--- test-execution.log file contents

agent/util-scripts/gold/pbench-stop-tools/test-07.txt

@@ -0,0 +1,26 @@
++++ Running test-07 pbench-stop-tools
+[warn][1900-01-01T00:00:00.000000] Too many pids for vmstat: 123456 123457 123458 123459 -- maybe old tools running? Use pbench-kill-tools.
+--- Finished test-07 pbench-stop-tools (status=0}
++++ pbench tree state
+/var/tmp/pbench-test-utils/pbench
+/var/tmp/pbench-test-utils/pbench/pbench.log
+/var/tmp/pbench-test-utils/pbench/tmp
+/var/tmp/pbench-test-utils/pbench/tmp/tools-default
+/var/tmp/pbench-test-utils/pbench/tools-default
+/var/tmp/pbench-test-utils/pbench/tools-default/vmstat
+/var/tmp/pbench-test-utils/pbench/tools-default/vmstat:
+--interval=50
+--- pbench tree state
++++ pbench.log file contents
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools]started: --dir=/var/tmp/pbench-test-utils/pbench/tmp
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools] /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/vmstat --stop --iteration=1 --group=default --dir=/var/tmp/pbench-test-utils/pbench/tmp --interval=50
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] stopping vmstat
+/var/tmp/pbench-test-utils/pbench/pbench.log:[warn][1900-01-01T00:00:00.000000] Too many pids for vmstat: 123456 123457 123458 123459 -- maybe old tools running? Use pbench-kill-tools.
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools]completed: 
+--- pbench.log file contents
++++ test-execution.log file contents
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pidof -x vmstat
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill -s TERM 123456
+/var/tmp/pbench-test-utils/test-execution.log:TERM/123456
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill -s TERM 123456 exited with 0
+--- test-execution.log file contents

agent/util-scripts/gold/pbench-stop-tools/test-08.txt

@@ -0,0 +1,24 @@
++++ Running test-08 pbench-stop-tools
+--- Finished test-08 pbench-stop-tools (status=0}
++++ pbench tree state
+/var/tmp/pbench-test-utils/pbench
+/var/tmp/pbench-test-utils/pbench/pbench.log
+/var/tmp/pbench-test-utils/pbench/tmp
+/var/tmp/pbench-test-utils/pbench/tmp/tools-default
+/var/tmp/pbench-test-utils/pbench/tools-default
+/var/tmp/pbench-test-utils/pbench/tools-default/numastat
+/var/tmp/pbench-test-utils/pbench/tools-default/numastat:
+--interval=50
+--- pbench tree state
++++ pbench.log file contents
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools]started: --dir=/var/tmp/pbench-test-utils/pbench/tmp
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools] /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/numastat --stop --iteration=1 --group=default --dir=/var/tmp/pbench-test-utils/pbench/tmp --interval=50
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] stopping numastat
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools]completed: 
+--- pbench.log file contents
++++ test-execution.log file contents
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pidof -x numastat
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill -s TERM 123456
+/var/tmp/pbench-test-utils/test-execution.log:TERM/123456
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill -s TERM 123456 exited with 0
+--- test-execution.log file contents

agent/util-scripts/gold/pbench-stop-tools/test-09.txt

@@ -0,0 +1,23 @@
++++ Running test-09 pbench-stop-tools
+--- Finished test-09 pbench-stop-tools (status=0}
++++ pbench tree state
+/var/tmp/pbench-test-utils/pbench
+/var/tmp/pbench-test-utils/pbench/pbench.log
+/var/tmp/pbench-test-utils/pbench/tmp
+/var/tmp/pbench-test-utils/pbench/tmp/tools-default
+/var/tmp/pbench-test-utils/pbench/tools-default
+/var/tmp/pbench-test-utils/pbench/tools-default/iostat
+/var/tmp/pbench-test-utils/pbench/tools-default/iostat:
+--interval=3
+--- pbench tree state
++++ pbench.log file contents
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools]started: --dir=/var/tmp/pbench-test-utils/pbench/tmp
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools] /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/iostat --stop --iteration=1 --group=default --dir=/var/tmp/pbench-test-utils/pbench/tmp --interval=3
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] stopping iostat
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools]completed: 
+--- pbench.log file contents
++++ test-execution.log file contents
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill 123456
+/var/tmp/pbench-test-utils/test-execution.log:NONE/123456
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill 123456 exited with 0
+--- test-execution.log file contents

agent/util-scripts/gold/pbench-stop-tools/test-10.txt

@@ -0,0 +1,24 @@
++++ Running test-10 pbench-stop-tools
+--- Finished test-10 pbench-stop-tools (status=0}
++++ pbench tree state
+/var/tmp/pbench-test-utils/pbench
+/var/tmp/pbench-test-utils/pbench/pbench.log
+/var/tmp/pbench-test-utils/pbench/tmp
+/var/tmp/pbench-test-utils/pbench/tmp/tools-default
+/var/tmp/pbench-test-utils/pbench/tools-default
+/var/tmp/pbench-test-utils/pbench/tools-default/sar
+/var/tmp/pbench-test-utils/pbench/tools-default/sar:
+--interval=40
+
+--- pbench tree state
++++ pbench.log file contents
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools]started: --dir=/var/tmp/pbench-test-utils/pbench/tmp
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools] /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/sar --stop --iteration=1 --group=default --dir=/var/tmp/pbench-test-utils/pbench/tmp --interval=40 
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] stopping sar
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools]completed: 
+--- pbench.log file contents
++++ test-execution.log file contents
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill 123456
+/var/tmp/pbench-test-utils/test-execution.log:NONE/123456
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill 123456 exited with 0
+--- test-execution.log file contents