diff --git a/agent/base b/agent/base
index 7f51290fae..ad491adb7b 100755
--- a/agent/base
+++ b/agent/base
@@ -138,3 +138,9 @@ function verify_tool_group {
 	fi
 	return 0
 }
+
+if [[ "${_PBENCH_BENCH_TESTS}" == 1 ]] ;then
+    # for unit tests, we use a mock kill
+    # disable the built-in
+    enable -n kill
+fi
diff --git a/agent/bench-scripts/test-bin/kill b/agent/bench-scripts/test-bin/kill
new file mode 100755
index 0000000000..7c112c1c63
--- /dev/null
+++ b/agent/bench-scripts/test-bin/kill
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+cmd="$0 $*"
+echo "$cmd" >> $_testlog
+
+# make pid 123463 unkillable, except on KILL
+pid=""
+case $1 in
+    -s)
+        signal=$2
+        pid=$3
+        ;;
+    *)
+        signal=NONE
+        pid=$1
+        shift
+        ;;
+esac
+
+echo $signal/$pid >> $_testlog
+if [ "$signal" == KILL ] ;then
+
+    echo "$cmd" exited with 0 >> $_testlog
+    exit 0
+fi
+
+if [ "$pid" == 123463 ] ;then
+    echo "$cmd" exited with 1 >> $_testlog
+    exit 1
+fi
+
+echo "$cmd" exited with 0 >> $_testlog
+exit 0
diff --git a/agent/bench-scripts/test-bin/pidof b/agent/bench-scripts/test-bin/pidof
new file mode 100755
index 0000000000..14c0dad947
--- /dev/null
+++ b/agent/bench-scripts/test-bin/pidof
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+echo "$0 $*" >> $_testlog
+
+case $1 in
+    -x)
+        shift
+        ;;
+esac
+case $1 in
+    turbostat)
+        # test-05
+        # multiple pids to trigger the warning
+        # 123463 is unkillable 
+        echo 123463 123464 123465 123466 123467
+        ;;
+    kvmstat)
+        # test-06
+        # single unkillable pid
+        echo 123463
+        ;;
+    vmstat|iostat)
+        # test-07 and test-09
+        # list of killable pids
+        # no effect for iostat: does not go through safe_kill currently.
+        echo 123456 123457 123458 123459
+        ;;
+    numastat|sar)
+        # test-09 and test-10
+        # killable pid
+        # no effect on sar: does not go through safe_kill currently.
+        echo 123456
+        ;;
+    *)
+        echo 123400
+        ;;
+esac
+
+exit 0
diff --git a/agent/tool-scripts/kvm-spinlock b/agent/tool-scripts/kvm-spinlock
index f70317d368..20e35c4f24 100755
--- a/agent/tool-scripts/kvm-spinlock
+++ b/agent/tool-scripts/kvm-spinlock
@@ -1,4 +1,5 @@
 #!/bin/bash
+# -*- mode: shell-script; indent-tabs-mode: t; sh-basic-offset: 8; sh-indentation: 8; sh-indent-for-case-alt: + -*-
 
 script_path=`dirname $0`
 script_name=`basename $0`
@@ -235,6 +236,63 @@ case "$tool" in
 	tool_cmd="$script_path/datalog/$tool-datalog $interval $username $password $tool_stdout_file"
 	;;
 esac
+
+function safe_kill() {
+	# safer kill: check for strange situations and deal with them.
+	# If there is a pid, try to kill it.  If the kill succeeds,
+	# return success, but if it fails, try to see if the pid is
+	# still running: if so, try to kill it with a KILL and return
+	# failure.  Eat any error message.
+	local tool=$1
+	local pid=$2
+	local signal=${3:-TERM}
+	local pidlist p pid_to_kill len rc
+
+	if [ -z "$pid" ] ;then
+		# should not happen
+		return 1
+	fi
+	
+	# check that the pid corresponds to the tool
+	pidlist=$(pidof -x $tool)
+	typeset -i len=0
+	for p in $pidlist ;do
+		len=$len+1
+		if [[ $p == $pid ]] ;then
+			pid_to_kill=$pid
+		fi
+	done
+	# hack: individual tools will have a different number of pids running.
+	# we might just delete the warning if it gets too cumbersome.
+	if [ $len -ge 4 -o $len -ge 3 -a $tool!="turbostat" ] ;then
+		warn_log "Too many pids for $tool: $pidlist -- maybe old tools running? Use pbench-kill-tools."
+	fi
+	
+	rc=0
+	if [ ! -z "$pid_to_kill" ] ;then
+		kill -s $signal $pid_to_kill 2>/dev/null
+		rc=$?
+	fi
+	
+	[[ $rc == 0 ]] && return 0
+
+	# check if the process is still running.
+	pidlist=$(pidof -x $tool)
+	for p in $pidlist ;do
+		if [[ $p == $pid ]] ;then
+			# why is the pid still there?
+			kill -s KILL $pid 2>/dev/null
+			return 2
+		# else
+			# why are there other tool processes running?
+			# should we kill them?
+		fi
+	done
+	return 0
+}
+
+# return code
+rc=0
 case "$mode" in
 	install)
 	case "$tool" in
@@ -294,29 +352,35 @@ case "$mode" in
 	debug_log "$script_name: running $tool_cmd"
 	"$tool_cmd_file" 2>"$tool_stderr_file" & echo $! >"$tool_pid_file"
 	wait
+	rc=0
 	popd >/dev/null
 	;;
 	stop)
 	debug_log "stopping $script_name"
 	pid=`cat $tool_pid_file`
-	if [ "$tool" == "kvmtrace" ]; then
-		if [ -z "$sleep_cmd" ]; then
-			# kill the perf cmd
-			kill -s SIGINT $pid
-		else
-			# kill the sleep process if it still exists
-			sleep_pid=`ps --ppid $pid | pgrep sleep`
-			if [ ! -z "$sleep_pid" ]; then
-				kill $sleep_pid
+	if [ ! -z "$pid" ] ;then
+		if [ "$tool" == "kvmtrace" ]; then
+			if [ -z "$sleep_cmd" ]; then
+				# kill the perf cmd
+				safe_kill $tool $pid INT
+			else
+				# kill the sleep process if it still exists
+				sleep_pid=`ps --ppid $pid | pgrep sleep`
+				if [ ! -z "$sleep_pid" ]; then
+					safe_kill $tool $sleep_pid
+				fi
 			fi
+			# wait for the trace-cmd pid to complete
+			while [ -d /proc/$pid ]; do
+				debug_log "waiting for PID $pid to die"
+				sleep 0.5
+			done
+			rc=0
+		else
+			safe_kill $tool $pid
+			rc=$?
+			/bin/rm -f "$tool_pid_file"
 		fi
-		# wait for the trace-cmd pid to complete
-		while [ -d /proc/$pid ]; do
-			debug_log "waiting for PID $pid to die"
-			sleep 0.5
-		done
-	else
-		kill $pid && /bin/rm -f "$tool_pid_file"
 	fi
 	;;
 	postprocess)
@@ -328,6 +392,7 @@ case "$mode" in
 		done
 		pushd $tool_output_dir >/dev/null
 		blkparse $devs >blkparse-stdout.txt 2>blkparse-stderr.txt
+		rc=$?
 		bzip2 blkparse-stdout.txt &
 		for dev in $devs; do
 			bzip2 $dev.blktrace.* &
@@ -344,6 +409,7 @@ case "$mode" in
 		fi
 		ssh $vm cat /proc/kallsyms >kallsyms-guest.txt
 		$script_path/postprocess/kvmtrace-postprocess .
+		rc=$?
 		bzip2 --fast kvmtrace-report-stdout.txt &
 		bzip2 --fast trace.dat &
 		bzip2 --fast kallsyms-guest.txt &
@@ -352,8 +418,11 @@ case "$mode" in
 		*)
 		debug_log "postprocessing $script_name"
 		$script_path/postprocess/$script_name-postprocess "$tool_output_dir"
+		rc=$?
 		;;
 	esac
 	popd >/dev/null
 	;;
 esac
+
+exit $rc
diff --git a/agent/util-scripts/gold/pbench-stop-tools/test-05.txt b/agent/util-scripts/gold/pbench-stop-tools/test-05.txt
new file mode 100644
index 0000000000..655afbdbc3
--- /dev/null
+++ b/agent/util-scripts/gold/pbench-stop-tools/test-05.txt
@@ -0,0 +1,30 @@
++++ Running test-05 pbench-stop-tools
+[warn][1900-01-01T00:00:00.000000] Too many pids for turbostat: 123463 123464 123465 123466 123467 -- maybe old tools running? Use pbench-kill-tools.
+--- Finished test-05 pbench-stop-tools (status=0}
++++ pbench tree state
+/var/tmp/pbench-test-utils/pbench
+/var/tmp/pbench-test-utils/pbench/pbench.log
+/var/tmp/pbench-test-utils/pbench/tmp
+/var/tmp/pbench-test-utils/pbench/tmp/tools-default
+/var/tmp/pbench-test-utils/pbench/tools-default
+/var/tmp/pbench-test-utils/pbench/tools-default/turbostat
+/var/tmp/pbench-test-utils/pbench/tools-default/turbostat:
+--interval=3
+--- pbench tree state
++++ pbench.log file contents
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools]started: --dir=/var/tmp/pbench-test-utils/pbench/tmp
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools] /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/turbostat --stop --iteration=1 --group=default --dir=/var/tmp/pbench-test-utils/pbench/tmp --interval=3
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] stopping turbostat
+/var/tmp/pbench-test-utils/pbench/pbench.log:[warn][1900-01-01T00:00:00.000000] Too many pids for turbostat: 123463 123464 123465 123466 123467 -- maybe old tools running? Use pbench-kill-tools.
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools]completed: 
+--- pbench.log file contents
++++ test-execution.log file contents
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pidof -x turbostat
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill -s TERM 123463
+/var/tmp/pbench-test-utils/test-execution.log:TERM/123463
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill -s TERM 123463 exited with 1
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pidof -x turbostat
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill -s KILL 123463
+/var/tmp/pbench-test-utils/test-execution.log:KILL/123463
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill -s KILL 123463 exited with 0
+--- test-execution.log file contents
diff --git a/agent/util-scripts/gold/pbench-stop-tools/test-06.txt b/agent/util-scripts/gold/pbench-stop-tools/test-06.txt
new file mode 100644
index 0000000000..be444125e8
--- /dev/null
+++ b/agent/util-scripts/gold/pbench-stop-tools/test-06.txt
@@ -0,0 +1,28 @@
++++ Running test-06 pbench-stop-tools
+--- Finished test-06 pbench-stop-tools (status=0}
++++ pbench tree state
+/var/tmp/pbench-test-utils/pbench
+/var/tmp/pbench-test-utils/pbench/pbench.log
+/var/tmp/pbench-test-utils/pbench/tmp
+/var/tmp/pbench-test-utils/pbench/tmp/tools-default
+/var/tmp/pbench-test-utils/pbench/tools-default
+/var/tmp/pbench-test-utils/pbench/tools-default/kvmstat
+/var/tmp/pbench-test-utils/pbench/tools-default/kvmstat:
+--interval="30"
+--- pbench tree state
++++ pbench.log file contents
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools]started: --dir=/var/tmp/pbench-test-utils/pbench/tmp
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools] /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/kvmstat --stop --iteration=1 --group=default --dir=/var/tmp/pbench-test-utils/pbench/tmp --interval="30"
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] stopping kvmstat
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools]completed: 
+--- pbench.log file contents
++++ test-execution.log file contents
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pidof -x kvmstat
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill -s TERM 123463
+/var/tmp/pbench-test-utils/test-execution.log:TERM/123463
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill -s TERM 123463 exited with 1
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pidof -x kvmstat
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill -s KILL 123463
+/var/tmp/pbench-test-utils/test-execution.log:KILL/123463
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill -s KILL 123463 exited with 0
+--- test-execution.log file contents
diff --git a/agent/util-scripts/gold/pbench-stop-tools/test-07.txt b/agent/util-scripts/gold/pbench-stop-tools/test-07.txt
new file mode 100644
index 0000000000..fcb243d433
--- /dev/null
+++ b/agent/util-scripts/gold/pbench-stop-tools/test-07.txt
@@ -0,0 +1,26 @@
++++ Running test-07 pbench-stop-tools
+[warn][1900-01-01T00:00:00.000000] Too many pids for vmstat: 123456 123457 123458 123459 -- maybe old tools running? Use pbench-kill-tools.
+--- Finished test-07 pbench-stop-tools (status=0}
++++ pbench tree state
+/var/tmp/pbench-test-utils/pbench
+/var/tmp/pbench-test-utils/pbench/pbench.log
+/var/tmp/pbench-test-utils/pbench/tmp
+/var/tmp/pbench-test-utils/pbench/tmp/tools-default
+/var/tmp/pbench-test-utils/pbench/tools-default
+/var/tmp/pbench-test-utils/pbench/tools-default/vmstat
+/var/tmp/pbench-test-utils/pbench/tools-default/vmstat:
+--interval=50
+--- pbench tree state
++++ pbench.log file contents
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools]started: --dir=/var/tmp/pbench-test-utils/pbench/tmp
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools] /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/vmstat --stop --iteration=1 --group=default --dir=/var/tmp/pbench-test-utils/pbench/tmp --interval=50
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] stopping vmstat
+/var/tmp/pbench-test-utils/pbench/pbench.log:[warn][1900-01-01T00:00:00.000000] Too many pids for vmstat: 123456 123457 123458 123459 -- maybe old tools running? Use pbench-kill-tools.
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools]completed: 
+--- pbench.log file contents
++++ test-execution.log file contents
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pidof -x vmstat
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill -s TERM 123456
+/var/tmp/pbench-test-utils/test-execution.log:TERM/123456
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill -s TERM 123456 exited with 0
+--- test-execution.log file contents
diff --git a/agent/util-scripts/gold/pbench-stop-tools/test-08.txt b/agent/util-scripts/gold/pbench-stop-tools/test-08.txt
new file mode 100644
index 0000000000..ed878839b9
--- /dev/null
+++ b/agent/util-scripts/gold/pbench-stop-tools/test-08.txt
@@ -0,0 +1,24 @@
++++ Running test-08 pbench-stop-tools
+--- Finished test-08 pbench-stop-tools (status=0}
++++ pbench tree state
+/var/tmp/pbench-test-utils/pbench
+/var/tmp/pbench-test-utils/pbench/pbench.log
+/var/tmp/pbench-test-utils/pbench/tmp
+/var/tmp/pbench-test-utils/pbench/tmp/tools-default
+/var/tmp/pbench-test-utils/pbench/tools-default
+/var/tmp/pbench-test-utils/pbench/tools-default/numastat
+/var/tmp/pbench-test-utils/pbench/tools-default/numastat:
+--interval=50
+--- pbench tree state
++++ pbench.log file contents
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools]started: --dir=/var/tmp/pbench-test-utils/pbench/tmp
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools] /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/numastat --stop --iteration=1 --group=default --dir=/var/tmp/pbench-test-utils/pbench/tmp --interval=50
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] stopping numastat
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools]completed: 
+--- pbench.log file contents
++++ test-execution.log file contents
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pidof -x numastat
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill -s TERM 123456
+/var/tmp/pbench-test-utils/test-execution.log:TERM/123456
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill -s TERM 123456 exited with 0
+--- test-execution.log file contents
diff --git a/agent/util-scripts/gold/pbench-stop-tools/test-09.txt b/agent/util-scripts/gold/pbench-stop-tools/test-09.txt
new file mode 100644
index 0000000000..8b8f617538
--- /dev/null
+++ b/agent/util-scripts/gold/pbench-stop-tools/test-09.txt
@@ -0,0 +1,23 @@
++++ Running test-09 pbench-stop-tools
+--- Finished test-09 pbench-stop-tools (status=0}
++++ pbench tree state
+/var/tmp/pbench-test-utils/pbench
+/var/tmp/pbench-test-utils/pbench/pbench.log
+/var/tmp/pbench-test-utils/pbench/tmp
+/var/tmp/pbench-test-utils/pbench/tmp/tools-default
+/var/tmp/pbench-test-utils/pbench/tools-default
+/var/tmp/pbench-test-utils/pbench/tools-default/iostat
+/var/tmp/pbench-test-utils/pbench/tools-default/iostat:
+--interval=3
+--- pbench tree state
++++ pbench.log file contents
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools]started: --dir=/var/tmp/pbench-test-utils/pbench/tmp
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools] /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/iostat --stop --iteration=1 --group=default --dir=/var/tmp/pbench-test-utils/pbench/tmp --interval=3
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] stopping iostat
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools]completed: 
+--- pbench.log file contents
++++ test-execution.log file contents
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill 123456
+/var/tmp/pbench-test-utils/test-execution.log:NONE/123456
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill 123456 exited with 0
+--- test-execution.log file contents
diff --git a/agent/util-scripts/gold/pbench-stop-tools/test-10.txt b/agent/util-scripts/gold/pbench-stop-tools/test-10.txt
new file mode 100644
index 0000000000..07ffcccc78
--- /dev/null
+++ b/agent/util-scripts/gold/pbench-stop-tools/test-10.txt
@@ -0,0 +1,24 @@
++++ Running test-10 pbench-stop-tools
+--- Finished test-10 pbench-stop-tools (status=0}
++++ pbench tree state
+/var/tmp/pbench-test-utils/pbench
+/var/tmp/pbench-test-utils/pbench/pbench.log
+/var/tmp/pbench-test-utils/pbench/tmp
+/var/tmp/pbench-test-utils/pbench/tmp/tools-default
+/var/tmp/pbench-test-utils/pbench/tools-default
+/var/tmp/pbench-test-utils/pbench/tools-default/sar
+/var/tmp/pbench-test-utils/pbench/tools-default/sar:
+--interval=40
+
+--- pbench tree state
++++ pbench.log file contents
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools]started: --dir=/var/tmp/pbench-test-utils/pbench/tmp
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools] /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/sar --stop --iteration=1 --group=default --dir=/var/tmp/pbench-test-utils/pbench/tmp --interval=40 
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] stopping sar
+/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-stop-tools]completed: 
+--- pbench.log file contents
++++ test-execution.log file contents
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill 123456
+/var/tmp/pbench-test-utils/test-execution.log:NONE/123456
+/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/kill 123456 exited with 0
+--- test-execution.log file contents
diff --git a/agent/util-scripts/pbench-postprocess-tools b/agent/util-scripts/pbench-postprocess-tools
index c358fc5d3f..aad39cd9cc 100755
--- a/agent/util-scripts/pbench-postprocess-tools
+++ b/agent/util-scripts/pbench-postprocess-tools
@@ -84,7 +84,7 @@ else
 	tool_group_dir="$pbench_run/tools-default"
 fi
 if [ ! -d "$tool_group_dir" ]; then
-	error_log "[$script_name] failed to find tools group directroy, \"$tool_group_dir\""
+	error_log "[$script_name] failed to find tools group directory, \"$tool_group_dir\""
 	exit 1
 fi
 
@@ -152,9 +152,11 @@ for this_tool_file in `/bin/ls $tool_group_dir`; do
 			if [ "$action" == "kill" ]; then
 				screens_to_kill=`screen -ls | grep "$screen_name" | awk '{print $1}'`
 				pids_to_kill=`echo $screens_to_kill | awk -F. '{print $1}'`
-				echo "killing the following screen sessions for $name:"
-				echo "$screens_to_kill"
-				kill $pids_to_kill
+				if [ ! -z "$pids_to_kill" ] ;then
+					echo -n "killing the following screen sessions for $name: "
+					echo "$screens_to_kill"
+					kill $pids_to_kill
+				fi
 			else
 				$pbench_bin/tool-scripts/$name --$action --iteration=$iteration --group=$group --dir=$dir "${tool_opts[@]}" &
 			fi
diff --git a/agent/util-scripts/samples/pbench-stop-tools/README.org b/agent/util-scripts/samples/pbench-stop-tools/README.org
new file mode 100644
index 0000000000..109ddae1b8
--- /dev/null
+++ b/agent/util-scripts/samples/pbench-stop-tools/README.org
@@ -0,0 +1,57 @@
+* Conventions uses for the pbench-stop-tools unit tests
+
+These unit tests test (mostly) the safe_kill) function in tool-scripts/kvm-spinlock
+(and all the tools that link to that). The basic premise is that we fake the environment
+as if pbench-start-tools had been called, and then call pbench-stop-tools. pidof and kill
+are mocked in the test-bin directory). kill is a bash builtin, so I disabled it in the
+base file conditionally when _PBENCH_BENCH_TESTS is set to 1 (which is set in the unittests
+scripts). The base file is sourced by every other script, so the built-in kill is disabled
+everywhere and the mock version controls.
+
+The following conventions are followed:
+
+- Since pbench-stop-tools calls the tool stop scripts asynchronously,
+  the order of the various entries in the output can vary. To get
+  around this problem, each unit test deals with a single tool.
+
+- The safe_kill function deals with the following possibilities:
+
+  + the stored pid is used to kill the tool successfully - no
+    instances of that tool remain: return success.
+
+  + the stored pid is used to kill the tool successfully - other
+    instances of that tool are still running: issue warning and return
+    success.
+
+  + the stored pid is used to kill the tool unsuccessfully - other
+    instances of the tool may or may not be running (a warning is
+    issued if necessary): try to kill the pid with the KILL signal
+    and return failure (status=2).
+
+- The mock pidof and kill commands simulate the various situations as
+  follows:
+
+  + pidof turbostat returns a list of two pids to simulate two instances
+    running: 123463 and 123464. The former is the stored pid for turbostat
+    and it also causes kill to fail unless called with a KILL signal.
+
+  + pidof kvmstat returns one pid: 123463, the "unkillable" pid.
+
+  + =pidof kvmtrace= and =pidof iostat= returns a list of "killable" pids.
+
+  + =pidof numastat= and =pidof sar=  returns one "killable" pid.
+
+Currently, iostat/sar do not go through safe_kill: they call kill directly
+and do not check the result. Assuming no problems are found with safe_kill,
+every tool is going to use it.
+
+* Test cases 
+- test-05: turbostat
+- test-06: kvmstat
+- test-07: kvmtrace
+- test-08: numastat
+- test-09: iostat
+- test-10: sar
+
+
+
diff --git a/agent/util-scripts/samples/pbench-stop-tools/test-05/tmp/default.1.turbostat.pid b/agent/util-scripts/samples/pbench-stop-tools/test-05/tmp/default.1.turbostat.pid
new file mode 100644
index 0000000000..200252bcfb
--- /dev/null
+++ b/agent/util-scripts/samples/pbench-stop-tools/test-05/tmp/default.1.turbostat.pid
@@ -0,0 +1 @@
+123463
diff --git a/agent/util-scripts/samples/pbench-stop-tools/test-05/tools-default/turbostat b/agent/util-scripts/samples/pbench-stop-tools/test-05/tools-default/turbostat
new file mode 100644
index 0000000000..230b04c853
--- /dev/null
+++ b/agent/util-scripts/samples/pbench-stop-tools/test-05/tools-default/turbostat
@@ -0,0 +1 @@
+--interval=3
diff --git a/agent/util-scripts/samples/pbench-stop-tools/test-06/tmp/default.1.kvmstat.pid b/agent/util-scripts/samples/pbench-stop-tools/test-06/tmp/default.1.kvmstat.pid
new file mode 100644
index 0000000000..200252bcfb
--- /dev/null
+++ b/agent/util-scripts/samples/pbench-stop-tools/test-06/tmp/default.1.kvmstat.pid
@@ -0,0 +1 @@
+123463
diff --git a/agent/util-scripts/samples/pbench-stop-tools/test-06/tools-default/kvmstat b/agent/util-scripts/samples/pbench-stop-tools/test-06/tools-default/kvmstat
new file mode 100644
index 0000000000..eb60d1a790
--- /dev/null
+++ b/agent/util-scripts/samples/pbench-stop-tools/test-06/tools-default/kvmstat
@@ -0,0 +1 @@
+--interval="30"
diff --git a/agent/util-scripts/samples/pbench-stop-tools/test-07/tmp/default.1.vmstat.pid b/agent/util-scripts/samples/pbench-stop-tools/test-07/tmp/default.1.vmstat.pid
new file mode 100644
index 0000000000..9f358a4add
--- /dev/null
+++ b/agent/util-scripts/samples/pbench-stop-tools/test-07/tmp/default.1.vmstat.pid
@@ -0,0 +1 @@
+123456
diff --git a/agent/util-scripts/samples/pbench-stop-tools/test-07/tools-default/vmstat b/agent/util-scripts/samples/pbench-stop-tools/test-07/tools-default/vmstat
new file mode 100644
index 0000000000..fbb5966de5
--- /dev/null
+++ b/agent/util-scripts/samples/pbench-stop-tools/test-07/tools-default/vmstat
@@ -0,0 +1 @@
+--interval=50
diff --git a/agent/util-scripts/samples/pbench-stop-tools/test-08/tmp/default.1.numastat.pid b/agent/util-scripts/samples/pbench-stop-tools/test-08/tmp/default.1.numastat.pid
new file mode 100644
index 0000000000..9f358a4add
--- /dev/null
+++ b/agent/util-scripts/samples/pbench-stop-tools/test-08/tmp/default.1.numastat.pid
@@ -0,0 +1 @@
+123456
diff --git a/agent/util-scripts/samples/pbench-stop-tools/test-08/tools-default/numastat b/agent/util-scripts/samples/pbench-stop-tools/test-08/tools-default/numastat
new file mode 100644
index 0000000000..fbb5966de5
--- /dev/null
+++ b/agent/util-scripts/samples/pbench-stop-tools/test-08/tools-default/numastat
@@ -0,0 +1 @@
+--interval=50
diff --git a/agent/util-scripts/samples/pbench-stop-tools/test-09/tmp/default.1.iostat.pid b/agent/util-scripts/samples/pbench-stop-tools/test-09/tmp/default.1.iostat.pid
new file mode 100644
index 0000000000..9f358a4add
--- /dev/null
+++ b/agent/util-scripts/samples/pbench-stop-tools/test-09/tmp/default.1.iostat.pid
@@ -0,0 +1 @@
+123456
diff --git a/agent/util-scripts/samples/pbench-stop-tools/test-09/tools-default/iostat b/agent/util-scripts/samples/pbench-stop-tools/test-09/tools-default/iostat
new file mode 100644
index 0000000000..230b04c853
--- /dev/null
+++ b/agent/util-scripts/samples/pbench-stop-tools/test-09/tools-default/iostat
@@ -0,0 +1 @@
+--interval=3
diff --git a/agent/util-scripts/samples/pbench-stop-tools/test-10/tmp/default.1.sar.pid b/agent/util-scripts/samples/pbench-stop-tools/test-10/tmp/default.1.sar.pid
new file mode 100644
index 0000000000..9f358a4add
--- /dev/null
+++ b/agent/util-scripts/samples/pbench-stop-tools/test-10/tmp/default.1.sar.pid
@@ -0,0 +1 @@
+123456
diff --git a/agent/util-scripts/samples/pbench-stop-tools/test-10/tools-default/sar b/agent/util-scripts/samples/pbench-stop-tools/test-10/tools-default/sar
new file mode 100644
index 0000000000..8e205476e3
--- /dev/null
+++ b/agent/util-scripts/samples/pbench-stop-tools/test-10/tools-default/sar
@@ -0,0 +1,2 @@
+--interval=40
+
diff --git a/agent/util-scripts/unittests b/agent/util-scripts/unittests
index 43baf28418..bb251703de 100755
--- a/agent/util-scripts/unittests
+++ b/agent/util-scripts/unittests
@@ -28,7 +28,7 @@ cp $_tdir/../base $_testopt/
 let res=res+$?
 mkdir -p $_testopt/tool-scripts
 let res=res+$?
-cp $_tdir/../tool-scripts/mpstat $_testopt/tool-scripts
+cp $_tdir/../tool-scripts/{iostat,kvmstat,mpstat,numastat,perf,pidstat,proc-interrupts,proc-vmstat,sar,turbostat,vmstat} $_testopt/tool-scripts
 let res=res+$?
 scripts="$_tdir/pbench-register-tool $_tdir/pbench-metadata-log"
 scripts="$_tdir/pbench-*"
@@ -142,7 +142,7 @@ let errs=0
 
 tests="$*"
 if [ -z "$tests" ] ;then
-    tests="test-00 test-01 test-02 test-03 test-04 test-14"
+    tests="$(seq -f "test-%02g" 0 10) test-14"
 fi
 
 declare -A tools=([test-00]="pbench-register-tool"
@@ -150,6 +150,12 @@ declare -A tools=([test-00]="pbench-register-tool"
                   [test-02]="pbench-metadata-log"
                   [test-03]="pbench-metadata-log"
                   [test-04]="pbench-metadata-log"
+                  [test-05]="pbench-stop-tools"
+                  [test-06]="pbench-stop-tools"
+                  [test-07]="pbench-stop-tools"
+                  [test-08]="pbench-stop-tools"
+                  [test-09]="pbench-stop-tools"
+                  [test-10]="pbench-stop-tools"
                   [test-14]="pbench-agent-config-activate"
                  )
 declare -A options=([test-00]="--name=mpstat --group=default -- --interval=10"
@@ -157,10 +163,15 @@ declare -A options=([test-00]="--name=mpstat --group=default -- --interval=10"
                     [test-02]="--dir=$_testdir/tmp beg"
                     [test-03]="--dir=$_testdir/tmp beg"
                     [test-04]="--dir=$_testdir/tmp beg"
+                    [test-05]="--dir=$_testdir/tmp"
+                    [test-06]="--dir=$_testdir/tmp"
+                    [test-07]="--dir=$_testdir/tmp"
+                    [test-08]="--dir=$_testdir/tmp"
+                    [test-09]="--dir=$_testdir/tmp"
+                    [test-10]="--dir=$_testdir/tmp"
                     [test-14]="$_testdir/tmp/pbench-agent.cfg"
                    )
 
-
 for tst in $tests; do
     tool=${tools[$tst]}
     opts=${options[$tst]}