Make client list file absolute if it is not already

agent/base

@@ -133,7 +133,8 @@ if [[ -z "$_PBENCH_BENCH_TESTS" ]]; then
 
 else
     function check_install_rpm {
-        echo $@
+        echo $1
+        #$2 is the version which will vary
         return 0
     }
 fi

agent/bench-scripts/gold/pbench-fio/test-04.txt

@@ -1,5 +1,6 @@
 +++ Running test-04 pbench-fio
-pbench-fio 2.14
+pbench-fio
+python-pandas
 The following jobfile was created: /var/tmp/pbench-test-bench/pbench/fio__1900-01-01_00:00:00/1-rw-4KiB/sample1/fio.job
 [global]
 bs=4k
@@ -1109,7 +1110,8 @@ Iteration 6-randrw-1024KiB complete (6 of 6), with 1 pass and 6 failures
 /var/tmp/pbench-test-bench/pbench/tools-default/sar
 --- pbench tree state
 +++ pbench.log file contents
-/var/tmp/pbench-test-bench/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-fio]pbench-fio is installed
+/var/tmp/pbench-test-bench/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-fio]pbench-fio 2.14 is installed
+/var/tmp/pbench-test-bench/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-fio]python-pandas is installed
 /var/tmp/pbench-test-bench/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] fio_device_check() /tmp/fio 
 /var/tmp/pbench-test-bench/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] fio: Going to run [/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/bm --output-format=json /var/tmp/pbench-test-bench/pbench/fio__1900-01-01_00:00:00/1-rw-4KiB/sample1/fio.job ]
 /var/tmp/pbench-test-bench/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] post-processing fio result

agent/bench-scripts/gold/pbench-uperf/test-00.txt

@@ -1,5 +1,5 @@
 +++ Running test-00 pbench-uperf
-pbench-uperf 1.0.4
+pbench-uperf
 Starting iteration  (1 of 2)
 test sample 1 of 2
 client[127.0.0.1]protocol[tcp]test[rr]instances[1]size[64] <-> server[127.0.0.1]

agent/bench-scripts/gold/pbench-uperf/test-01.txt

@@ -1,5 +1,5 @@
 +++ Running test-01 pbench-uperf
-pbench-uperf 1.0.4
+pbench-uperf
 Starting iteration  (1 of 2)
 test sample 1 of 2
 client[c1]node[1]protocol[tcp]test[rr]instances[1]size[64] <-> server[s1]node[0]

agent/bench-scripts/pbench-fio

@@ -40,6 +40,7 @@ maxstddevpct=5 # maximum allowable standard deviation in percent
 max_failures=6 # after N failed attempts to hit below $maxstddevpct, move on to the nest test
 supported_test_types="read,write,rw,randread,randwrite,randrw"
 install_only="n"
+remote_only="n"
 config=""
 rate_iops=""
 test_types="read,randread"		# default is -non- destructive
@@ -134,10 +135,13 @@ function fio_usage() {
                 printf -- "\t--install\n"
                 printf "\t\tinstall only\n"
                 printf "\t\tDefault is n\n"
+		printf -- "\t--remote-only\n"
+		printf "\trun this on the remotes only\n"
 }
 
 function fio_process_options() {
-	opts=$(getopt -q -o jic:t:b:s:d:r: --longoptions "help,max-stddev:,max-failures:,samples:,direct:,sync:,install,clients:,client-file:,iodepth:,ioengine:,config:,jobs-per-dev:,job-mode:,rate-iops:,ramptime:,runtime:,test-types:,block-sizes:,file-size:,targets:,tool-group:,postprocess-only:,run-dir:,directory:,numjobs:,job-file:,pre-iteration-script:," -n "getopt.sh" -- "$@");
+	opts=$(getopt -q -o jic:t:b:s:d:r: --longoptions "help,max-stddev:,max-failures:,samples:,direct:,sync:,install,remote-only,clients:,client-file:,iodepth:,ioengine:,config:,jobs-per-dev:,job-mode:,rate-iops:,ramptime:,runtime:,test-types:,block-sizes:,file-size:,targets:,tool-group:,postprocess-only:,run-dir:,directory:,numjobs:,job-file:" -n "getopt.sh" -- "$@");
+
 	if [ $? -ne 0 ]; then
 		printf "\t${benchmark}: you specified an invalid option\n\n"
 		fio_usage
@@ -154,6 +158,10 @@ function fio_process_options() {
 			shift;
 			install_only="y"
 			;;
+			--remote-only)
+			shift;
+			remote_only="y"
+			;;
 			--max-stddev)
 			shift;
 			if [ -n "$1" ]; then
@@ -251,6 +259,10 @@ function fio_process_options() {
 			if [ -n "$1" ]; then
 				if [ -e $1 ]; then
 					client_file=$1
+					if [[ "$client_file" != /* ]] ;then
+						# make it absolute
+						client_file=$PWD/$client_file
+					fi
 					while read line; do
 						clients="$clients,$line"
 					done <$1
@@ -359,7 +371,7 @@ function fio_process_options() {
 			;;
 		esac
 	done
-	if [ $postprocess_only == "n" ]; then
+	if [ "$postprocess_only" = "n" ]; then
 		benchmark_run_dir="$pbench_run/${benchmark}_${config}_$date"
 	else
 		if [ -z "$run_dir" ]; then
@@ -375,24 +387,36 @@ function fio_process_options() {
 
 # Ensure the right version of the benchmark is installed
 function fio_install() {
-	if [ "$postprocess_only" == "n" ]; then
+	if [ "$postprocess_only" = "n" ]; then
 		if check_install_rpm $benchmark_rpm $ver; then
-			debug_log "[$script_name]$benchmark_rpm is installed"
+			debug_log "[$script_name]$benchmark_rpm $ver is installed"
 		else
-			debug_log "[$script_name]$benchmark_rpm installation failed, exiting"
+			debug_log "[$script_name]$benchmark_rpm $ver installation failed, exiting"
 			exit 1
-		fi 
+		fi
+
 	fi
 	if [ ! -z "$clients" ] ; then
 		debug_log "verifying clients have fio installed"
 		echo "verifying clients have fio installed"
 		for client in `echo $clients | sed -e s/,/" "/g`; do
-			ssh $ssh_opts $client ${pbench_install_dir}/bench-scripts/$script_name --install &
+			ssh $ssh_opts $client ${pbench_install_dir}/bench-scripts/$script_name --remote-only --install &
 		done
 		wait
 	fi
-	if [ "$install_only" == "y" ]; then
-		exit 0
+}
+
+# install python-pandas on the controller: fiologparser_hist.py needs it
+function pandas_install() {
+	if [ "$postprocess_only" = "n" ]; then
+		if check_install_rpm python-pandas; then
+			debug_log "[$script_name]python-pandas is installed"
+		else
+			debug_log "[$script_name]python-pandas installation failed, exiting."
+                    	debug_log "[$script_name]On RHEL, python-pandas is in the EPEL repo."
+                    	debug_log "[$script_name]See https://fedoraproject.org/wiki/EPEL for details."
+			exit 1
+		fi
 	fi
 }
 
@@ -401,7 +425,7 @@ function print_iteration {
 	printf "\n%28s" "$1" >>$benchmark_summary_txt_file
 	printf "\n%s" "$1" >>$benchmark_summary_csv_file
 	hists=`cat $iteration_dir/reference-result/fio.job 2>/dev/null | grep "^log_hist_msec"`
-	if [ $1 == "iteration" ]; then
+	if [ "$1" = "iteration" ]; then
 		# this is just a label, so no links here
 		if [ ! -z "$hist_interval" ]; then
 			printf "\n%28s %s %s %s" "iteration" "summary" "hist-results" "tools">>$benchmark_summary_html_file
@@ -442,7 +466,7 @@ function fio_device_check() {
 	local dev=""
 	local client=""
 	local rc=0
-	if [ "$postprocess_only" == "n" ]; then
+	if [ "$postprocess_only" = "n" ]; then
 		debug_log "fio_device_check() $devs $clients"
 		for dev in `echo $devs | sed -e s/,/" "/g`; do
 			if echo $dev | grep -q "^/dev/"; then
@@ -543,7 +567,7 @@ function fio_run_job() {
 	local client_opts=""
 
 	if [ ! -z "$client_file" ] ;then
-		typeset -i nclients=$(wc -l $client_file)
+		typeset -i nclients=$(wc -l $client_file | cut -d ' ' -f 1)
 		client_opts="--client=$client_file --max-jobs=$nclients"
 	elif [ ! -z "$clients" ]; then
 		local max_jobs=0
@@ -570,12 +594,12 @@ function fio_run_job() {
 		debug_log "getting log files from clients"
 		for client in `echo $clients | sed -e s/,/" "/g`; do
 			mkdir -p $benchmark_results_dir/clients/$client
-			scp $ssh_opts $client:"$benchmark_results_dir/*.log" $benchmark_results_dir/clients/$client/ >/dev/null &
+			scp $ssh_opts $client:"$benchmark_results_dir/*.log" $benchmark_results_dir/clients/$client/ 2>/dev/null &
 		done
 		wait
 		for client in `echo $clients | sed -e s/,/" "/g`; do
 			mkdir -p $benchmark_results_dir/clients/$client
-			ssh $ssh_opts $client /bin/rm "$benchmark_results_dir/*.log"  &
+			ssh $ssh_opts $client /bin/rm -f "$benchmark_results_dir/*.log"  &
 		done
 		wait
 	else
@@ -624,7 +648,7 @@ function fio_run_benchmark() {
 						echo "Restarting iteration $iteration ($count of $total_iterations)"
 						log "Restarting iteration $iteration ($count of $total_iterations)"
 					fi
-					if [ "$postprocess_only" == "n" ]; then
+					if [ "$postprocess_only" = "n" ]; then
 						mkdir -p $iteration_dir
 					else
 						if [ ! -e $iteration_dir ]; then
@@ -659,7 +683,7 @@ function fio_run_benchmark() {
 							fi
 							benchmark_results_dir="$iteration_dir/sample$sample"
 							benchmark_tools_dir="$benchmark_results_dir/tools-$tool_group"
-							if [ "$postprocess_only" == "n" ]; then
+							if [ "$postprocess_only" = "n" ]; then
 								mkdir -p $benchmark_results_dir
 								fio_job_file="$benchmark_results_dir/fio.job"
 								fio_create_jobfile "$test_type" "$ioengine" "$block_size" "$iodepth" "$direct" "$sync" "$runtime" "$ramptime" "$file_size" "$rate_iops" "$dev" "$fio_job_file"
@@ -699,7 +723,7 @@ function fio_run_benchmark() {
 								# NOTE: If --log_unix_epoch gets enabled in fio.job, then fiologparser_hist.py will not merge
 								# histograms into an "average" across the samples.
 								fiologparser_hist.py \
-									$iteration_dir/sample*/clients/*/*_clat_hist.* \
+									$iteration_dir/sample*/fio_clat_hist.*.log.* \
 									> $hist_results/hist.csv
 								$script_path/postprocess/$benchmark-postprocess-viz.py \
 									--job-file=$iteration_dir/sample1/fio.job \
@@ -757,11 +781,11 @@ function fio_run_benchmark() {
 								else
 									# delete the tool data [and respose time log for rr tests] from the other samples to save space
 									# this option is off by default
-									if [ "$keep_failed_tool_data" == "n" ]; then
+									if [ "$keep_failed_tool_data" = "n" ]; then
 										/bin/rm -rf $iteration_dir/$sample_dir/tools-* $iteration_dir/$sample_dir/response-times.txt
 									fi
 									# since non reference-result sample data is rarely referenced, tar it up to reduce the number of files used
-									if [ "$tar_nonref_data" == "y" ]; then
+									if [ "$tar_nonref_data" = "y" ]; then
 										pushd "$iteration_dir" >/dev/null; tar --create --xz --force-local --file=$sample_dir.tar.xz $sample_dir && /bin/rm -rf $sample_dir; popd >/dev/null
 									fi
 								fi
@@ -850,6 +874,14 @@ function fio_print_summary() {
 fio_process_options "$@"
 fio_install
 
+# pandas installed only on controller
+if [ "$remote_only" = "n" ] ;then
+	pandas_install
+fi
+if [ "$install_only" = "y" ]; then
+	exit 0
+fi
+
 mkdir -p $benchmark_run_dir
 export benchmark config
 pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir beg

agent/bench-scripts/postprocess/fio-postprocess

@@ -14,7 +14,7 @@ my $js_str;
 my $interval = `cat $dir/fio.job | grep "^log_hist_msec"`;
 if ( ! $interval =~ /^\s*$/ ) {
 	system("mkdir -p $dir/hist");
-	system("fiologparser_hist.py $dir/clients/*/*_clat_hist.* > $dir/hist/hist.csv");
+	system("fiologparser_hist.py $dir/fio_clat_hist.*.log.* > $dir/hist/hist.csv");
 	system(abs_path($0) . "-viz.py --job-file=$dir/fio.job $dir/hist");
 }
 

agent/bench-scripts/unittests

@@ -127,90 +127,53 @@ function _reset_state {
 
 let errs=0
 
-#
-# 1. Verify uperf
-#
-_setup_state
-_run test-00 pbench-uperf  --config=test-00  --test-types=rr,stream  --message-sizes=64  --instances=1  --protocols=tcp  --runtime=20  --samples=2
-res=$?
-_save_tree
-_dump_logs
-_verify_output $res test-00 pbench-uperf
-res=$?
-let errs=$errs+$res
-_reset_state
-
-#
-# 2. Verify uperf numa opts
-#
-_setup_state
-_run test-01 pbench-uperf  --config=test-01  --test-types=rr,stream  --message-sizes=64  --instances=1  --protocols=tcp  --runtime=20  --samples=2  --servers=s1,s2,s3  --clients=c1,c2,c3  --server-node=0,-1,2  --client-node=1,-1,3
-res=$?
-_save_tree
-_dump_logs
-_verify_output $res test-01 pbench-uperf
-res=$?
-let errs=$errs+$res
-_reset_state
-
-#
-# 3. Verify dbench
-#
-_setup_state
-_run test-02 pbench-dbench --threads=24,48  --client-nodes=0,1  --clients=127.0.0.1,127.0.0.1  --max-stddev=10
-res=$?
-_save_tree
-_dump_logs
-_verify_output $res test-02 pbench-dbench
-res=$?
-let errs=$errs+$res
-_reset_state
-
-#
-# 4. Verify moongen
-#
-_setup_state
-_run test-03 pbench-moongen --test-type=throughput --samples=3 --search-runtime=10 --validation-runtime=10 --max-drop-pct=0,1 --frame-sizes=64,1024
-res=$?
-_save_tree
-_dump_logs
-_verify_output $res test-03 pbench-moongen
-res=$?
-let errs=$errs+$res
-_reset_state
-
-#
-# 5. Verify fio
-#
-_setup_state
-_run test-04 pbench-fio --test-types=rw,randrw
-res=$?
-_save_tree
-_dump_logs
-_verify_output $res test-04 pbench-fio
-res=$?
-let errs=$errs+$res
-_reset_state
-
-_setup_state
-_run test-05 pbench-fio --test-type=throughput --samples=1 --clients=foo,bar
-res=$?
-_save_tree
-_dump_logs
-_verify_output $res test-05 pbench-fio
-res=$?
-let errs=$errs+$res
-_reset_state
-
-_setup_state
-_run test-06 pbench-fio --test-type=throughput --samples=1 --client-file=foo
-res=$?
-_save_tree
-_dump_logs
-_verify_output $res test-06 pbench-fio
-res=$?
-let errs=$errs+$res
-_reset_state
+declare -A cmds=(
+    [test-00]="pbench-uperf  --config=test-00  --test-types=rr,stream  --message-sizes=64  --instances=1  --protocols=tcp  --runtime=20  --samples=2"
+    [test-01]="pbench-uperf  --config=test-01  --test-types=rr,stream  --message-sizes=64  --instances=1  --protocols=tcp  --runtime=20  --samples=2  --servers=s1,s2,s3  --clients=c1,c2,c3  --server-node=0,-1,2  --client-node=1,-1,3"
+    [test-02]="pbench-dbench --threads=24,48  --client-nodes=0,1  --clients=127.0.0.1,127.0.0.1  --max-stddev=10"
+    [test-03]="pbench-moongen --test-type=throughput --samples=3 --search-runtime=10 --validation-runtime=10 --max-drop-pct=0,1 --frame-sizes=64,1024"
+    [test-04]="pbench-fio --test-types=rw,randrw"
+    [test-05]="pbench-fio --test-type=throughput --samples=1 --clients=foo,bar"
+    [test-06]="pbench-fio --test-type=throughput --samples=1 --client-file=/tmp/foo"
+)
+
+declare -A pre=(
+    [test-06]="echo foo bar baz | tr ' ' '\n' > /tmp/foo"
+)
+
+declare -A post=(
+    [test-06]="rm -f /tmp/foo"
+)
+
+
+tests="$*"
+if [ -z "$tests" ] ;then
+    typeset -i len=${#cmds[@]}
+    len=$len-1
+    tests="$(seq -f "test-%02g" 0 $len)"
+fi
+
+for tst in $tests; do
+    cmd=${cmds[$tst]}
+    benchmark=${cmd## }
+    tpre=${pre[$tst]}
+    tpost=${post[$tst]}
+    _setup_state
+    if [ ! -z "$tpre" ] ;then
+        eval "$tpre"
+    fi
+    _run $tst $cmd
+    res=$?
+    if [ ! -z "$tpost" ] ;then
+        eval "$tpost"
+    fi
+    _save_tree
+    _dump_logs
+    _verify_output $res $tst $benchmark
+    res=$?
+    let errs=$errs+$res
+    _reset_state
+done
 
 if [[ $errs -gt 0 ]]; then
     sts=1

agent/util-scripts/gold/pbench-register-tool/test-00.txt

@@ -1,5 +1,5 @@
 +++ Running test-00 pbench-register-tool
-pbench-sysstat 11.2.0
+pbench-sysstat
 mpstat tool is now registered in group default
 --- Finished test-00 pbench-register-tool (status=0}
 +++ pbench tree state