More backports for b0.71

agent/base

@@ -291,20 +291,18 @@ function generate_inventory {
     done
 }
 
-function resolve_benchmark_bin {
-    # Encapsulation of method for resolving the actual benchmark
-    # binary.
-    which --skip-alias --skip-functions "${1}"
-}
-
 function vercmp {
     # Compare two package versions via `rpmdev-vercmp`.
     local _package="${1}"
     local _match="${2}"
     local _exp_ver="${3}"
     local _ins_ver="${4}"
 
-    local _stderr=$(rpmdev-vercmp "${_ins_ver}" "${_exp_ver}" 2>&1 > /dev/null)
+    # Note we declare `_stderr` local without an initializer because the return
+    # code of rpmdev-vercmp would be overwritten by the return code of the
+    # "local" declaration.
+    local _stderr
+    _stderr=$(rpmdev-vercmp "${_ins_ver}" "${_exp_ver}" 2>&1 > /dev/null)
     local _res=${?}
     if [[ ${_res} -ne 0 && ${_res} -ne 11 && ${_res} -ne 12 ]]; then
         error_log "rpmdev-vercmp - ${_stderr}"

agent/bench-scripts/driver/linpack

@@ -10,6 +10,7 @@ linpack_input_file_name="linpack.input"
 linpack_metadata_file_name="linpack.meta"
 linpack_output_file_name="linpack.out"
 linpack_command_file_name="linpack.cmd"
+linpack_pid_file_name="linpack.pid"
 
 binary=""
 
@@ -38,15 +39,19 @@ kmp_affinity_args_def="nowarnings,compact,1,0,granularity=fine"
 kmp_affinity_args=""
 numactl_args=""
 
-# Output directory required.
+# Output directory, defaults to the current working directory if not provided.
 output_dir=""
 
+# The linpack driver always performs pre-checks. When requested, the driver will
+# only run the pre-check and exit.
+pre_check_only=0
+
 function usage() {
 	cat <<-__EOF__
 Usage: ${script_name} [--problem-sizes=#[[,#]...]] [--leading-dimenions=#[[,#]...]]
         [--run-trials=#[[,#]]...] [--alignment-values=#[[,#]...]] [--use-omp={y|n}]
         [--threads=#] [--kmp-affinity=<options>] [--numactl-args=<args>]
-        [--output-dir=<directory>] <linpack binary path>
+        [--output-dir=<directory>] <linpack version number installed>
 
 Optional output control:
 
@@ -128,7 +133,7 @@ an invocation errors.
 }
 
 # Process options and arguments
-opts=$(getopt -q -o h --longoptions "header:,subheader:,problem-sizes:,leading-dimensions:,alignment-values:,run-trials:,threads:,use-omp:,kmp-affinity:,numactl-args:,output-dir:,help" -n "${script_name}" -- "${@}")
+opts=$(getopt -q -o h --longoptions "header:,subheader:,problem-sizes:,leading-dimensions:,alignment-values:,run-trials:,threads:,use-omp:,kmp-affinity:,numactl-args:,output-dir:,pre-check-only,help" -n "${script_name}" -- "${@}")
 if [[ ${?} -ne 0 ]]; then
 	printf -- "%s %s\n\n\tunrecognized option specified\n\n" "${script_name}" "${*}" >&2
 	usage >&2
@@ -211,6 +216,9 @@ while true; do
 			shift
 		fi
 		;;
+	--pre-check-only)
+		pre_check_only=1
+		;;
 	-h|--help)
 		help
 		exit 0
@@ -225,16 +233,26 @@ while true; do
 	esac
 done
 
-binary=${1}
-if [[ -z "${binary}" ]]; then
-	printf -- "[%s] ERROR: You must specify the location of the LINPACK binary\n\n" "${script_name}" >&2
+ver=${1}
+if [[ -z "${ver}" ]]; then
+	printf -- "[%s] ERROR: You must specify the version of the LINPACK binary\n\n" "${script_name}" >&2
 	usage >&2
 	exit 2
 fi
+
+# Installation directory, optional 2nd argument, defaults to /usr/local.
+install_prefix_dir=${2:-"/usr/local"}
+
+binary="${install_prefix_dir}/pbench-linpack-${ver}/benchmarks/linpack/xlinpack_xeon64"
 if [[ ! -x "${binary}" ]]; then
-	printf -- "[%s] ERROR: The --binary must exist and be executable\n" "${script_name}" >&2
+	printf -- "[%s] ERROR: The --binary, '${binary}', must exist and be executable\n" "${script_name}" >&2
 	exit 2
 fi
+
+if [[ ${pre_check_only} -ne 0 ]]; then
+	exit 0
+fi
+
 output_dir=${output_dir:-$(pwd)}
 if [[ ! -d "${output_dir}" ]]; then
 	printf -- "[%s] ERROR: Specified --output-dir '${output_dir}' is not a directory\n" "${script_name}" >&2
@@ -270,6 +288,7 @@ linpack_input_file="${output_dir}/${linpack_input_file_name}"
 linpack_metadata_file="${output_dir}/${linpack_metadata_file_name}"
 linpack_output_file="${output_dir}/${linpack_output_file_name}"
 linpack_command_file="${output_dir}/${linpack_command_file_name}"
+linpack_pid_file="${output_dir}/${linpack_pid_file_name}"
 
 # N.B. - Trailing whitespace is required.
 numactl_cmd=${numactl_args:+"numactl ${numactl_args} "}
@@ -312,6 +331,9 @@ cat > ${linpack_input_file} <<-__EOF__
 	${alignment_values} # alingment values (in KBytes)
 	__EOF__
 
+# Declare ourselves to anybody waiting.
+echo "$$" > ${linpack_pid_file}
+
 # Now we can execute the final LINPACK command.
 source ${linpack_command_file}
 exit_code=${?}

agent/bench-scripts/driver/linpack-wait

@@ -3,7 +3,30 @@
 
 # A very simple method to wait for the linpack driver to report it has
 # finished.
-while [[ ! -e "${1}/linpack.meta" ]]; do
+
+if [[ ! -d "${1}" ]]; then
+	printf -- "linpack-wait: target directory, '${1}', does not exist!\n" >&2
+	exit 1
+fi
+
+# Wait for the linpack pid file to show up.
+let cnt=60
+while [[ ${cnt} -gt 0 && -d "${1}" && ! -e "${1}/linpack.pid" ]]; do
+	sleep 1
+	(( cnt-- ))
+done
+if [[ ! -d "${1}" ]]; then
+	printf -- "linpack-wait: target directory, '${1}', no longer exists!\n" >&2
+	exit 1
+fi
+if [[ ! -e "${1}/linpack.pid" ]]; then
+	printf -- "linpack-wait: linpack pid file, '${1}/linpack.pid', failed to show up\n" >&2
+	exit 1
+fi
+
+# At this point, we wait for the linpack process to stop running.
+pid=$(< "${1}/linpack.pid")
+while [[ -d /proc/${pid} ]]; do
 	sleep 1
 done
 exit 0

agent/bench-scripts/pbench-dbench

@@ -24,7 +24,7 @@ warn_log "${script_name} is deprecated and will be removed in the next release"
 benchmark_rpm=$script_name
 export benchmark="dbench"
 if [[ -z "${benchmark_bin}" ]]; then
-	benchmark_bin="$(resolve_benchmark_bin "${benchmark}")"
+	benchmark_bin="$(command -v "${benchmark}")"
 	if [[ -z "${benchmark_bin}" ]]; then
 		error_log "[${script_name}] ${benchmark} executable not found on PATH=${PATH}"
 		exit 1

agent/bench-scripts/pbench-fio

@@ -11,16 +11,7 @@ pbench_bin="`cd ${script_path}/..; /bin/pwd`"
 . "$pbench_bin"/base
 
 export benchmark="fio"
-benchmark_rpm=${benchmark}
 export benchmark_run_dir=""
-# allow unit tests to override
-if [[ -z "${benchmark_bin}" ]]; then
-	benchmark_bin="$(resolve_benchmark_bin "${benchmark}")"
-	if [[ -z "${benchmark_bin}" ]]; then
-		error_log "[${script_name}] ${benchmark} executable not found on PATH=${PATH}"
-		exit 1
-	fi
-fi
 
 job_file="${script_path}/templates/fio.job"
 
@@ -168,6 +159,7 @@ function usage {
 }
 
 function fio_process_options() {
+	local opts
 	opts=$(getopt -q -o b:c:d:hj:r:s:t: --longoptions "help,max-stddev:,max-failures:,samples:,direct:,sync:,pre-check,clients:,client-file:,iodepth:,ioengine:,config:,jobs-per-dev:,job-mode:,rate-iops:,ramptime:,runtime:,test-types:,block-sizes:,file-size:,targets:,tool-group:,postprocess-only:,run-dir:,numjobs:,job-file:,sysinfo:,pre-iteration-script:,histogram-interval-sec:,histogram-interval-msec:,unique-ports" -n "getopt.sh" -- "$@")
 	if [ $? -ne 0 ]; then
 		printf -- "${script_name} $*\n"
@@ -177,6 +169,7 @@ function fio_process_options() {
 		exit 1
 	fi
 	eval set -- "$opts"
+	local arg
 	while true; do
 		arg="${1}"
 		shift
@@ -479,7 +472,12 @@ function local_pre_check() {
 
 	# First check for the supported version of fio in use.
 
-	local _stdout=$(${benchmark_bin} --version)
+	local _stdout
+	_stdout=$(${benchmark} --version 2>&1)
+	if [[ ${?} -ne 0 ]]; then
+		error_log "[${script_name}] ${benchmark} failed reporting its version: '${_stdout}'"
+		exit 1
+	fi
 	local ver_installed=${_stdout##*-}
 
 	vercmp "${benchmark}" "${match}" "${ver}" "${ver_installed}"
@@ -528,8 +526,7 @@ function fio_pre_check() {
 	fi
 	if [[ ${do_local_pre_check} -eq 1 ]]; then
 		debug_log "Running pre-check locally"
-		local_pre_check "${devs}" "${ver}" "${match}"
-		rc=${?}
+		local_pre_check "${devs}" "${ver}" "${match}" || rc=${?}
 	fi
 	if [[ ${rc} -ne 0 ]]; then
 		error_log "pre-check(s) failed, exiting"
@@ -598,7 +595,7 @@ function fio_run_job() {
 
 	if [ -n "${clients}" ]; then
 		for client in "${client_names[@]}"; do
-			local bash_c_cmd="'${benchmark_bin}' --server=,${client_ports[${client}]} > client-result.txt 2> client-result.err"
+			local bash_c_cmd="${benchmark} --server=,${client_ports[${client}]} > client-result.txt 2> client-result.err"
 			local screen_it="screen -L -Logfile fio-server.screen.log -dmS fio-server bash -c ${bash_c_cmd}"
 			if pbench-is-local ${client}; then
 				debug_log "killing any old local fio processes and starting new a fio process for the local client"
@@ -653,9 +650,9 @@ function fio_run_job() {
 	pbench-start-tools --group=$tool_group --dir=$benchmark_results_dir
 
 	# create a command file and keep it with the results for debugging later, or user can run outside of pbench
-	echo "$benchmark_bin $client_opts $bench_opts" >$benchmark_results_dir/fio.cmd
+	echo "$benchmark $client_opts $bench_opts" >$benchmark_results_dir/fio.cmd
 	chmod +x $benchmark_results_dir/fio.cmd
-	debug_log "$benchmark: Going to run [$benchmark_bin $bench_opts $client_opts]"
+	debug_log "$benchmark: Going to run [$benchmark $bench_opts $client_opts]"
 	pushd $benchmark_results_dir >/dev/null
 	$benchmark_results_dir/fio.cmd >$benchmark_results_dir/fio-result.txt
 	local _fio_exit_code=$?
@@ -896,6 +893,7 @@ function fio_run_benchmark() {
 }
 
 fio_process_options "$@"
+
 if [ "$postprocess_only" = "n" ]; then
 	ver="$(pbench-config version ${benchmark})"
 	if [[ -z "${ver}" ]]; then

agent/bench-scripts/pbench-iozone

@@ -25,7 +25,7 @@ warn_log "${script_name} is deprecated and will be removed in the next release"
 benchmark_rpm=$script_name
 export benchmark="iozone"
 if [[ -z "${benchmark_bin}" ]]; then
-	benchmark_bin="$(resolve_benchmark_bin "${benchmark}")"
+	benchmark_bin="$(command -v "${benchmark}")"
 	if [[ -z "${benchmark_bin}" ]]; then
 		error_log "[${script_name}] ${benchmark} executable not found on PATH=${PATH}"
 		exit 1

agent/bench-scripts/pbench-linpack

@@ -27,6 +27,11 @@ pbench_bin="$(realpath -e ${script_path}/..)"
 export benchmark="linpack"
 
 # Defaults
+
+# This script always runs a pre-check. When non-local clients are used, as part
+# of the pre-check operation, it invokes itself remotely to ONLY run a local
+# pre-check and exit.
+pre_check_only=0
 def_threads=$(cat /proc/cpuinfo | grep processor | wc -l)
 threads=${def_threads}
 def_nr_samples=2
@@ -49,7 +54,7 @@ function usage {
 }
 
 # Process options and arguments
-opts=$(getopt -q -o C:c:h --longoptions "config:,clients:,samples:,threads:,tool-group:,sysinfo:,help" -n "getopt.sh" -- "${@}")
+opts=$(getopt -q -o C:c:h --longoptions "config:,clients:,help,pre-check-only,samples:,sysinfo:,threads:,tool-group:" -n "getopt.sh" -- "${@}")
 if [[ ${?} -ne 0 ]]; then
 	printf -- "%s %s\n\n\tunrecognized option specified\n\n" "${script_name}" "${*}" >&2
 	usage >&2
@@ -72,27 +77,30 @@ while true; do
 			shift
 		fi
 		;;
+	--pre-check-only)
+		pre_check_only=1
+		;;
 	--samples)
 		if [[ -n "${1}" ]]; then
 			nr_samples="${1}"
 			shift
 		fi
 		;;
-	--threads)
+	--sysinfo)
 		if [[ -n "${1}" ]]; then
-			threads="${1}"
+			sysinfo="${1}"
 			shift
 		fi
 		;;
-	--tool-group)
+	--threads)
 		if [[ -n "${1}" ]]; then
-			tool_group="${1}"
+			threads="${1}"
 			shift
 		fi
 		;;
-	--sysinfo)
+	--tool-group)
 		if [[ -n "${1}" ]]; then
-			sysinfo="${1}"
+			tool_group="${1}"
 			shift
 		fi
 		;;
@@ -113,24 +121,49 @@ while true; do
 done
 verify_common_bench_script_options ${tool_group} ${sysinfo}
 
-ver="$(pbench-config version ${benchmark})"
-if [[ -z "${ver}" ]]; then
-	error_log "${script_name}: package version is missing in config file"
-	exit 1
-fi
-if [[ -z "${linpack_dir}" ]]; then
-	linpack_dir="/usr/local/${script_name}-${ver}/benchmarks/linpack"
-	linpack_dir_kind="default"
-else
-	linpack_dir_kind="provided"
+function pre_check {
+	# Invoke the linpack driver to perform a pre-check that it will be able to
+	# execute the benchmark. The expected version is the first argument, and the
+	# second is the expected directory prefix for the linpack installation.
+	local ver=${1}
+	local install_prefix_arg=${2}
+
+	${pbench_bin}/bench-scripts/driver/linpack --pre-check-only ${ver} ${install_prefix_arg}
+	return ${?}
+}
+
+if [[ ${pre_check_only} -ne 0 ]]; then
+	# We have been invoked remotely to check for the expected version of
+	# linpack installed.  The remote invocation has provided the arguments to
+	# pass to the pre-check function (see --pre-check-only invocation below).
+	pre_check ${@}
+	exit ${?}
 fi
-if [[ ! -d "${linpack_dir}" ]]; then
-	error_log "${script_name}: the ${linpack_dir_kind} linpack directory, ${linpack_dir}, does not exist"
+
+linpack_ver="$(pbench-config version ${benchmark})"
+if [[ -z "${linpack_ver}" ]]; then
+	error_log "${script_name}: package version is missing in config file"
 	exit 1
 fi
-linpack_cmd="${linpack_dir}/xlinpack_xeon64"
-if [[ ! -x "${linpack_cmd}" ]]; then
-	error_log "${script_name}: the expected linpack command, ${linpack_cmd}, does not exist"
+
+# Run the pre-check.
+let not_found=0
+for client in ${clients//,/ }; do
+	if pbench-is-local "${client}"; then
+		pre_check ${linpack_ver} ${PBENCH_LINPACK_INSTALL_PREFIX_DIR}
+		if [[ ${?} -ne 0 ]]; then
+			error_log "${script_name}: linpack not installed locally"
+			(( not_found++ ))
+		fi
+	else
+		ssh ${ssh_opts} ${client} ${script_name} --pre-check-only ${linpack_ver} ${PBENCH_LINPACK_INSTALL_PREFIX_DIR}
+		if [[ ${?} -ne 0 ]]; then
+			error_log "${script_name}: linpack not installed on client ${client}"
+			(( not_found++ ))
+		fi
+	fi
+done
+if [[ ${not_found} -gt 0 ]]; then
 	exit 1
 fi
 
@@ -243,7 +276,8 @@ for thread in ${threads//,/ }; do
 		pbench-start-tools --group=${tool_group} --dir="${sample_dir}"
 
 		run_it="pbench-linpack ${pbench_bin}/bench-scripts/driver/linpack"
-		run_it+=" --output-dir=${sample_dir} --threads=${thread} ${linpack_cmd}"
+		run_it+=" --output-dir=${sample_dir} --threads=${thread}"
+		run_it+=" ${linpack_ver} ${PBENCH_LINPACK_INSTALL_PREFIX_DIR}"
 		screen_it="screen -dmS ${run_it}"
 		for client in ${clients//,/ }; do
 			if pbench-is-local "${client}"; then