Skip to content

Tool Metadata Abstraction #1778

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 4 commits into from
Closed

Tool Metadata Abstraction #1778

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5db1dcf
First pass implementation of the "Tool Meister"
portante Jun 28, 2019
5a7e56d
Prometheus/Node_Exporter Full V1 Integration Commit
Maxusmusti Jul 15, 2020
303cee7
Added DCGM Tool to pbench-agent (keshavm02)
keshavm02 Jul 27, 2020
ea990c8
tool-meisters now load tool metadata from redis
Maxusmusti Sep 3, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
97 changes: 70 additions & 27 deletions agent/base
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -214,58 +214,90 @@ function check_install_rpm {
return ${rc}
}

# Constants
tools_group_prefix="tools-v1-"

function gen_tools_group_list() {
pushd "${pbench_run}" >/dev/null
if [[ ${?} -ne 0 ]]; then
return 1
fi
local groups=$(/bin/ls -d ${tools_group_prefix}* 2> /dev/null | sed -e s/^${tools_group_prefix}//g)
popd >/dev/null
printf "${groups}"
return 0
}

function gen_tools_group_dir {
local group="${1}"
if [[ -z "${group}" ]]; then
return 1
fi
local tools_group_dir="${pbench_run}/${tools_group_prefix}${group}"
printf -- "${tools_group_dir}"
return 0
}

function verify_tool_group {
local group=$1
if [[ -z "$group" ]]; then
error_log "Bad argument to verify_tool_group"
# Ensure we have a tools group directory to work with
local group="${1}"
local param_name=${2:-"--group"}
local tools_group_dir="$(gen_tools_group_dir ${group})"
if [[ -z "${tools_group_dir}" ]]; then
error_log "Empty argument passed to verify_tool_group()"
exit 1
fi
# Ensure we have a tools group directory to work with
local tool_group_dir="$pbench_run/tools-$group"
if [[ ! -d "$tool_group_dir" ]]; then
printf -- "\t${script_name}: invalid --tool-group option (\"$group\"), directory not found: $tool_group_dir\n"
if [[ ! -d "${tools_group_dir}" ]]; then
printf -- "\t${script_name}: invalid ${param_name} option (\"${group}\"), directory not found: ${tools_group_dir}\n" >&2
return 1
fi
printf -- "${tools_group_dir}"
return 0
}

function verify_collect_sysinfo {
local sysinfo=$1
pbench-collect-sysinfo --sysinfo=$sysinfo --check
local ret=$?
if [[ $ret -ne 0 ]]; then
printf -- "\t${script_name}: invalid --sysinfo option (\"$sysinfo\")\n"
local sysinfo="${1}"
pbench-collect-sysinfo --sysinfo=${sysinfo} --check
local ret=${?}
if [[ ${ret} -ne 0 ]]; then
printf -- "\t${script_name}: invalid --sysinfo option (\"${sysinfo}\")\n" >&2
fi
return $ret
return ${ret}
}

function verify_common_bench_script_options {
verify_tool_group "${1}"
verify_tool_group "${1}" --tool-group > /dev/null
local ret_tg=${?}
verify_collect_sysinfo "${2}"
local ret_cs=${?}
if [[ ${ret_tg} -ne 0 || ${ret_cs} -ne 0 ]]; then
printf -- "\n"
usage
usage >&2
exit 1
fi
}

# generate inventory file with controller and remotes
# Generate inventory file with controller and remotes.
# FIXME: need to make this work with new tools-v1 directory format.
function generate_inventory {
local component=$1
if [[ $component == "stockpile" ]]; then
echo "[stockpile]"
echo $HOSTNAME
echo "[all]"
echo $HOSTNAME
ls $tools_group_dir | awk -F@ '/^remote/ {print $2};'
local component="${1}"
local preamble=""
if [[ ${component} == "stockpile" ]]; then
preamble="[stockpile]\n%s\n\n[all]\n"
else
echo "[controller]"
echo $HOSTNAME
echo "[remote]"
ls $tools_group_dir | awk -F@ '/^remote/ {print $2};'
preamble="[controller]\n%s\n\n[remote]\n"
fi
printf -- ${preamble} "${hostname}"
for dirent in $(/bin/ls -1 ${tool_group_dir}); do
if [[ "${dirent}" == "__trigger__" ]]; then
# Ignore trigger files
continue
elif [[ ! -d ${tool_group_dir}/${dirent} ]]; then
# Ignore spurious files
continue
fi
printf -- "%s\n" "${dirent}"
done
}

function resolve_benchmark_bin {
Expand All @@ -274,6 +306,17 @@ function resolve_benchmark_bin {
which --skip-alias --skip-functions "${1}"
}

function interrupt() {
# On abnormal exit, stop the tool meister ...
pbench-tool-meister-stop
if [[ ${?} != 0 ]]; then
error_log "[${script_name}]: failed to stop the tool meisters on interrupt."
fi
# ... and make sure that the metadata log exists and is complete.
pbench-metadata-log --group=${tool_group} --dir=${benchmark_run_dir} int
return 0
}

if [[ "${_PBENCH_UNIT_TESTS}" == 1 ]] ;then
# For unit tests, we use a mock kill:
# disable the built-in.
Expand Down
39 changes: 27 additions & 12 deletions agent/bench-scripts/pbench-cyclictest
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ pbench_bin="`cd ${script_path}/..; /bin/pwd`"
# source the base script
. "$pbench_bin"/base

benchmark="cyclictest"
export benchmark="cyclictest"
benchmark_rpm=rt-tests
ver=0.87

Expand All @@ -30,7 +30,7 @@ baseconfig="`uname -r`"
runtimes="1m"
orig_cmd="$*"
tool_group="default"
config=""
export config=""
stress=0
cpu=""
sysinfo="default"
Expand Down Expand Up @@ -141,13 +141,18 @@ printf "\n" >>$benchmark_summary_txt_file
printf "<pre>\n# these results generated with:\n# uperf %s\n\n" "$scrtip_name $orig_cmd" >$benchmark_summary_html_file
printf "\n" >>$benchmark_summary_html_file

export benchmark config

count=1
pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg

# Start the tool meisters on each registered local/remote host
pbench-tool-meister-start "${tool_group}"
if [[ ${?} != 0 ]]; then
error_log "[${script_name}]: failed to start the tool meisters."
exit 1
fi

pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir beg
# on abnormal exit, make sure that the metadata log exists and is complete.
trap "pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir int" INT QUIT

trap "interrupt" INT QUIT TERM

if [ $stress ]; then
if [ ! -z "$cpu" ]; then
Expand All @@ -158,14 +163,16 @@ if [ $stress ]; then
stress_pid=$!
fi
fi
sample="sample1"
count=1
for runtime in `echo $runtimes | sed -e s/,/" "/g`; do
benchmark_cmd="cyclictest -m -n -q -p99 -h60 -D $runtime"
iteration="${count}-runtime_$runtime"
if [ ! -z "$cpu" ]; then
iteration="$iteration-cpu_$cpu"
iteration="${count}-runtime_${runtime}"
if [[ ! -z "${cpu}" ]]; then
iteration="${iteration}-cpu_${cpu}"
fi
echo $iteration >> $benchmark_iterations
benchmark_results_dir="$benchmark_run_dir/$iteration/sample1/"
benchmark_results_dir="${benchmark_run_dir}/${iteration}/${sample}"
result_file=$benchmark_results_dir/result.txt
benchmark_cmd_file=$benchmark_results_dir/cyclictest.cmd
mkdir -p $benchmark_results_dir
Expand All @@ -175,8 +182,9 @@ for runtime in `echo $runtimes | sed -e s/,/" "/g`; do
pbench-start-tools --group=$tool_group --dir=$benchmark_results_dir
$benchmark_cmd_file | tee $result_file
pbench-stop-tools --group=$tool_group --dir=$benchmark_results_dir
pbench-send-tools --group=$tool_group --dir=$benchmark_results_dir
pbench-postprocess-tools --group=$tool_group --dir=$benchmark_results_dir
ln -s sample1 $benchmark_run_dir/$iteration/reference-result
ln -s ${sample} ${benchmark_run_dir}/${iteration}/reference-result
min=`grep "Min Latencies:" $result_file | awk '{print $4}'`
avg=`grep "Avg Latencies:" $result_file | awk '{print $4}'`
max=`grep "Max Latencies:" $result_file | awk '{print $4}'`
Expand All @@ -187,6 +195,13 @@ if [ $stress ]; then
kill $stress_pid
fi
pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir end

# Stop the tool meisters on each registered local/remote host
pbench-tool-meister-stop
if [[ ${?} != 0 ]]; then
error_log "[${script_name}]: failed to stop the tool meisters."
fi

pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end
cat $benchmark_summary_txt_file

Expand Down
27 changes: 22 additions & 5 deletions agent/bench-scripts/pbench-dbench
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ pbench_bin="`cd ${script_path}/..; /bin/pwd`"
. "$pbench_bin"/base

benchmark_rpm=$script_name
benchmark="dbench"
export benchmark="dbench"
if [[ -z "${benchmark_bin}" ]]; then
benchmark_bin="$(resolve_benchmark_bin "${benchmark}")"
if [[ -z "${benchmark_bin}" ]]; then
Expand All @@ -45,7 +45,7 @@ fi
# Defaults

export benchmark_run_dir=""
config=""
export config=""
loadfile="/usr/local/share/doc/dbench/loadfiles/client.txt"
warmup=15
nr_samples=5
Expand Down Expand Up @@ -326,11 +326,19 @@ done
let total_iterations=count
count=1
max_key_length=0
export benchmark config

pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg

# Start the tool meisters on each registered local/remote host
pbench-tool-meister-start "${tool_group}"
if [[ ${?} != 0 ]]; then
error_log "[${script_name}]: failed to start the tool meisters."
exit 1
fi

pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir beg
# on abnormal exit, make sure that the metadata log exists and is complete.
trap "pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir int" INT QUIT

trap "interrupt" INT QUIT TERM

for thread_count in `echo $thread_counts | sed -e s/,/" "/g`; do
if [ $count -ge $start_iteration_num ]; then
Expand Down Expand Up @@ -418,6 +426,7 @@ for thread_count in `echo $thread_counts | sed -e s/,/" "/g`; do

# stop tools and clean up
pbench-stop-tools --group=$tool_group --dir=$benchmark_results_dir
pbench-send-tools --group=$tool_group --dir=$benchmark_results_dir
pbench-postprocess-tools --group=$tool_group --dir=$benchmark_results_dir
else
if [[ ! -d $benchmark_results_dir ]]; then
Expand Down Expand Up @@ -541,7 +550,15 @@ for thread_count in `echo $thread_counts | sed -e s/,/" "/g`; do
done

printf -- "</pre>\n" >>$benchmark_summary_html_file

pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir end

# Stop the tool meisters on each registered local/remote host
pbench-tool-meister-stop
if [[ ${?} != 0 ]]; then
error_log "[${script_name}]: failed to stop the tool meisters."
fi

pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end

rmdir $benchmark_run_dir/.running
36 changes: 27 additions & 9 deletions agent/bench-scripts/pbench-fio
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ pbench_bin="`cd ${script_path}/..; /bin/pwd`"
# source the base script
. "$pbench_bin"/base

benchmark="fio"
export benchmark="fio"
benchmark_rpm=${benchmark}
export benchmark_run_dir=""
# allow unit tests to override
Expand Down Expand Up @@ -56,7 +56,7 @@ max_failures=6 # after N failed attempts to hit below $maxstddevpct, move on to
supported_test_types="read,write,rw,randread,randwrite,randrw"
test_types="read,randread" # default is -non- destructive
install_only="n"
config=""
export config=""
rate_iops=""
block_sizes="4,64,1024"
targets="/tmp/fio"
Expand Down Expand Up @@ -587,6 +587,7 @@ function does_wildcard_exist() {
function fio_run_job() {
local iteration="$1"
local benchmark_results_dir="$2"
local sample_name="$(basename ${benchmark_results_dir})"
local fio_job_file="$3"
local bench_opts="--output-format=json $fio_job_file"
local tgts="$4"
Expand Down Expand Up @@ -663,6 +664,7 @@ function fio_run_job() {
printf "running pre-iteration-script command: $pre_iteration_script\n"
eval "$pre_iteration_script"
fi

pbench-start-tools --group=$tool_group --dir=$benchmark_results_dir

# create a command file and keep it with the results for debugging later, or user can run outside of pbench
Expand All @@ -674,6 +676,7 @@ function fio_run_job() {
popd >/dev/null

pbench-stop-tools --group=$tool_group --dir=$benchmark_results_dir
pbench-send-tools --group=$tool_group --dir=$benchmark_results_dir
pbench-postprocess-tools --group=$tool_group --dir=$benchmark_results_dir

if [ ! -z "$clients" ]; then
Expand Down Expand Up @@ -759,10 +762,10 @@ function fio_run_benchmark() {
for block_size in `echo $block_sizes | sed -e s/,/" "/g`; do
job_num=1
iteration="${count}-${test_type}-${block_size}KiB"
if [ "$job_mode" = "serial" ]; then
dev_short_name="`basename $dev`"
if [[ "$job_mode" = "serial" ]]; then
dev_short_name="$(basename $dev)"
# easier to identify what job used what device when having 1 job per device
iteration="$iteration-${dev_short_name}"
iteration="${iteration}-${dev_short_name}"
fi
record_iteration ${count} ${test_type} ${block_size} ${dev} ${iteration}
iteration_dir=$benchmark_run_dir/$iteration
Expand Down Expand Up @@ -874,17 +877,32 @@ fi
benchmark_iterations="${benchmark_run_dir}/.iterations"
> ${benchmark_iterations}

export benchmark config
pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg

# Start the tool meisters on each registered local/remote host
pbench-tool-meister-start "${tool_group}"
if [[ ${?} != 0 ]]; then
error_log "[${script_name}]: failed to start the tool meisters."
exit 1
fi

pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir beg
# on abnormal exit, make sure that the metadata log exists and is complete.
trap "pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir int" INT QUIT

trap "interrupt" INT QUIT TERM

fio_run_benchmark
echo "$script_path/postprocess/generate-benchmark-summary \"$benchmark\" \"$orig_cmd\" \"$benchmark_run_dir\"" >"$benchmark_run_dir/generate-benchmark-summary.cmd"
chmod +x "$benchmark_run_dir/generate-benchmark-summary.cmd"
$benchmark_run_dir/generate-benchmark-summary.cmd

pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir end
pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end

# Stop the tool meisters on each registered local/remote host
pbench-tool-meister-stop
if [[ ${?} != 0 ]]; then
error_log "[${script_name}]: failed to stop the tool meisters."
fi

pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end

rmdir $benchmark_run_dir/.running
Loading