Skip to content

Canonically determine local vs remote hosts/ips #2542

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Feb 24, 2022
Merged

Canonically determine local vs remote hosts/ips #2542

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
d4b8afc
A class for determining local vs remote hosts
portante Nov 13, 2021
ea25360
Use `LocalHostRemote.is_remote()` in Tool Meister
portante Nov 15, 2021
3fbb363
Replace `pbench-uperf/is_remote` with new `pbench-is-remote` CLI
portante Nov 16, 2021
df186b5
Endow `pbench-fio` with local/remote host knowledge
portante Nov 16, 2021
c3a35eb
Rework the base PR to
dbutenhof Feb 11, 2022
1770014
Blackened
dbutenhof Feb 11, 2022
25a0245
More cleanup and review comments.
dbutenhof Feb 11, 2022
13ee683
Bring local fio cleanup into loop
dbutenhof Feb 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions agent/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ click-scripts = \
pbench-clear-results \
pbench-clear-tools \
pbench-config \
pbench-is-local \
pbench-list-tools \
pbench-list-triggers \
pbench-results-move \
Expand Down
35 changes: 29 additions & 6 deletions agent/bench-scripts/pbench-fio
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -510,20 +510,29 @@ function fio_pre_check() {
local match="${4}"

local rc=0
local do_local_pre_check=0

debug_log "fio_pre_check(targets=\"${targets}\", clients=\"${clients}\", ver=\"${ver}\", match=\"${match}\")"
if [ -n "${clients}" ]; then
local client=""
for client in "${client_names[@]}" ; do
debug_log "Running pre-check on client ${client}"
ssh ${ssh_opts} ${client} pbench-fio --pre-check --targets="${targets}" "${ver}" "${match}" || rc=1
if pbench-is-local ${client}; then
do_local_pre_check=1
else
debug_log "Running pre-check on client ${client}"
ssh ${ssh_opts} ${client} pbench-fio --pre-check --targets="${targets}" "${ver}" "${match}" || rc=1
fi
done
else
do_local_pre_check=1
fi
if [[ ${do_local_pre_check} -eq 1 ]]; then
debug_log "Running pre-check locally"
local_pre_check "${devs}" "${ver}" "${match}"
rc=${?}
fi
if [[ ${rc} -ne 0 ]]; then
error_log "pre-checks failed, exiting"
error_log "pre-check(s) failed, exiting"
exit 1
fi
}
Expand Down Expand Up @@ -588,11 +597,17 @@ function fio_run_job() {
echo "running fio job: $fio_job_file ($(basename $benchmark_results_dir))"

if [ -n "${clients}" ]; then
debug_log "killing any old fio processes, creating directories, and starting new a fio process on the clients"
for client in "${client_names[@]}"; do
ssh ${ssh_opts} ${client} "killall ${benchmark} >/dev/null 2>&1; mkdir -p ${benchmark_results_dir} && cd ${benchmark_results_dir} && screen -dmS fio-server bash -c '${benchmark_bin} --server=,${client_ports[${client}]} > client-result.txt 2> client-result.err'"
if pbench-is-local ${client}; then
debug_log "killing any old local fio processes and starting new a fio process for the local client"
killall ${benchmark} > /dev/null 2>&1
screen -dmS fio-server bash -c '${benchmark_bin} --server=,${client_ports[${client}]} > client-result.txt 2> client-result.err'
else
debug_log "killing any old fio processes, creating directories, and starting new a fio process on remote client ${client}:${client_ports[${client}]}"
ssh ${ssh_opts} ${client} "killall ${benchmark} >/dev/null 2>&1; mkdir -p ${benchmark_results_dir} && cd ${benchmark_results_dir} && screen -dmS fio-server bash -c '${benchmark_bin} --server=,${client_ports[${client}]} > client-result.txt 2> client-result.err'"
fi
done
debug_log "waiting for fio process(server) to start on clients"
debug_log "waiting for fio process(server) to start on the remote clients"
for client in "${client_names[@]}"; do
timeout --kill-after=1 60 bash -c 'until printf "" 2>>/dev/null >>/dev/tcp/$0/$1; do sleep 1; done' "${client}" "${client_ports[${client}]}"
if [[ $? -eq 124 ]]; then
Expand Down Expand Up @@ -897,6 +912,7 @@ if [ "$postprocess_only" = "n" ]; then
fi

if [[ -n "${clients}" ]] ; then
let local_client_count=0
for client in ${clients//,/ } ; do
if [[ $client = *':'* ]] ; then
client_name=${client%%:*}
Expand All @@ -908,6 +924,13 @@ if [ "$postprocess_only" = "n" ]; then
fio_server_port=$((fio_server_port+1))
fi
fi
if pbench-is-local ${client}; then
(( local_client_count++ ))
if [[ ${local_client_count} -gt 1 ]]; then
error_log "${script_name}: more than one client refers to the local host"
exit 1
fi
fi
client_names+=("${client_name}")
client_ports["${client_name}"]=${client_port}
done
Expand Down
41 changes: 18 additions & 23 deletions agent/bench-scripts/pbench-uperf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,31 +176,26 @@ function gen_xml {
echo "</profile>"
}

function is_remote {
# Returns 0 if the argument is considered "remote", and 1 if not.
[[ "${1}" != "127.0.0.1" && "${1}" != "localhost" ]]
}

function stop_server {
local server="${1}"
local server_port="${2}"
local message="${3}"

local ss_output
if is_remote ${server}; then
ss_output=$(ssh ${ssh_opts} ${server} ss -tlnp --vsock)
else
if pbench-is-local ${server}; then
ss_output=$(ss -tlnp --vsock)
else
ss_output=$(ssh ${ssh_opts} ${server} ss -tlnp --vsock)
fi
local uperf_pids=$(sed -En -e "/:${server_port} / s/.*pid=([^,]+),.*/\1/p" <<< "${ss_output}" | sort -u)
if [[ -n "${uperf_pids}" ]]; then
if [[ ${message} -eq 1 ]]; then
echo "found uperf pid(s) ${uperf_pids} on ${server}:${server_port}, killing"
fi
if is_remote ${server}; then
ssh ${ssh_opts} ${server} "for pid in ${uperf_pids}; do kill \${pid}; done"
else
if pbench-is-local ${server}; then
for pid in ${uperf_pids}; do kill ${pid}; done
else
ssh ${ssh_opts} ${server} "for pid in ${uperf_pids}; do kill \${pid}; done"
fi
fi
}
Expand Down Expand Up @@ -412,7 +407,7 @@ if [[ "${postprocess_only}" != "y" ]]; then
let c_cnt=0
for client in ${clients//,/ }; do
(( c_cnt++ ))
if ! is_remote "${client}"; then
if pbench-is-local "${client}"; then
check_local=1
fi
done
Expand All @@ -423,7 +418,7 @@ if [[ "${postprocess_only}" != "y" ]]; then
let s_cnt=0
for server in ${servers//,/ }; do
(( s_cnt++ ))
if ! is_remote "${server}"; then
if pbench-is-local "${server}"; then
check_local=1
fi
done
Expand Down Expand Up @@ -468,7 +463,7 @@ if [[ "${postprocess_only}" != "y" ]]; then
let err_cnt=0
err_clients=""
for client in ${clients//,/ }; do
if is_remote "${client}"; then
if ! pbench-is-local "${client}"; then
debug_log "performing uperf pre-checks on client ${client}"
ssh ${ssh_opts} ${client} pbench-uperf --pre-check "${ver}" "${match}"
if [ ${?} -ne 0 ]; then
Expand All @@ -479,7 +474,7 @@ if [[ "${postprocess_only}" != "y" ]]; then
done
err_servers=""
for server in ${servers//,/ }; do
if is_remote "${server}"; then
if ! pbench-is-local "${server}"; then
debug_log "performing uperf pre-checks on server ${server}"
ssh ${ssh_opts} ${server} pbench-uperf --pre-check "${ver}" "${match}"
if [ ${?} -ne 0 ]; then
Expand Down Expand Up @@ -662,25 +657,25 @@ for protocol in ${protocols//,/ }; do
chmod +x $benchmark_client_cmd_file

# prepare test files and dirs if using remote clients
if is_remote "${client}"; then
if ! pbench-is-local "${client}"; then
ssh $ssh_opts $client mkdir -p $benchmark_results_dir
scp $scp_opts $xml_file $client:$xml_file >/dev/null
scp $scp_opts $benchmark_client_cmd_file $client:$benchmark_client_cmd_file >/dev/null
fi

# prepare test files and dirs on servers
if is_remote "${server}"; then
if ! pbench-is-local "${server}"; then
ssh $ssh_opts $server mkdir -p $benchmark_results_dir
scp $scp_opts $benchmark_server_cmd_file $server:$benchmark_server_cmd_file >/dev/null
fi

# start the uperf server(s)
stop_server $server $server_port 1

if is_remote "${server}"; then
ssh $ssh_opts $server "screen -dmS uperf-server $benchmark_server_cmd_file"
else
if pbench-is-local "${server}"; then
screen -dmS uperf-server $benchmark_server_cmd_file
else
ssh $ssh_opts $server "screen -dmS uperf-server $benchmark_server_cmd_file"
fi

((server_nr++))
Expand Down Expand Up @@ -718,7 +713,7 @@ for protocol in ${protocols//,/ }; do
debug_log "client[$client]${client_nodeinfo}protocol[$protocol]test[$test_type]instances[$instance]size[$message_size] <-> server[$server]${server_nodeinfo}"
echo "client[$client]${client_nodeinfo}protocol[$protocol]test[$test_type]instances[$instance]size[$message_size] <-> server[$server]${server_nodeinfo}"

if ! is_remote "${client}"; then
if pbench-is-local "${client}"; then
# using local client
debug_log "screen -dmS uperf-client $benchmark_client_cmd_file"
screen -dmS uperf-client $benchmark_client_cmd_file
Expand All @@ -744,11 +739,11 @@ for protocol in ${protocols//,/ }; do

stop_server $server $server_port 0

if is_remote "${server}"; then
if ! pbench-is-local "${server}"; then
server_log="$benchmark_results_dir/$uperf_identifier--server.log"
scp $scp_opts $server:$server_log $server_log >/dev/null
fi
if is_remote "${client}"; then
if ! pbench-is-local "${client}"; then
result_file="$benchmark_results_dir/$uperf_identifier--client_output.txt"
scp $scp_opts $client:$result_file $result_file >/dev/null
fi
Expand Down
1 change: 1 addition & 0 deletions agent/bench-scripts/tests/pbench-fio/test-04.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -514,6 +514,7 @@ fio job complete
--- pbench tree state
+++ pbench.log file contents
[debug][1900-01-01T00:00:00.000000] fio_pre_check(targets="/tmp/fio", clients="", ver="3.21", match="gte")
[debug][1900-01-01T00:00:00.000000] Running pre-check locally
[debug][1900-01-01T00:00:00.000000] local_pre_check(devs="", ver="3.21", match="gte")
[debug][1900-01-01T00:00:00.000000] found fio-3.42
[debug][1900-01-01T00:00:00.000000] fio: Going to run [/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/bm --output-format=json /var/tmp/pbench-test-bench/pbench-agent/fio_test-04_1900.01.01T00.00.00/1-rw-4KiB/fio.job ]
Expand Down
15 changes: 9 additions & 6 deletions agent/bench-scripts/tests/pbench-fio/test-05.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,24 +142,27 @@ fio job complete
[debug][1900-01-01T00:00:00.000000] fio_pre_check(targets="/tmp/fio", clients="foo,bar", ver="3.21", match="gte")
[debug][1900-01-01T00:00:00.000000] Running pre-check on client foo
[debug][1900-01-01T00:00:00.000000] Running pre-check on client bar
[debug][1900-01-01T00:00:00.000000] killing any old fio processes, creating directories, and starting new a fio process on the clients
[debug][1900-01-01T00:00:00.000000] waiting for fio process(server) to start on clients
[debug][1900-01-01T00:00:00.000000] killing any old fio processes, creating directories, and starting new a fio process on remote client foo:8765
[debug][1900-01-01T00:00:00.000000] killing any old fio processes, creating directories, and starting new a fio process on remote client bar:8765
[debug][1900-01-01T00:00:00.000000] waiting for fio process(server) to start on the remote clients
[debug][1900-01-01T00:00:00.000000] fio: Going to run [/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/bm --output-format=json /var/tmp/pbench-test-bench/pbench-agent/fio_test-05_1900.01.01T00.00.00/1-rw-4KiB/fio.job --client=/var/tmp/pbench-test-bench/pbench-agent/fio_test-05_1900.01.01T00.00.00/fio-client.file --max-jobs=2]
[warn][1900-01-01T00:00:00.000000] log_hist_msec specified in job file but failed to find any log files (.../clients/foo/fio_clat_hist.*.log*) to process for histograms
[warn][1900-01-01T00:00:00.000000] log_hist_msec specified in job file but failed to find any log files (.../clients/bar/fio_clat_hist.*.log*) to process for histograms
[warn][1900-01-01T00:00:00.000000] log_hist_msec specified in job file but failed to find any log files (.../clients/*/fio_clat_hist.*.log*) to process for histograms
[debug][1900-01-01T00:00:00.000000] post-processing tool data
[debug][1900-01-01T00:00:00.000000] post-processing fio result
[debug][1900-01-01T00:00:00.000000] killing any old fio processes, creating directories, and starting new a fio process on the clients
[debug][1900-01-01T00:00:00.000000] waiting for fio process(server) to start on clients
[debug][1900-01-01T00:00:00.000000] killing any old fio processes, creating directories, and starting new a fio process on remote client foo:8765
[debug][1900-01-01T00:00:00.000000] killing any old fio processes, creating directories, and starting new a fio process on remote client bar:8765
[debug][1900-01-01T00:00:00.000000] waiting for fio process(server) to start on the remote clients
[debug][1900-01-01T00:00:00.000000] fio: Going to run [/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/bm --output-format=json /var/tmp/pbench-test-bench/pbench-agent/fio_test-05_1900.01.01T00.00.00/2-rw-64KiB/fio.job --client=/var/tmp/pbench-test-bench/pbench-agent/fio_test-05_1900.01.01T00.00.00/fio-client.file --max-jobs=2]
[warn][1900-01-01T00:00:00.000000] log_hist_msec specified in job file but failed to find any log files (.../clients/foo/fio_clat_hist.*.log*) to process for histograms
[warn][1900-01-01T00:00:00.000000] log_hist_msec specified in job file but failed to find any log files (.../clients/bar/fio_clat_hist.*.log*) to process for histograms
[warn][1900-01-01T00:00:00.000000] log_hist_msec specified in job file but failed to find any log files (.../clients/*/fio_clat_hist.*.log*) to process for histograms
[debug][1900-01-01T00:00:00.000000] post-processing tool data
[debug][1900-01-01T00:00:00.000000] post-processing fio result
[debug][1900-01-01T00:00:00.000000] killing any old fio processes, creating directories, and starting new a fio process on the clients
[debug][1900-01-01T00:00:00.000000] waiting for fio process(server) to start on clients
[debug][1900-01-01T00:00:00.000000] killing any old fio processes, creating directories, and starting new a fio process on remote client foo:8765
[debug][1900-01-01T00:00:00.000000] killing any old fio processes, creating directories, and starting new a fio process on remote client bar:8765
[debug][1900-01-01T00:00:00.000000] waiting for fio process(server) to start on the remote clients
[debug][1900-01-01T00:00:00.000000] fio: Going to run [/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/bm --output-format=json /var/tmp/pbench-test-bench/pbench-agent/fio_test-05_1900.01.01T00.00.00/3-rw-1024KiB/fio.job --client=/var/tmp/pbench-test-bench/pbench-agent/fio_test-05_1900.01.01T00.00.00/fio-client.file --max-jobs=2]
[warn][1900-01-01T00:00:00.000000] log_hist_msec specified in job file but failed to find any log files (.../clients/foo/fio_clat_hist.*.log*) to process for histograms
[warn][1900-01-01T00:00:00.000000] log_hist_msec specified in job file but failed to find any log files (.../clients/bar/fio_clat_hist.*.log*) to process for histograms
Expand Down
18 changes: 12 additions & 6 deletions agent/bench-scripts/tests/pbench-fio/test-06.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,26 +149,32 @@ fio job complete
[debug][1900-01-01T00:00:00.000000] Running pre-check on client foo
[debug][1900-01-01T00:00:00.000000] Running pre-check on client bar
[debug][1900-01-01T00:00:00.000000] Running pre-check on client baz
[debug][1900-01-01T00:00:00.000000] killing any old fio processes, creating directories, and starting new a fio process on the clients
[debug][1900-01-01T00:00:00.000000] waiting for fio process(server) to start on clients
[debug][1900-01-01T00:00:00.000000] killing any old fio processes, creating directories, and starting new a fio process on remote client foo:8765
[debug][1900-01-01T00:00:00.000000] killing any old fio processes, creating directories, and starting new a fio process on remote client bar:8765
[debug][1900-01-01T00:00:00.000000] killing any old fio processes, creating directories, and starting new a fio process on remote client baz:8765
[debug][1900-01-01T00:00:00.000000] waiting for fio process(server) to start on the remote clients
[debug][1900-01-01T00:00:00.000000] fio: Going to run [/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/bm --output-format=json /var/tmp/pbench-test-bench/pbench-agent/fio_test-06_1900.01.01T00.00.00/1-rw-4KiB/fio.job --client=/var/tmp/pbench-test-bench/pbench-agent/fio_test-06_1900.01.01T00.00.00/fio-client.file --max-jobs=3]
[warn][1900-01-01T00:00:00.000000] log_hist_msec specified in job file but failed to find any log files (.../clients/foo/fio_clat_hist.*.log*) to process for histograms
[warn][1900-01-01T00:00:00.000000] log_hist_msec specified in job file but failed to find any log files (.../clients/bar/fio_clat_hist.*.log*) to process for histograms
[warn][1900-01-01T00:00:00.000000] log_hist_msec specified in job file but failed to find any log files (.../clients/baz/fio_clat_hist.*.log*) to process for histograms
[warn][1900-01-01T00:00:00.000000] log_hist_msec specified in job file but failed to find any log files (.../clients/*/fio_clat_hist.*.log*) to process for histograms
[debug][1900-01-01T00:00:00.000000] post-processing tool data
[debug][1900-01-01T00:00:00.000000] post-processing fio result
[debug][1900-01-01T00:00:00.000000] killing any old fio processes, creating directories, and starting new a fio process on the clients
[debug][1900-01-01T00:00:00.000000] waiting for fio process(server) to start on clients
[debug][1900-01-01T00:00:00.000000] killing any old fio processes, creating directories, and starting new a fio process on remote client foo:8765
[debug][1900-01-01T00:00:00.000000] killing any old fio processes, creating directories, and starting new a fio process on remote client bar:8765
[debug][1900-01-01T00:00:00.000000] killing any old fio processes, creating directories, and starting new a fio process on remote client baz:8765
[debug][1900-01-01T00:00:00.000000] waiting for fio process(server) to start on the remote clients
[debug][1900-01-01T00:00:00.000000] fio: Going to run [/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/bm --output-format=json /var/tmp/pbench-test-bench/pbench-agent/fio_test-06_1900.01.01T00.00.00/2-rw-64KiB/fio.job --client=/var/tmp/pbench-test-bench/pbench-agent/fio_test-06_1900.01.01T00.00.00/fio-client.file --max-jobs=3]
[warn][1900-01-01T00:00:00.000000] log_hist_msec specified in job file but failed to find any log files (.../clients/foo/fio_clat_hist.*.log*) to process for histograms
[warn][1900-01-01T00:00:00.000000] log_hist_msec specified in job file but failed to find any log files (.../clients/bar/fio_clat_hist.*.log*) to process for histograms
[warn][1900-01-01T00:00:00.000000] log_hist_msec specified in job file but failed to find any log files (.../clients/baz/fio_clat_hist.*.log*) to process for histograms
[warn][1900-01-01T00:00:00.000000] log_hist_msec specified in job file but failed to find any log files (.../clients/*/fio_clat_hist.*.log*) to process for histograms
[debug][1900-01-01T00:00:00.000000] post-processing tool data
[debug][1900-01-01T00:00:00.000000] post-processing fio result
[debug][1900-01-01T00:00:00.000000] killing any old fio processes, creating directories, and starting new a fio process on the clients
[debug][1900-01-01T00:00:00.000000] waiting for fio process(server) to start on clients
[debug][1900-01-01T00:00:00.000000] killing any old fio processes, creating directories, and starting new a fio process on remote client foo:8765
[debug][1900-01-01T00:00:00.000000] killing any old fio processes, creating directories, and starting new a fio process on remote client bar:8765
[debug][1900-01-01T00:00:00.000000] killing any old fio processes, creating directories, and starting new a fio process on remote client baz:8765
[debug][1900-01-01T00:00:00.000000] waiting for fio process(server) to start on the remote clients
[debug][1900-01-01T00:00:00.000000] fio: Going to run [/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/bm --output-format=json /var/tmp/pbench-test-bench/pbench-agent/fio_test-06_1900.01.01T00.00.00/3-rw-1024KiB/fio.job --client=/var/tmp/pbench-test-bench/pbench-agent/fio_test-06_1900.01.01T00.00.00/fio-client.file --max-jobs=3]
[warn][1900-01-01T00:00:00.000000] log_hist_msec specified in job file but failed to find any log files (.../clients/foo/fio_clat_hist.*.log*) to process for histograms
[warn][1900-01-01T00:00:00.000000] log_hist_msec specified in job file but failed to find any log files (.../clients/bar/fio_clat_hist.*.log*) to process for histograms
Expand Down
Loading