From af60170db3617f7e373db7bbf74082ea39db62d2 Mon Sep 17 00:00:00 2001 From: Peter Portante Date: Mon, 16 Nov 2020 10:47:08 -0500 Subject: [PATCH 1/3] Remove unnecessary explicit 'object' sub-classing --- agent/util-scripts/pbench-tool-meister-start | 2 +- lib/pbench/agent/tool_meister.py | 4 ++-- server/bin/pbench-server-prep-shim-002.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/agent/util-scripts/pbench-tool-meister-start b/agent/util-scripts/pbench-tool-meister-start index c269ab2b21..1ec1b6b3f5 100755 --- a/agent/util-scripts/pbench-tool-meister-start +++ b/agent/util-scripts/pbench-tool-meister-start @@ -60,7 +60,7 @@ channel = "tool-meister-chan" REDIS_MAX_WAIT = 60 -class ToolGroup(object): +class ToolGroup: tg_prefix = "tools-v1" def __init__(self, group): diff --git a/lib/pbench/agent/tool_meister.py b/lib/pbench/agent/tool_meister.py index 858fae637d..8369dbe9fe 100644 --- a/lib/pbench/agent/tool_meister.py +++ b/lib/pbench/agent/tool_meister.py @@ -155,7 +155,7 @@ def stop(self): return 0 -class Tool(object): +class Tool: """Encapsulates all the state needed to manage a tool running as a background process. @@ -280,7 +280,7 @@ class ToolMeisterError(Exception): pass -class ToolMeister(object): +class ToolMeister: """Encapsulate tool life-cycle The goal of this class is to make sure all necessary state and behaviors diff --git a/server/bin/pbench-server-prep-shim-002.py b/server/bin/pbench-server-prep-shim-002.py index 84111270a4..25c394630c 100755 --- a/server/bin/pbench-server-prep-shim-002.py +++ b/server/bin/pbench-server-prep-shim-002.py @@ -27,7 +27,7 @@ _NAME_ = "pbench-server-prep-shim-002" -class Results(object): +class Results: def __init__( self, nstatus="", ntotal=0, ntbs=0, nquarantined=0, ndups=0, nerrs=0, ): From 689cb498a0aae52add203fa0ce602a0e15bfaaf7 Mon Sep 17 00:00:00 2001 From: Peter Portante Date: Mon, 16 Nov 2020 10:48:56 -0500 Subject: [PATCH 2/3] Stop using f-strings with logging --- lib/pbench/agent/toolmetadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pbench/agent/toolmetadata.py b/lib/pbench/agent/toolmetadata.py index 6f32c6ead9..78d6473fa1 100644 --- a/lib/pbench/agent/toolmetadata.py +++ b/lib/pbench/agent/toolmetadata.py @@ -69,7 +69,7 @@ def __dataCheck(self): if not self.data: self.data == self.__getInitialData() if not self.data: - self.logger.error(f"Unable to access data through {self.mode}") + self.logger.error("Unable to access data through %s", self.mode) return False return True From c77461c96e005876bf3f7a94a180aa132b6aa309 Mon Sep 17 00:00:00 2001 From: Peter Portante Date: Thu, 22 Oct 2020 18:10:54 -0400 Subject: [PATCH 3/3] Add support for sysinfo via Tool Meister The `pbench-collect-sysinfo` CLI interface has been modified to send a message to the Tool Meisters to have them collect all the requested system configuration information. Much like tool data is collected and sent back to the Tool Data Sink from each Tool Meister, the system information is collected by each Tool Meister and sent back to the Tool Data sink to be stored in the requested location. This means that `pbench-collect-sysinfo` no longer uses `ssh` to collect and copy the system information from remote hosts. As it is for tools running on the local host, the local Tool Meister write the collected data into the local volume directly without sending it through the Tool Data Sink. This change allows us to delete the `pbench-remote-sysinfo-dump` trampoline interface since the Tool Meister takes are of remote operations. Each benchmark-script has been modified to invoke the `pbench-collect-sysinfo` following the `pbench-tool-meister-start` call, and before the `pbench-tool-meister-stop` call. We fixed a small bug for `pbench-trafficgen` which was overwriting the "beginning" (`.../sysinfo/beg`) collected system information when collecting the "ending" (`.../sysinfo/end`) information. --- agent/Makefile | 3 +- agent/bench-scripts/pbench-cyclictest | 7 +- agent/bench-scripts/pbench-dbench | 8 +- agent/bench-scripts/pbench-fio | 8 +- agent/bench-scripts/pbench-iozone | 8 +- agent/bench-scripts/pbench-linpack | 8 +- agent/bench-scripts/pbench-migrate | 8 +- agent/bench-scripts/pbench-netperf | 8 +- agent/bench-scripts/pbench-specjbb2005 | 10 +- agent/bench-scripts/pbench-trafficgen | 8 +- agent/bench-scripts/pbench-uperf | 8 +- agent/bench-scripts/pbench-user-benchmark | 8 +- .../tests/pbench-trafficgen/test-22.txt | 2 +- .../tests/pbench-trafficgen/test-39.txt | 2 +- .../tests/pbench-trafficgen/test-40.txt | 2 +- agent/rpm/pbench-agent.spec.j2 | 2 +- .../gold/pbench-collect-sysinfo/test-55.txt | 3 +- .../gold/test-client-tool-meister/test-53.txt | 21 ++ .../gold/test-client-tool-meister/test-56.txt | 93 ++++- .../gold/test-client-tool-meister/test-57.txt | 93 ++++- agent/util-scripts/pbench-collect-sysinfo | 62 +--- agent/util-scripts/pbench-remote-sysinfo-dump | 35 -- agent/util-scripts/pbench-tool-meister-client | 8 +- agent/util-scripts/pbench-tool-meister-stop | 2 +- .../util-scripts/test-bin/pbench-sysinfo-dump | 28 +- agent/util-scripts/test-bin/ssh | 2 +- .../test-bin/test-client-tool-meister | 14 + agent/util-scripts/unittests | 3 +- lib/pbench/agent/tool_data_sink.py | 82 +++-- lib/pbench/agent/tool_meister.py | 324 ++++++++++++------ 30 files changed, 552 insertions(+), 318 deletions(-) delete mode 100755 agent/util-scripts/pbench-remote-sysinfo-dump mode change 120000 => 100755 agent/util-scripts/test-bin/pbench-sysinfo-dump diff --git a/agent/Makefile b/agent/Makefile index 076fc2e72f..ba72c29290 100644 --- a/agent/Makefile +++ b/agent/Makefile @@ -50,7 +50,6 @@ util-scripts = \ pbench-register-tool \ pbench-register-tool-set \ pbench-register-tool-trigger \ - pbench-remote-sysinfo-dump \ pbench-send-tools \ pbench-start-tools \ pbench-stop-tools \ @@ -136,7 +135,7 @@ tool-scripts = \ prometheus-metrics \ qemu-migrate \ rabbit \ - README \ + README.md \ sar \ strace \ sysfs \ diff --git a/agent/bench-scripts/pbench-cyclictest b/agent/bench-scripts/pbench-cyclictest index b8d8992999..bc3f457254 100755 --- a/agent/bench-scripts/pbench-cyclictest +++ b/agent/bench-scripts/pbench-cyclictest @@ -141,8 +141,6 @@ printf "\n" >>$benchmark_summary_txt_file printf "
\n# these results generated with:\n# uperf %s\n\n" "$scrtip_name $orig_cmd" >$benchmark_summary_html_file
 printf "\n" >>$benchmark_summary_html_file
 
-pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg
-
 # Start the tool meisters on each registered local/remote host
 pbench-tool-meister-start "${tool_group}"
 if [[ ${?} != 0 ]]; then
@@ -150,6 +148,8 @@ if [[ ${?} != 0 ]]; then
 	exit 1
 fi
 
+pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg
+
 pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir beg
 
 trap "interrupt" INT QUIT TERM
@@ -198,13 +198,14 @@ if [ $stress ]; then
 fi
 pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir end
 
+pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end
+
 # Stop the tool meisters on each registered local/remote host
 pbench-tool-meister-stop
 if [[ ${?} != 0 ]]; then
 	error_log "[${script_name}]: failed to stop the tool meisters."
 fi
 
-pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end
 cat $benchmark_summary_txt_file
 
 # signal to move/copy-results that this run is done.
diff --git a/agent/bench-scripts/pbench-dbench b/agent/bench-scripts/pbench-dbench
index 80d9e2f5b1..371cefcab8 100755
--- a/agent/bench-scripts/pbench-dbench
+++ b/agent/bench-scripts/pbench-dbench
@@ -327,8 +327,6 @@ let total_iterations=count
 count=1
 max_key_length=0
 
-pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg
-
 # Start the tool meisters on each registered local/remote host
 pbench-tool-meister-start "${tool_group}"
 if [[ ${?} != 0 ]]; then
@@ -336,6 +334,8 @@ if [[ ${?} != 0 ]]; then
 	exit 1
 fi
 
+pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg
+
 pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir beg
 
 trap "interrupt" INT QUIT TERM
@@ -555,12 +555,12 @@ printf -- "
\n" >>$benchmark_summary_html_file pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir end +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end + # Stop the tool meisters on each registered local/remote host pbench-tool-meister-stop if [[ ${?} != 0 ]]; then error_log "[${script_name}]: failed to stop the tool meisters." fi -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end - rmdir $benchmark_run_dir/.running diff --git a/agent/bench-scripts/pbench-fio b/agent/bench-scripts/pbench-fio index ea007b5572..a0f7c01778 100755 --- a/agent/bench-scripts/pbench-fio +++ b/agent/bench-scripts/pbench-fio @@ -877,8 +877,6 @@ fi benchmark_iterations="${benchmark_run_dir}/.iterations" > ${benchmark_iterations} -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg - # Start the tool meisters on each registered local/remote host pbench-tool-meister-start "${tool_group}" if [[ ${?} != 0 ]]; then @@ -886,6 +884,8 @@ if [[ ${?} != 0 ]]; then exit 1 fi +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg + pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir beg trap "interrupt" INT QUIT TERM @@ -899,12 +899,12 @@ $benchmark_run_dir/generate-benchmark-summary.cmd pbench-end-tools --group=$tool_group --dir=$benchmark_run_dir pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir end +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end + # Stop the tool meisters on each registered local/remote host pbench-tool-meister-stop if [[ ${?} != 0 ]]; then error_log "[${script_name}]: failed to stop the tool meisters." fi -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end - rmdir $benchmark_run_dir/.running diff --git a/agent/bench-scripts/pbench-iozone b/agent/bench-scripts/pbench-iozone index 96a4b57cd4..571b094ae4 100755 --- a/agent/bench-scripts/pbench-iozone +++ b/agent/bench-scripts/pbench-iozone @@ -193,8 +193,6 @@ mkdir -p $benchmark_run_dir/.running # now that the benchmark_run_dir directory exists, we can initialize the iterations file > ${benchmark_iterations} -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg - # Start the tool meisters on each registered local/remote host pbench-tool-meister-start "${tool_group}" if [[ ${?} != 0 ]]; then @@ -202,6 +200,8 @@ if [[ ${?} != 0 ]]; then exit 1 fi +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg + pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir beg trap "interrupt" INT QUIT TERM @@ -232,12 +232,12 @@ ln -s ${sample} $benchmark_run_dir/$iteration/reference-result pbench-end-tools --group=$tool_group --dir=$benchmark_run_dir pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir end +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end + # Stop the tool meisters on each registered local/remote host pbench-tool-meister-stop if [[ ${?} != 0 ]]; then error_log "[${script_name}]: failed to stop the tool meisters." fi -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end - rmdir $benchmark_run_dir/.running diff --git a/agent/bench-scripts/pbench-linpack b/agent/bench-scripts/pbench-linpack index 4ac84801fb..16bd23b302 100755 --- a/agent/bench-scripts/pbench-linpack +++ b/agent/bench-scripts/pbench-linpack @@ -135,8 +135,6 @@ benchmark_results_dir="$benchmark_run_dir/$iteration/${sample}" result_file=$benchmark_results_dir/result.txt mkdir -p $benchmark_results_dir -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg - # Start the tool meisters on each registered local/remote host pbench-tool-meister-start "${tool_group}" if [[ ${?} != 0 ]]; then @@ -144,6 +142,8 @@ if [[ ${?} != 0 ]]; then exit 1 fi +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg + pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir beg trap "interrupt" INT QUIT TERM @@ -161,12 +161,12 @@ ln -s ${sample} $benchmark_run_dir/$iteration/reference-result pbench-end-tools --group=$tool_group --dir=$benchmark_run_dir pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir end +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end + # Stop the tool meisters on each registered local/remote host pbench-tool-meister-stop if [[ ${?} != 0 ]]; then error_log "[${script_name}]: failed to stop the tool meisters." fi -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end - rmdir $benchmark_run_dir/.running diff --git a/agent/bench-scripts/pbench-migrate b/agent/bench-scripts/pbench-migrate index 0ccde7d008..2a8afd3047 100755 --- a/agent/bench-scripts/pbench-migrate +++ b/agent/bench-scripts/pbench-migrate @@ -348,8 +348,6 @@ if [ $loops -eq 0 ]; then loops=1 fi -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg - # Start the tool meisters on each registered local/remote host pbench-tool-meister-start "${tool_group}" if [[ ${?} != 0 ]]; then @@ -357,6 +355,8 @@ if [[ ${?} != 0 ]]; then exit 1 fi +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg + pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir beg trap "interrupt" INT QUIT TERM @@ -418,12 +418,12 @@ wait pbench-end-tools --group=$tool_group --dir=$benchmark_run_dir pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir end +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end + # Stop the tool meisters on each registered local/remote host pbench-tool-meister-stop if [[ ${?} != 0 ]]; then error_log "[${script_name}]: failed to stop the tool meisters." fi -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end - rmdir $benchmark_run_dir/.running diff --git a/agent/bench-scripts/pbench-netperf b/agent/bench-scripts/pbench-netperf index ede408bcd8..d13e0ff58c 100755 --- a/agent/bench-scripts/pbench-netperf +++ b/agent/bench-scripts/pbench-netperf @@ -439,8 +439,6 @@ printf "\n" >>$benchmark_summary_html_file # disable firewall on local system systemctl stop firewalld -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg - # Start the tool meisters on each registered local/remote host pbench-tool-meister-start "${tool_group}" if [[ ${?} != 0 ]]; then @@ -448,6 +446,8 @@ if [[ ${?} != 0 ]]; then exit 1 fi +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg + pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir beg trap "interrupt" INT QUIT TERM @@ -641,12 +641,12 @@ printf -- "\n" >>$benchmark_summary_html_file pbench-end-tools --group=$tool_group --dir=$benchmark_run_dir pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir end +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end + # Stop the tool meisters on each registered local/remote host pbench-tool-meister-stop if [[ ${?} != 0 ]]; then error_log "[${script_name}]: failed to stop the tool meisters." fi -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end - rmdir $benchmark_run_dir/.running diff --git a/agent/bench-scripts/pbench-specjbb2005 b/agent/bench-scripts/pbench-specjbb2005 index 9137dd6a46..a223747b67 100755 --- a/agent/bench-scripts/pbench-specjbb2005 +++ b/agent/bench-scripts/pbench-specjbb2005 @@ -267,9 +267,6 @@ benchmark_iterations="${benchmark_run_dir}/.iterations" pbench-register-tool-trigger --group=$tool_group --start-trigger="Timing Measurement began" --stop-trigger="Timing Measurement ended" -# Collect system information before we begin, and record the beginning. -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg - # Start the tool meisters on each registered local/remote host pbench-tool-meister-start "${tool_group}" if [[ ${?} != 0 ]]; then @@ -277,6 +274,9 @@ if [[ ${?} != 0 ]]; then exit 1 fi +# Collect system information before we begin, and record the beginning. +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg + pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir beg trap "interrupt" INT QUIT TERM @@ -392,12 +392,12 @@ popd > /dev/null 2>&1 pbench-end-tools --group=$tool_group --dir=$benchmark_run_dir pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir end +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end + # Stop the tool meisters on each registered local/remote host pbench-tool-meister-stop if [[ ${?} != 0 ]]; then error_log "[${script_name}]: failed to stop the tool meisters." fi -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end - rmdir $benchmark_run_dir/.running diff --git a/agent/bench-scripts/pbench-trafficgen b/agent/bench-scripts/pbench-trafficgen index 5932e95e53..ff135a0d26 100755 --- a/agent/bench-scripts/pbench-trafficgen +++ b/agent/bench-scripts/pbench-trafficgen @@ -997,8 +997,6 @@ echo "Total number of benchmark iterations: $total_iterations" export benchmark=${benchmark_name} -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg - # Start the tool meisters on each registered local/remote host pbench-tool-meister-start "${tool_group}" if [[ ${?} != 0 ]]; then @@ -1006,6 +1004,8 @@ if [[ ${?} != 0 ]]; then exit 1 fi +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg + pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir beg trap "interrupt" INT QUIT TERM @@ -1308,12 +1308,12 @@ fi pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir end +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end + # Stop the tool meisters on each registered local/remote host pbench-tool-meister-stop if [[ ${?} != 0 ]]; then error_log "[${script_name}]: failed to stop the tool meisters." fi -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg - rmdir $benchmark_run_dir/.running diff --git a/agent/bench-scripts/pbench-uperf b/agent/bench-scripts/pbench-uperf index f19a15f160..4bbdf0b3da 100755 --- a/agent/bench-scripts/pbench-uperf +++ b/agent/bench-scripts/pbench-uperf @@ -499,8 +499,6 @@ for protocol in `echo $protocols | sed -e s/,/" "/g`; do done done -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg - # Start the tool meisters on each registered local/remote host pbench-tool-meister-start "${tool_group}" if [[ ${?} != 0 ]]; then @@ -508,6 +506,8 @@ if [[ ${?} != 0 ]]; then exit 1 fi +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg + pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir beg trap "interrupt" INT QUIT TERM @@ -746,12 +746,12 @@ $benchmark_run_dir/generate-benchmark-summary.cmd pbench-end-tools --group=$tool_group --dir=$benchmark_run_dir pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir end +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end + # Stop the tool meisters on each registered local/remote host pbench-tool-meister-stop if [[ ${?} != 0 ]]; then error_log "[${script_name}]: failed to stop the tool meisters." fi -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end - rmdir $benchmark_run_dir/.running diff --git a/agent/bench-scripts/pbench-user-benchmark b/agent/bench-scripts/pbench-user-benchmark index 4f6c3d503e..56ea16041c 100755 --- a/agent/bench-scripts/pbench-user-benchmark +++ b/agent/bench-scripts/pbench-user-benchmark @@ -355,17 +355,17 @@ for rdir in ${result_dirs}; do ln -s "$(basename "${rdir}")" "$(dirname "${rdir}")/reference-result" done +${script_path}/postprocess/user-benchmark-wrapper "${benchmark_run_dir}" "${total_duration}" + pbench-metadata-log --group="${tool_group}" --dir="${benchmark_run_dir}" end +pbench-collect-sysinfo --group="${tool_group}" --dir="${benchmark_run_dir}" --sysinfo="${sysinfo}" end + # Stop the tool meisters on each registered local/remote host pbench-tool-meister-stop if [[ ${?} != 0 ]]; then error_log "[${script_name}]: failed to stop the tool meisters." fi -${script_path}/postprocess/user-benchmark-wrapper "${benchmark_run_dir}" "${total_duration}" - -pbench-collect-sysinfo --group="${tool_group}" --dir="${benchmark_run_dir}" --sysinfo="${sysinfo}" end - rmdir ${benchmark_run_dir}/.running exit 0 diff --git a/agent/bench-scripts/tests/pbench-trafficgen/test-22.txt b/agent/bench-scripts/tests/pbench-trafficgen/test-22.txt index b5559fd83b..539c3389e5 100644 --- a/agent/bench-scripts/tests/pbench-trafficgen/test-22.txt +++ b/agent/bench-scripts/tests/pbench-trafficgen/test-22.txt @@ -32172,7 +32172,7 @@ Iteration 3-revunidirectional-74B-1000flows-0.0pct_drop complete (3 of 3), with --- pbench.log file contents +++ test-execution.log file contents /var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-collect-sysinfo --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent/trafficgen_test-22_1970.01.01T00.00.00 --sysinfo=none beg -/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-collect-sysinfo --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent/trafficgen_test-22_1970.01.01T00.00.00 --sysinfo=none beg +/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-collect-sysinfo --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent/trafficgen_test-22_1970.01.01T00.00.00 --sysinfo=none end /var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-collect-sysinfo --sysinfo=none --check /var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-metadata-log --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent/trafficgen_test-22_1970.01.01T00.00.00 beg /var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-metadata-log --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent/trafficgen_test-22_1970.01.01T00.00.00 end diff --git a/agent/bench-scripts/tests/pbench-trafficgen/test-39.txt b/agent/bench-scripts/tests/pbench-trafficgen/test-39.txt index 6977fe3be9..113dd85e9c 100644 --- a/agent/bench-scripts/tests/pbench-trafficgen/test-39.txt +++ b/agent/bench-scripts/tests/pbench-trafficgen/test-39.txt @@ -114,7 +114,7 @@ Iteration 3-revunidirectional-74B-1000flows-0.0pct_drop complete (3 of 3), with /var/tmp/pbench-test-bench/opt/pbench-agent/bench-scripts/postprocess/trafficgen-postprocess /var/tmp/pbench-test-bench/pbench-agent/3-revunidirectional-74B-1000flows-0.0pct_drop/sample1 3-revunidirectional-74B-1000flows-0.0pct_drop 0.0 unknown default /var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/cat /proc/cmdline /var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-collect-sysinfo --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent --sysinfo=none beg -/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-collect-sysinfo --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent --sysinfo=none beg +/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-collect-sysinfo --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent --sysinfo=none end /var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-collect-sysinfo --sysinfo=none --check /var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-end-tools --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent /var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-init-tools --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent diff --git a/agent/bench-scripts/tests/pbench-trafficgen/test-40.txt b/agent/bench-scripts/tests/pbench-trafficgen/test-40.txt index 189b4891e1..67e64295f0 100644 --- a/agent/bench-scripts/tests/pbench-trafficgen/test-40.txt +++ b/agent/bench-scripts/tests/pbench-trafficgen/test-40.txt @@ -111,7 +111,7 @@ Iteration 3-revunidirectional-74B-1000flows-0.0pct_drop complete (3 of 3), with /var/tmp/pbench-test-bench/opt/pbench-agent/bench-scripts/postprocess/trafficgen-postprocess /var/tmp/pbench-test-bench/pbench-agent/3-revunidirectional-74B-1000flows-0.0pct_drop/sample1 3-revunidirectional-74B-1000flows-0.0pct_drop 0.0 unknown default /var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/cat /proc/cmdline /var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-collect-sysinfo --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent --sysinfo=none beg -/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-collect-sysinfo --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent --sysinfo=none beg +/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-collect-sysinfo --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent --sysinfo=none end /var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-collect-sysinfo --sysinfo=none --check /var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-end-tools --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent /var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-init-tools --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent diff --git a/agent/rpm/pbench-agent.spec.j2 b/agent/rpm/pbench-agent.spec.j2 index 82cd668790..2255ada0e7 100644 --- a/agent/rpm/pbench-agent.spec.j2 +++ b/agent/rpm/pbench-agent.spec.j2 @@ -122,7 +122,7 @@ fi /%{installdir}/tool-scripts %attr(664,root,root) /%{installdir}/tool-scripts/meta.json %attr(664,root,root) /%{installdir}/tool-scripts/perf.README -%attr(664,root,root) /%{installdir}/tool-scripts/README +%attr(664,root,root) /%{installdir}/tool-scripts/README.md /%{installdir}/bench-scripts %attr(664,root,root) /%{installdir}/bench-scripts/pbench-fio.md %attr(664,root,root) /%{installdir}/bench-scripts/pbench-uperf.md diff --git a/agent/util-scripts/gold/pbench-collect-sysinfo/test-55.txt b/agent/util-scripts/gold/pbench-collect-sysinfo/test-55.txt index de73fd51e8..2789b81f4e 100644 --- a/agent/util-scripts/gold/pbench-collect-sysinfo/test-55.txt +++ b/agent/util-scripts/gold/pbench-collect-sysinfo/test-55.txt @@ -16,7 +16,8 @@ Collecting system information +++ pbench.log file contents [debug][1900-01-01T00:00:00.000000] tool_opts: "" [info][1900-01-01T00:00:00.000000] "mpstat" tool is now registered for host "testhost.example.com" in group "default" +[info][1900-01-01T00:00:00.000000] Collecting system information --- pbench.log file contents +++ test-execution.log file contents -/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end block,libvirt,kernel_config,security_mitigations,sos,topology +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-tool-meister-client default /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end sysinfo block,libvirt,kernel_config,security_mitigations,sos,topology --- test-execution.log file contents diff --git a/agent/util-scripts/gold/test-client-tool-meister/test-53.txt b/agent/util-scripts/gold/test-client-tool-meister/test-53.txt index f463a43d05..97ef0b7372 100644 --- a/agent/util-scripts/gold/test-client-tool-meister/test-53.txt +++ b/agent/util-scripts/gold/test-client-tool-meister/test-53.txt @@ -1,6 +1,8 @@ +++ Running test-53 test-client-tool-meister "mpstat" tool is now registered for host "testhost.example.com" in group "default" "iostat" tool is now registered for host "testhost.example.com" in group "default" +Collecting system information +Collecting system information --- Finished test-53 test-client-tool-meister (status=0) +++ pbench tree state /var/tmp/pbench-test-utils/pbench @@ -51,6 +53,17 @@ /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-default/testhost.example.com/tm-mpstat-start.out /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-default/testhost.example.com/tm-mpstat-stop.err /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-default/testhost.example.com/tm-mpstat-stop.out +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/testhost.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/testhost.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/tm-sysinfo.err +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/tm-sysinfo.out +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/testhost.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/testhost.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/tm-sysinfo.err +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/tm-sysinfo.out /var/tmp/pbench-test-utils/pbench/mock-run/tm /var/tmp/pbench-test-utils/pbench/mock-run/tm/pbench-tool-data-sink.err /var/tmp/pbench-test-utils/pbench/mock-run/tm/pbench-tool-data-sink.log @@ -79,6 +92,8 @@ [info][1900-01-01T00:00:00.000000] "mpstat" tool is now registered for host "testhost.example.com" in group "default" [debug][1900-01-01T00:00:00.000000] tool_opts: "--interval=42 --options=forty-two" [info][1900-01-01T00:00:00.000000] "iostat" tool is now registered for host "testhost.example.com" in group "default" +[info][1900-01-01T00:00:00.000000] Collecting system information +[info][1900-01-01T00:00:00.000000] Collecting system information --- pbench.log file contents +++ mock-run/tm/pbench-tool-data-sink.err file contents Bottle v#.##.## server starting up (using DataSinkWsgiServer(handler_class=.DataSinkWsgiRequestHandler'>))... @@ -116,6 +131,8 @@ port 17001 --- mock-run/tm/tm-default-testhost.example.com.err file contents +++ mock-run/tm/tm-default-testhost.example.com.log file contents INFO pbench-tool-meister main -- params_key (tm-default-testhost.example.com): b'{"benchmark_run_dir": "/var/tmp/pbench-test-utils/pbench/mock-run", "channel": "tool-meister-chan", "controller": "testhost.example.com", "group": "default", "hostname": "testhost.example.com", "tools": {"iostat": "--interval=42 --options=forty-two", "mpstat": "--interval=42 --options=forty-two"}}' +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg block,security_mitigations,sos +INFO pbench-tool-meister sysinfo -- testhost.example.com: sysinfo send (no-op) default /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg INFO pbench-tool-meister start -- iostat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/iostat --start --dir=/var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-default/testhost.example.com --interval=42 --options=forty-two INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-default/testhost.example.com --interval=42 --options=forty-two INFO pbench-tool-meister stop -- iostat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/iostat --stop --dir=/var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-default/testhost.example.com --interval=42 --options=forty-two @@ -130,12 +147,16 @@ INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils INFO pbench-tool-meister wait -- waiting for stop iostat INFO pbench-tool-meister wait -- waiting for stop mpstat INFO pbench-tool-meister send_tools -- testhost.example.com: send_tools (no-op) default /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-default/testhost.example.com +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end block,security_mitigations,sos +INFO pbench-tool-meister sysinfo -- testhost.example.com: sysinfo send (no-op) default /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end INFO pbench-tool-meister main -- terminating INFO pbench-tool-meister main -- Remove pid file ... (tm-default-testhost.example.com.pid) --- mock-run/tm/tm-default-testhost.example.com.log file contents +++ mock-run/tm/tm-default-testhost.example.com.out file contents --- mock-run/tm/tm-default-testhost.example.com.out file contents +++ test-execution.log file contents +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg block,security_mitigations,sos +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end block,security_mitigations,sos /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pidof -x iostat /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pidof -x iostat /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pidof -x mpstat diff --git a/agent/util-scripts/gold/test-client-tool-meister/test-56.txt b/agent/util-scripts/gold/test-client-tool-meister/test-56.txt index 672183191b..d950acd94f 100644 --- a/agent/util-scripts/gold/test-client-tool-meister/test-56.txt +++ b/agent/util-scripts/gold/test-client-tool-meister/test-56.txt @@ -4,6 +4,8 @@ "mpstat" tool is now registered for host "remote_a.example.com" in group "lite" "mpstat" tool is now registered for host "remote_b.example.com" in group "lite" "mpstat" tool is now registered for host "remote_c.example.com" in group "lite" +Collecting system information +Collecting system information --- Finished test-56 test-client-tool-meister (status=0) +++ pbench tree state /var/tmp/pbench-test-utils/pbench @@ -126,6 +128,29 @@ /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/testhost.example.com/tm-mpstat-start.out /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/testhost.example.com/tm-mpstat-stop.err /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/testhost.example.com/tm-mpstat-stop.out +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/remote_a.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/remote_a.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/remote_b.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/remote_b.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/remote_c.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/remote_c.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/testhost.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/testhost.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/tm-sysinfo.err +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/tm-sysinfo.out +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/remote_a.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/remote_a.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/remote_b.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/remote_b.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/remote_c.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/remote_c.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/testhost.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/testhost.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/tm-sysinfo.err +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/tm-sysinfo.out /var/tmp/pbench-test-utils/pbench/mock-run/tm /var/tmp/pbench-test-utils/pbench/mock-run/tm/pbench-tool-data-sink.err /var/tmp/pbench-test-utils/pbench/mock-run/tm/pbench-tool-data-sink.log @@ -184,6 +209,8 @@ [info][1900-01-01T00:00:00.000000] "mpstat" tool is now registered for host "remote_b.example.com" in group "lite" [debug][1900-01-01T00:00:00.000000] tool_opts: "--interval=42 --options=forty-two" [info][1900-01-01T00:00:00.000000] "mpstat" tool is now registered for host "remote_c.example.com" in group "lite" +[info][1900-01-01T00:00:00.000000] Collecting system information +[info][1900-01-01T00:00:00.000000] Collecting system information --- pbench.log file contents +++ mock-run/tm/pbench-tool-data-sink.err file contents Bottle v#.##.## server starting up (using DataSinkWsgiServer(handler_class=.DataSinkWsgiRequestHandler'>))... @@ -193,6 +220,12 @@ Hit Ctrl-C to quit. --- mock-run/tm/pbench-tool-data-sink.err file contents +++ mock-run/tm/pbench-tool-data-sink.log file contents INFO pbench-tool-data-sink execute -- Terminate bottle server +INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /sysinfo-data/27c00bc325171c4893ef3862b4340952/remote_a.example.com HTTP/1.1" 200 0 +INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /sysinfo-data/27c00bc325171c4893ef3862b4340952/remote_b.example.com HTTP/1.1" 200 0 +INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /sysinfo-data/27c00bc325171c4893ef3862b4340952/remote_c.example.com HTTP/1.1" 200 0 +INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /sysinfo-data/9df83db2144bfc399f1f1343b788a63c/remote_a.example.com HTTP/1.1" 200 0 +INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /sysinfo-data/9df83db2144bfc399f1f1343b788a63c/remote_b.example.com HTTP/1.1" 200 0 +INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /sysinfo-data/9df83db2144bfc399f1f1343b788a63c/remote_c.example.com HTTP/1.1" 200 0 INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /tool-data/4a200d14fbab61d26581ad8e0129a73d/remote_a.example.com HTTP/1.1" 200 0 INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /tool-data/4a200d14fbab61d26581ad8e0129a73d/remote_b.example.com HTTP/1.1" 200 0 INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /tool-data/4a200d14fbab61d26581ad8e0129a73d/remote_c.example.com HTTP/1.1" 200 0 @@ -227,14 +260,18 @@ port 17001 --- mock-run/tm/tm-lite-remote_a.example.com.err file contents +++ mock-run/tm/tm-lite-remote_a.example.com.log file contents INFO pbench-tool-meister main -- params_key (tm-lite-remote_a.example.com): b'{"benchmark_run_dir": "/var/tmp/pbench-test-utils/pbench/mock-run", "channel": "tool-meister-chan", "controller": "localhost", "group": "lite", "hostname": "remote_a.example.com", "tools": {"mpstat": "--interval=42 --options=forty-two"}}' -INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two -INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +INFO pbench-tool-meister _send_directory -- remote_a.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com +INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two INFO pbench-tool-meister wait -- waiting for stop mpstat -INFO pbench-tool-meister send_tools -- remote_a.example.com: send_tools completed lite /var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com -INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two -INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister _send_directory -- remote_a.example.com: PUT tool-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com +INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two INFO pbench-tool-meister wait -- waiting for stop mpstat -INFO pbench-tool-meister send_tools -- remote_a.example.com: send_tools completed lite /var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com +INFO pbench-tool-meister _send_directory -- remote_a.example.com: PUT tool-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +INFO pbench-tool-meister _send_directory -- remote_a.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com INFO pbench-tool-meister main -- terminating INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-remote_a.example.com.pid) --- mock-run/tm/tm-lite-remote_a.example.com.log file contents @@ -244,14 +281,18 @@ INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-remote_a.example.c --- mock-run/tm/tm-lite-remote_b.example.com.err file contents +++ mock-run/tm/tm-lite-remote_b.example.com.log file contents INFO pbench-tool-meister main -- params_key (tm-lite-remote_b.example.com): b'{"benchmark_run_dir": "/var/tmp/pbench-test-utils/pbench/mock-run", "channel": "tool-meister-chan", "controller": "localhost", "group": "lite", "hostname": "remote_b.example.com", "tools": {"mpstat": "--interval=42 --options=forty-two"}}' -INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two -INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +INFO pbench-tool-meister _send_directory -- remote_b.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com +INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two INFO pbench-tool-meister wait -- waiting for stop mpstat -INFO pbench-tool-meister send_tools -- remote_b.example.com: send_tools completed lite /var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com -INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two -INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister _send_directory -- remote_b.example.com: PUT tool-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com +INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two INFO pbench-tool-meister wait -- waiting for stop mpstat -INFO pbench-tool-meister send_tools -- remote_b.example.com: send_tools completed lite /var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com +INFO pbench-tool-meister _send_directory -- remote_b.example.com: PUT tool-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +INFO pbench-tool-meister _send_directory -- remote_b.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com INFO pbench-tool-meister main -- terminating INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-remote_b.example.com.pid) --- mock-run/tm/tm-lite-remote_b.example.com.log file contents @@ -261,14 +302,18 @@ INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-remote_b.example.c --- mock-run/tm/tm-lite-remote_c.example.com.err file contents +++ mock-run/tm/tm-lite-remote_c.example.com.log file contents INFO pbench-tool-meister main -- params_key (tm-lite-remote_c.example.com): b'{"benchmark_run_dir": "/var/tmp/pbench-test-utils/pbench/mock-run", "channel": "tool-meister-chan", "controller": "localhost", "group": "lite", "hostname": "remote_c.example.com", "tools": {"mpstat": "--interval=42 --options=forty-two"}}' -INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two -INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +INFO pbench-tool-meister _send_directory -- remote_c.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com +INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two INFO pbench-tool-meister wait -- waiting for stop mpstat -INFO pbench-tool-meister send_tools -- remote_c.example.com: send_tools completed lite /var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com -INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two -INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister _send_directory -- remote_c.example.com: PUT tool-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com +INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two INFO pbench-tool-meister wait -- waiting for stop mpstat -INFO pbench-tool-meister send_tools -- remote_c.example.com: send_tools completed lite /var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com +INFO pbench-tool-meister _send_directory -- remote_c.example.com: PUT tool-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +INFO pbench-tool-meister _send_directory -- remote_c.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com INFO pbench-tool-meister main -- terminating INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-remote_c.example.com.pid) --- mock-run/tm/tm-lite-remote_c.example.com.log file contents @@ -278,6 +323,8 @@ INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-remote_c.example.c --- mock-run/tm/tm-lite-testhost.example.com.err file contents +++ mock-run/tm/tm-lite-testhost.example.com.log file contents INFO pbench-tool-meister main -- params_key (tm-lite-testhost.example.com): b'{"benchmark_run_dir": "/var/tmp/pbench-test-utils/pbench/mock-run", "channel": "tool-meister-chan", "controller": "testhost.example.com", "group": "lite", "hostname": "testhost.example.com", "tools": {"iostat": "--interval=42 --options=forty-two", "mpstat": "--interval=42 --options=forty-two"}}' +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg block,security_mitigations,sos +INFO pbench-tool-meister sysinfo -- testhost.example.com: sysinfo send (no-op) lite /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg INFO pbench-tool-meister start -- iostat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/iostat --start --dir=/var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/testhost.example.com --interval=42 --options=forty-two INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/testhost.example.com --interval=42 --options=forty-two INFO pbench-tool-meister stop -- iostat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/iostat --stop --dir=/var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/testhost.example.com --interval=42 --options=forty-two @@ -292,6 +339,8 @@ INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils INFO pbench-tool-meister wait -- waiting for stop iostat INFO pbench-tool-meister wait -- waiting for stop mpstat INFO pbench-tool-meister send_tools -- testhost.example.com: send_tools (no-op) lite /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/testhost.example.com +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end block,security_mitigations,sos +INFO pbench-tool-meister sysinfo -- testhost.example.com: sysinfo send (no-op) lite /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end INFO pbench-tool-meister main -- terminating INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-testhost.example.com.pid) --- mock-run/tm/tm-lite-testhost.example.com.log file contents @@ -304,6 +353,14 @@ INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-testhost.example.c /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/mpstat -P ALL --options=forty-two 42 /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/mpstat -P ALL --options=forty-two 42 /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/mpstat -P ALL --options=forty-two 42 +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg block,security_mitigations,sos +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end block,security_mitigations,sos +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pidof -x iostat /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pidof -x iostat /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pidof -x mpstat diff --git a/agent/util-scripts/gold/test-client-tool-meister/test-57.txt b/agent/util-scripts/gold/test-client-tool-meister/test-57.txt index 30a72a0f2a..10e1741427 100644 --- a/agent/util-scripts/gold/test-client-tool-meister/test-57.txt +++ b/agent/util-scripts/gold/test-client-tool-meister/test-57.txt @@ -4,6 +4,8 @@ "mpstat" tool is now registered for host "remote_a.example.com" in group "lite" "mpstat" tool is now registered for host "remote_b.example.com" in group "lite" "mpstat" tool is now registered for host "remote_c.example.com" in group "lite" +Collecting system information +Collecting system information --- Finished test-57 test-client-tool-meister (status=0) +++ pbench tree state /var/tmp/pbench-test-utils/pbench @@ -126,6 +128,29 @@ /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/testhost.example.com/tm-mpstat-start.out /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/testhost.example.com/tm-mpstat-stop.err /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/testhost.example.com/tm-mpstat-stop.out +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/remote_a.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/remote_a.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/remote_b.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/remote_b.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/remote_c.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/remote_c.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/testhost.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/testhost.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/tm-sysinfo.err +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/tm-sysinfo.out +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/remote_a.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/remote_a.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/remote_b.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/remote_b.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/remote_c.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/remote_c.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/testhost.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/testhost.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/tm-sysinfo.err +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/tm-sysinfo.out /var/tmp/pbench-test-utils/pbench/mock-run/tm /var/tmp/pbench-test-utils/pbench/mock-run/tm/pbench-tool-data-sink.err /var/tmp/pbench-test-utils/pbench/mock-run/tm/pbench-tool-data-sink.log @@ -184,6 +209,8 @@ [info][1900-01-01T00:00:00.000000] "mpstat" tool is now registered for host "remote_b.example.com" in group "lite" [debug][1900-01-01T00:00:00.000000] tool_opts: "--interval=42 --options=forty-two" [info][1900-01-01T00:00:00.000000] "mpstat" tool is now registered for host "remote_c.example.com" in group "lite" +[info][1900-01-01T00:00:00.000000] Collecting system information +[info][1900-01-01T00:00:00.000000] Collecting system information --- pbench.log file contents +++ mock-run/tm/pbench-tool-data-sink.err file contents Bottle v#.##.## server starting up (using DataSinkWsgiServer(handler_class=.DataSinkWsgiRequestHandler'>))... @@ -193,6 +220,12 @@ Hit Ctrl-C to quit. --- mock-run/tm/pbench-tool-data-sink.err file contents +++ mock-run/tm/pbench-tool-data-sink.log file contents INFO pbench-tool-data-sink execute -- Terminate bottle server +INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /sysinfo-data/27c00bc325171c4893ef3862b4340952/remote_a.example.com HTTP/1.1" 200 0 +INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /sysinfo-data/27c00bc325171c4893ef3862b4340952/remote_b.example.com HTTP/1.1" 200 0 +INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /sysinfo-data/27c00bc325171c4893ef3862b4340952/remote_c.example.com HTTP/1.1" 200 0 +INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /sysinfo-data/9df83db2144bfc399f1f1343b788a63c/remote_a.example.com HTTP/1.1" 200 0 +INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /sysinfo-data/9df83db2144bfc399f1f1343b788a63c/remote_b.example.com HTTP/1.1" 200 0 +INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /sysinfo-data/9df83db2144bfc399f1f1343b788a63c/remote_c.example.com HTTP/1.1" 200 0 INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /tool-data/4a200d14fbab61d26581ad8e0129a73d/remote_a.example.com HTTP/1.1" 200 0 INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /tool-data/4a200d14fbab61d26581ad8e0129a73d/remote_b.example.com HTTP/1.1" 200 0 INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /tool-data/4a200d14fbab61d26581ad8e0129a73d/remote_c.example.com HTTP/1.1" 200 0 @@ -227,14 +260,18 @@ port 17001 --- mock-run/tm/tm-lite-remote_a.example.com.err file contents +++ mock-run/tm/tm-lite-remote_a.example.com.log file contents INFO pbench-tool-meister main -- params_key (tm-lite-remote_a.example.com): b'{"benchmark_run_dir": "/var/tmp/pbench-test-utils/pbench/mock-run", "channel": "tool-meister-chan", "controller": "localhost", "group": "lite", "hostname": "remote_a.example.com", "tools": {"mpstat": "--interval=42 --options=forty-two"}}' -INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two -INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +INFO pbench-tool-meister _send_directory -- remote_a.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com +INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two INFO pbench-tool-meister wait -- waiting for stop mpstat -INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two -INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two INFO pbench-tool-meister wait -- waiting for stop mpstat -INFO pbench-tool-meister send_tools -- remote_a.example.com: send_tools completed lite /var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com -INFO pbench-tool-meister send_tools -- remote_a.example.com: send_tools completed lite /var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com +INFO pbench-tool-meister _send_directory -- remote_a.example.com: PUT tool-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com +INFO pbench-tool-meister _send_directory -- remote_a.example.com: PUT tool-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +INFO pbench-tool-meister _send_directory -- remote_a.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com INFO pbench-tool-meister main -- terminating INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-remote_a.example.com.pid) --- mock-run/tm/tm-lite-remote_a.example.com.log file contents @@ -244,14 +281,18 @@ INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-remote_a.example.c --- mock-run/tm/tm-lite-remote_b.example.com.err file contents +++ mock-run/tm/tm-lite-remote_b.example.com.log file contents INFO pbench-tool-meister main -- params_key (tm-lite-remote_b.example.com): b'{"benchmark_run_dir": "/var/tmp/pbench-test-utils/pbench/mock-run", "channel": "tool-meister-chan", "controller": "localhost", "group": "lite", "hostname": "remote_b.example.com", "tools": {"mpstat": "--interval=42 --options=forty-two"}}' -INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two -INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +INFO pbench-tool-meister _send_directory -- remote_b.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com +INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two INFO pbench-tool-meister wait -- waiting for stop mpstat -INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two -INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two INFO pbench-tool-meister wait -- waiting for stop mpstat -INFO pbench-tool-meister send_tools -- remote_b.example.com: send_tools completed lite /var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com -INFO pbench-tool-meister send_tools -- remote_b.example.com: send_tools completed lite /var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com +INFO pbench-tool-meister _send_directory -- remote_b.example.com: PUT tool-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com +INFO pbench-tool-meister _send_directory -- remote_b.example.com: PUT tool-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +INFO pbench-tool-meister _send_directory -- remote_b.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com INFO pbench-tool-meister main -- terminating INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-remote_b.example.com.pid) --- mock-run/tm/tm-lite-remote_b.example.com.log file contents @@ -261,14 +302,18 @@ INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-remote_b.example.c --- mock-run/tm/tm-lite-remote_c.example.com.err file contents +++ mock-run/tm/tm-lite-remote_c.example.com.log file contents INFO pbench-tool-meister main -- params_key (tm-lite-remote_c.example.com): b'{"benchmark_run_dir": "/var/tmp/pbench-test-utils/pbench/mock-run", "channel": "tool-meister-chan", "controller": "localhost", "group": "lite", "hostname": "remote_c.example.com", "tools": {"mpstat": "--interval=42 --options=forty-two"}}' -INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two -INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +INFO pbench-tool-meister _send_directory -- remote_c.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com +INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two INFO pbench-tool-meister wait -- waiting for stop mpstat -INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two -INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two INFO pbench-tool-meister wait -- waiting for stop mpstat -INFO pbench-tool-meister send_tools -- remote_c.example.com: send_tools completed lite /var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com -INFO pbench-tool-meister send_tools -- remote_c.example.com: send_tools completed lite /var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com +INFO pbench-tool-meister _send_directory -- remote_c.example.com: PUT tool-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com +INFO pbench-tool-meister _send_directory -- remote_c.example.com: PUT tool-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +INFO pbench-tool-meister _send_directory -- remote_c.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com INFO pbench-tool-meister main -- terminating INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-remote_c.example.com.pid) --- mock-run/tm/tm-lite-remote_c.example.com.log file contents @@ -278,6 +323,8 @@ INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-remote_c.example.c --- mock-run/tm/tm-lite-testhost.example.com.err file contents +++ mock-run/tm/tm-lite-testhost.example.com.log file contents INFO pbench-tool-meister main -- params_key (tm-lite-testhost.example.com): b'{"benchmark_run_dir": "/var/tmp/pbench-test-utils/pbench/mock-run", "channel": "tool-meister-chan", "controller": "testhost.example.com", "group": "lite", "hostname": "testhost.example.com", "tools": {"iostat": "--interval=42 --options=forty-two", "mpstat": "--interval=42 --options=forty-two"}}' +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg block,security_mitigations,sos +INFO pbench-tool-meister sysinfo -- testhost.example.com: sysinfo send (no-op) lite /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg INFO pbench-tool-meister start -- iostat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/iostat --start --dir=/var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/testhost.example.com --interval=42 --options=forty-two INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/testhost.example.com --interval=42 --options=forty-two INFO pbench-tool-meister stop -- iostat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/iostat --stop --dir=/var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/testhost.example.com --interval=42 --options=forty-two @@ -292,6 +339,8 @@ INFO pbench-tool-meister wait -- waiting for stop iostat INFO pbench-tool-meister wait -- waiting for stop mpstat INFO pbench-tool-meister send_tools -- testhost.example.com: send_tools (no-op) lite /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/testhost.example.com INFO pbench-tool-meister send_tools -- testhost.example.com: send_tools (no-op) lite /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/testhost.example.com +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end block,security_mitigations,sos +INFO pbench-tool-meister sysinfo -- testhost.example.com: sysinfo send (no-op) lite /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end INFO pbench-tool-meister main -- terminating INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-testhost.example.com.pid) --- mock-run/tm/tm-lite-testhost.example.com.log file contents @@ -304,6 +353,14 @@ INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-testhost.example.c /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/mpstat -P ALL --options=forty-two 42 /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/mpstat -P ALL --options=forty-two 42 /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/mpstat -P ALL --options=forty-two 42 +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg block,security_mitigations,sos +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end block,security_mitigations,sos +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pidof -x iostat /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pidof -x iostat /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pidof -x mpstat diff --git a/agent/util-scripts/pbench-collect-sysinfo b/agent/util-scripts/pbench-collect-sysinfo index 8aa3716301..698356bc44 100755 --- a/agent/util-scripts/pbench-collect-sysinfo +++ b/agent/util-scripts/pbench-collect-sysinfo @@ -1,5 +1,5 @@ #!/bin/bash -# -*- mode: shell-script; indent-tabs-mode: t; sh-basic-offset: 8; sh-indentation: 8; sh-indent-for-case-alt: + -*- +# -*- mode: shell-script; indent-tabs-mode: t; sh-basic-offset: 8; sh-indentation: 8; tab-width: 8 -*- script_path=`dirname $0` script_name=`basename $0` @@ -156,9 +156,6 @@ if [[ ! -d ${dir} ]]; then exit 1 fi -# always notify the user, as collection can sometimes take a while -echo "Collecting system information" - # Ensure we have a tools group directory to work with tool_group_dir="$(verify_tool_group "${group}")" if [[ ${?} -ne 0 || -z "${tool_group_dir}" ]]; then @@ -177,55 +174,10 @@ if [[ ! -d ${sysinfo_path} ]]; then exit 1 fi +# Always notify the user, as collection can sometimes take a while +msg="Collecting system information" +echo "${msg}"; log "${msg}" -function gather_remote_sysinfo_data { - local remote_host=${1} - local remote_label=${2} - debug_log "[${script_name}]running sysinfo-dump on ${remote_host}" - cmd="ssh ${ssh_opts} -n ${remote_host} pbench-remote-sysinfo-dump ${sysinfo_path} \"${sysinfo}\" ${remote_label}" - debug_log "[${script_name}] ${cmd} > ${sysinfo_path}/${remote_label}${remote_host}.tar 2> ${sysinfo_path}/${remote_label}${remote_host}.err" - ${cmd} > ${sysinfo_path}/${remote_label}${remote_host}.tar 2> ${sysinfo_path}/${remote_label}${remote_host}.err - if [[ ${?} -ne 0 ]]; then - error_log "Remote sysinfo-dump failed" - error_log "$(cat ${sysinfo_path}/${remote_label}${remote_host}.err)" - else - debug_log "[${script_name}]completed: deleting sysinfo-dump data on ${remote_host}" - debug_log "$(cat ${sysinfo_path}/${remote_label}${remote_host}.err)" - pushd "${sysinfo_path}" >/dev/null - tar xf ${remote_label}${remote_host}.tar - tar_status=${?} - popd >/dev/null - if [[ ${tar_status} -ne 0 ]]; then - error_log "Remote tar ball was bad: ${sysinfo_path}/${remote_label}${remote_host}.tar" - fi - fi - rm -f ${sysinfo_path}/${remote_label}${remote_host}.tar - rm -f ${sysinfo_path}/${remote_label}${remote_host}.err -} - - -## Collect the sysinfo-dumps in parallel - -for dirent in $(/bin/ls -1 ${tool_group_dir}); do - if [[ "${dirent}" == "__trigger__" ]]; then - # Ignore trigger files - continue - elif [[ ! -d ${tool_group_dir}/${dirent} ]]; then - # Skip spurious files of ${tool_group_dir} - warn_log "[$script_name] \"${this_tool_file}\" is a file in \"${tool_group_dir}\"; that should not happen. Please consider deleting it." - continue - fi - hostdir="${dirent}" - label="$(cat ${tool_group_dir}/${hostdir}/__label__ 2> /dev/null)" - if [[ "${hostdir}" == "localhost" || "${hostdir}" == "${_pbench_hostname}" || "${hostdir}" == "${_pbench_full_hostname}" ]]; then - # We found local tools specified in the tools group directory, - # so we should collect sysinfo data locally. - pbench-sysinfo-dump "${sysinfo_path}" "${sysinfo}" "${label}" & - else - # We have a remote directory. - gather_remote_sysinfo_data ${hostdir} ${label} & - fi -done -wait - -exit 0 +# Direct the Tool Meister's and Tool Data Sink to collect the system information. +pbench-tool-meister-client "${group}" "${sysinfo_path}" "sysinfo" "${sysinfo}" +exit ${?} diff --git a/agent/util-scripts/pbench-remote-sysinfo-dump b/agent/util-scripts/pbench-remote-sysinfo-dump deleted file mode 100755 index 9081dfedda..0000000000 --- a/agent/util-scripts/pbench-remote-sysinfo-dump +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -script_path=`dirname $0` -script_name=`basename $0` -pbench_bin="`cd ${script_path}/..; /bin/pwd`" - -# source the base script -. "$pbench_bin"/base - -# This is a wrapper script to allow sysinfo-dump to be invoked remotely -# sending on stdout the contents of the sysinfo tar ball, capturing all -# other output and emitting that on stderr. - -sysinfo_path=$1 -sysinfo=$2 -label=$3 - -mkdir -p $sysinfo_path -if [ ! -d $sysinfo_path ]; then - echo "Failed to create $sysinfo_path" >&2 - exit 1 -fi - -pbench-sysinfo-dump $sysinfo_path "$sysinfo" $label -if [ $? -ne 0 ]; then - echo "\"pbench-sysinfo-dump $sysinfo_path $label\" failed!" >&2 - exit 1 -fi - -cd $sysinfo_path >&2 -echo "pbench-sysinfo-dump data size: $(du -sm $sysinfo_path | awk '{print $1}') MB" >&2 -tar cf - * -exit_status=$? -rm -rf $sysinfo_path >&2 -exit $exit_status diff --git a/agent/util-scripts/pbench-tool-meister-client b/agent/util-scripts/pbench-tool-meister-client index fa52b2c142..8259b8b78b 100755 --- a/agent/util-scripts/pbench-tool-meister-client +++ b/agent/util-scripts/pbench-tool-meister-client @@ -25,7 +25,7 @@ tm_channel = "tool-meister-chan" cl_channel = "tool-meister-client" # List of allowed actions -allowed_actions = ("end", "init", "send", "start", "stop", "kill") +allowed_actions = ("end", "init", "send", "start", "stop", "kill", "sysinfo") def main(argv): @@ -68,6 +68,10 @@ def main(argv): # tool meisters, locally and remotely, and ensuring they are all # properly shut down. return 0 + try: + args = argv[4:] + except IndexError: + args = [] logger.debug("constructing Redis() object") try: @@ -141,7 +145,7 @@ def main(argv): # The caller of tool-meister-client must be sure the directory argument # is accessible by the tool-data-sink. logger.debug("publish %s", tm_channel) - msg = dict(action=action, group=group, directory=directory) + msg = dict(action=action, group=group, directory=directory, args=args) try: num_present = redis_server.publish(tm_channel, json.dumps(msg)) except Exception: diff --git a/agent/util-scripts/pbench-tool-meister-stop b/agent/util-scripts/pbench-tool-meister-stop index 7987022604..5f8088a2a6 100755 --- a/agent/util-scripts/pbench-tool-meister-stop +++ b/agent/util-scripts/pbench-tool-meister-stop @@ -101,7 +101,7 @@ def main(argv): ret_val = 0 logger.debug("terminating %d pids at localhost:%d", expected_pids, redis_port) - terminate_msg = dict(action="terminate", group=None, directory=None) + terminate_msg = dict(action="terminate", group=None, directory=None, args=None) try: num_present = redis_server.publish( channel, json.dumps(terminate_msg, sort_keys=True) diff --git a/agent/util-scripts/test-bin/pbench-sysinfo-dump b/agent/util-scripts/test-bin/pbench-sysinfo-dump deleted file mode 120000 index 7773fb5be0..0000000000 --- a/agent/util-scripts/test-bin/pbench-sysinfo-dump +++ /dev/null @@ -1 +0,0 @@ -mpstat \ No newline at end of file diff --git a/agent/util-scripts/test-bin/pbench-sysinfo-dump b/agent/util-scripts/test-bin/pbench-sysinfo-dump new file mode 100755 index 0000000000..878c4a9de4 --- /dev/null +++ b/agent/util-scripts/test-bin/pbench-sysinfo-dump @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 + +import sys +import os + +from pathlib import Path + + +with open(os.environ["_testlog"], "a") as ofp: + args = " ".join(sys.argv) + ofp.write(f"{args}\n") + +# Mock the behavior of the real pbench-sysinfo-dump + +tgtdir = Path(sys.argv[1]) +sysinfo = sys.argv[2] +try: + label = sys.argv[3] +except IndexError: + label = "" +hostname = os.environ["_pbench_full_hostname"] +hostdir = f"{label}:{hostname}" if label else hostname + +(tgtdir / hostdir).mkdir() + +(tgtdir / hostdir / "contents.lis").write_text(sysinfo) +sys.exit(0) diff --git a/agent/util-scripts/test-bin/ssh b/agent/util-scripts/test-bin/ssh index 2108546374..d4e449358a 100755 --- a/agent/util-scripts/test-bin/ssh +++ b/agent/util-scripts/test-bin/ssh @@ -26,7 +26,7 @@ if [[ "${1}" == "hostname" && "${2}" == "-s" ]]; then exit_code=0 elif [[ "$(basename -- "${1}")" == "pbench-tool-meister-remote" ]]; then _dir=$(dirname ${0}) - _tool_bin=${_dir}/mpstat ${1} localhost "${3}" "${4}" + _pbench_full_hostname="${remote}" _pbench_hostname="${remote}" _tool_bin=${_dir}/mpstat ${1} localhost "${3}" "${4}" exit_code=$? fi diff --git a/agent/util-scripts/test-bin/test-client-tool-meister b/agent/util-scripts/test-bin/test-client-tool-meister index cdcaf1f609..1f1721242b 100755 --- a/agent/util-scripts/test-bin/test-client-tool-meister +++ b/agent/util-scripts/test-bin/test-client-tool-meister @@ -68,6 +68,13 @@ if [[ ${status} -ne 0 ]]; then exit 1 fi +_timeout pbench-collect-sysinfo --group="${group}" --dir="${benchmark_run_dir}" --sysinfo="block,security_mitigations,sos" beg +status=${?} +if [[ ${status} -ne 0 ]]; then + printf -- "ERROR - \"pbench-collect-sysinfo\" failed to execute successfully (exit code: %s)\n" "${status}" >&2 + exit 1 +fi + _timeout pbench-init-tools --group="${group}" --dir="${benchmark_run_dir}" status=${?} if [[ ${status} -ne 0 ]]; then @@ -127,6 +134,13 @@ if [[ ${status} -ne 0 ]]; then exit 1 fi +_timeout pbench-collect-sysinfo --group="${group}" --dir="${benchmark_run_dir}" --sysinfo="block,security_mitigations,sos" end +status=${?} +if [[ ${status} -ne 0 ]]; then + printf -- "ERROR - \"pbench-collect-sysinfo\" failed to execute successfully (exit code: %s)\n" "${status}" >&2 + exit 1 +fi + _timeout pbench-tool-meister-stop status=${?} if [[ ${status} -ne 0 ]]; then diff --git a/agent/util-scripts/unittests b/agent/util-scripts/unittests index 4a84a4c240..c8a4e79f2b 100755 --- a/agent/util-scripts/unittests +++ b/agent/util-scripts/unittests @@ -530,7 +530,7 @@ declare -A pre_hooks=( [test-46]='mkdir ${_testdir}/tmp; printf -- "# bad list\none.example.com\ntwo.example.com,labelTwo\n\nthree.example.com,labelThree,junk\n" > ${_testdir}/tmp/remotes.lis' [test-47]='mkdir ${_testdir}/tmp; printf -- "# good list with no labels\none.example.com\ntwo.example.com\nthree.example.com\n" > ${_testdir}/tmp/remotes.lis' [test-48]='mkdir ${_testdir}/tmp; printf -- "%s\n" "30%" > ${_testdir}/tmp/foo.txt' - [test-55]='pbench-register-tool --name=mpstat --remote=localhost > /dev/null; mkdir ${_testdir}/mock-run' + [test-55]='pbench-register-tool --name=mpstat --remote=localhost > /dev/null; mkdir ${_testdir}/mock-run; ln -s mock-pbench-tool-meister-client ${_testopt}/unittest-scripts/pbench-tool-meister-client' [test-61]='ln -s mock-pbench-tool-meister-client ${_testopt}/unittest-scripts/pbench-tool-meister-client' [test-62]='ln -s mock-pbench-tool-meister-client ${_testopt}/unittest-scripts/pbench-tool-meister-client; mkdir -p ${_testdir}/mock-run/tools-default' ) @@ -551,6 +551,7 @@ declare -A post_hooks=( [test-07]='cat ${_testdir}/42-iter/sample42/tools-foobar/testhost.example.com/postprocess.log >> ${_testout} 2>&1' [test-19]='rm ${_testopt}/unittest-scripts/pbench-tool-meister-client' [test-53]='sort_testlog' + [test-55]='rm ${_testopt}/unittest-scripts/pbench-tool-meister-client' [test-56]='sort_testlog; sort_tdslog' [test-57]='sort_testlog; sort_tdslog' [test-61]='rm ${_testopt}/unittest-scripts/pbench-tool-meister-client' diff --git a/lib/pbench/agent/tool_data_sink.py b/lib/pbench/agent/tool_data_sink.py index 4b162c95d6..62f68a9759 100644 --- a/lib/pbench/agent/tool_data_sink.py +++ b/lib/pbench/agent/tool_data_sink.py @@ -297,7 +297,7 @@ def __init__(self, redis_server, channel, benchmark_run_dir, tool_group, logger) # Initialize internal state self._hostname = os.environ["_pbench_full_hostname"] self.state = None - self.tool_data_ctx = None + self.data_ctx = None self.directory = None self.tool_metadata = toolmetadata.ToolMetadata("redis", redis_server, logger) self._data = None @@ -307,10 +307,17 @@ def __init__(self, redis_server, channel, benchmark_run_dir, tool_group, logger) self._cv = Condition(lock=self._lock) # Setup the Bottle server route and the WSGI server instance. self.route( - "/tool-data//", + "/tool-data//", method="PUT", callback=self.put_document, ) + self.route( + "/sysinfo-data//", + method="PUT", + callback=self.put_document, + ) + # The list of states where we expect Tool Meisters to send data to us. + self._data_states = frozenset(("send", "sysinfo")) self._server = DataSinkWsgiServer(host="0.0.0.0", port=8080, logger=logger) # Setup the Redis server channel subscription logger.debug("pubsub") @@ -494,10 +501,13 @@ def _wait_for_all_data(self): """wait_for_all_data - block the caller until all of the registered tool meisters have sent their data. - Waiting is a no-op for all states except 'send'. In the 'send' state, - we are expecting to hear from all registered tool meisters. + Waiting is a no-op for all states except the "data states" (see + self._data_states). In a "data" state, we are expecting to hear from + all registered tool meisters. """ - assert self.state == "send", f"expected state 'send' not '{self.state}'" + assert ( + self.state in self._data_states + ), f"expected state to be one of {self._data_states!r} not {self.state!r}" assert self._tm_tracking is not None, "Logic bomb! self._tm_tracking is None" done = False @@ -597,7 +607,7 @@ def state_change(self, data): # data we receive to that directory, but expect them to provide the # opaque context in the URL for the PUT method. directory_bytes = directory_str.encode("utf-8") - self.tool_data_ctx = hashlib.md5(directory_bytes).hexdigest() + self.data_ctx = hashlib.md5(directory_bytes).hexdigest() # Transition to "send" state should reset self._tm_tracking with self._lock: @@ -626,7 +636,7 @@ def state_change(self, data): elif self.state == "end": if self._prom_server: self._prom_server.terminate() - elif self.state == "send": + elif self.state in self._data_states: self._change_tm_tracking("dormant", "waiting") # The Tool Data Sink cannot send success until all the Tool # Meisters have sent their collected data, so wait for all the @@ -678,7 +688,7 @@ def _send_client_status(self, status): ret_val = 0 return ret_val - def put_document(self, tool_data_ctx, hostname): + def put_document(self, data_ctx, hostname): """put_document - PUT callback method for Bottle web server end point The put_document method is called by threads serving web requests. @@ -689,16 +699,16 @@ def put_document(self, tool_data_ctx, hostname): """ with self._lock: - if self.state != "send": + if self.state not in self._data_states: # FIXME: Don't we have a race condition if the tool meisters # process their messages first? Seems like we need to send to # the Tool Data Sink first, and then send to all the tool # meisters. abort(400, f"Can't accept PUT requests in state '{self.state}'") - if self.tool_data_ctx != tool_data_ctx: + if self.data_ctx != data_ctx: # Tool Data Sink and this Tool Meister are out of sink as to # what data is expected. - abort(400, f"Unexpected tool data context, '{tool_data_ctx}'") + abort(400, f"Unexpected data context, '{data_ctx}'") # Fetch the Tool Meister tracking record for this host and verify # it is in the expected waiting state. tm_tracker = self._tm_tracking[hostname] @@ -735,11 +745,11 @@ def put_document(self, tool_data_ctx, hostname): target_dir = self.directory if not target_dir.is_dir(): self.logger.error("ERROR - directory, '%s', does not exist", target_dir) - abort(500, f"Invalid URL, path {target_dir} does not exist") - host_tool_data_tb_name = target_dir / f"{hostname}.tar.xz" - if host_tool_data_tb_name.exists(): - abort(409, f"{host_tool_data_tb_name} already uploaded") - host_tool_data_tb_md5 = Path(f"{host_tool_data_tb_name}.md5") + abort(500, "INTERNAL ERROR") + host_data_tb_name = target_dir / f"{hostname}.tar.xz" + if host_data_tb_name.exists(): + abort(409, f"{host_data_tb_name} already uploaded") + host_data_tb_md5 = Path(f"{host_data_tb_name}.md5") with tempfile.NamedTemporaryFile(mode="wb", dir=target_dir) as ofp: total_bytes = 0 @@ -766,45 +776,45 @@ def put_document(self, tool_data_ctx, hostname): # First write the .md5 try: - with host_tool_data_tb_md5.open("w") as md5fp: - md5fp.write(f"{exp_md5} {host_tool_data_tb_name.name}\n") + with host_data_tb_md5.open("w") as md5fp: + md5fp.write(f"{exp_md5} {host_data_tb_name.name}\n") except Exception: try: - os.remove(host_tool_data_tb_md5) + os.remove(host_data_tb_md5) except Exception as exc: self.logger.warning( "Failed to remove .md5 %s when trying to clean up: %s", - host_tool_data_tb_md5, + host_data_tb_md5, exc, ) self.logger.exception( - "Failed to write .md5 file, '%s'", host_tool_data_tb_md5 + "Failed to write .md5 file, '%s'", host_data_tb_md5 ) raise # Then create the final filename link to the temporary file. try: - os.link(ofp.name, host_tool_data_tb_name) + os.link(ofp.name, host_data_tb_name) except Exception: try: - os.remove(host_tool_data_tb_md5) + os.remove(host_data_tb_md5) except Exception as exc: self.logger.warning( "Failed to remove .md5 %s when trying to clean up: %s", - host_tool_data_tb_md5, + host_data_tb_md5, exc, ) self.logger.exception( "Failed to rename tar ball '%s' to '%s'", ofp.name, - host_tool_data_tb_md5, + host_data_tb_md5, ) raise else: self.logger.debug( "Successfully wrote %s (%s.md5)", - host_tool_data_tb_name, - host_tool_data_tb_name, + host_data_tb_name, + host_data_tb_name, ) # Now unpack that tar ball @@ -814,34 +824,32 @@ def put_document(self, tool_data_ctx, hostname): # Invoke tar directly for efficiency. with o_file.open("w") as ofp, e_file.open("w") as efp: cp = subprocess.run( - [tar_path, "-xf", host_tool_data_tb_name], + [tar_path, "-xf", host_data_tb_name], cwd=target_dir, stdin=None, stdout=ofp, stderr=efp, ) except Exception: - self.logger.exception( - "Failed to extract tools tar ball, '%s'", host_tool_data_tb_name - ) - raise + self.logger.exception("Failed to extract tar ball, '%s'", host_data_tb_name) + abort(500, "INTERNAL ERROR") else: if cp.returncode != 0: self.logger.error( - "Failed to create tools tar ball; return code: %d", cp.returncode + "Failed to create tar ball; return code: %d", cp.returncode ) abort(500, "INTERNAL ERROR") else: - self.logger.debug("Successfully unpacked %s", host_tool_data_tb_name) + self.logger.debug("Successfully unpacked %s", host_data_tb_name) try: o_file.unlink() e_file.unlink() - host_tool_data_tb_md5.unlink() - host_tool_data_tb_name.unlink() + host_data_tb_md5.unlink() + host_data_tb_name.unlink() except Exception: self.logger.exception( "Error removing unpacked tar ball '%s' and it's .md5", - host_tool_data_tb_name, + host_data_tb_name, ) # Tell the waiting "watcher" thread engaging in a "state" change diff --git a/lib/pbench/agent/tool_meister.py b/lib/pbench/agent/tool_meister.py index 8369dbe9fe..cb752dabb2 100644 --- a/lib/pbench/agent/tool_meister.py +++ b/lib/pbench/agent/tool_meister.py @@ -32,10 +32,6 @@ and the redis key to fetch its configuration from for its operation. [1] https://redis.io/ - -$ sudo dnf install python3-redis -$ sudo pip3 install python-daemon -$ sudo pip3 install python-pidfile """ import errno @@ -65,6 +61,9 @@ # Path to external tar executable. tar_path = None +# Path to external pbench-sysinfo-dump executable. +sysinfo_dump = None + # FIXME: The client response channel should be in a shared constants module. client_channel = "tool-meister-client" @@ -283,9 +282,8 @@ class ToolMeisterError(Exception): class ToolMeister: """Encapsulate tool life-cycle - The goal of this class is to make sure all necessary state and behaviors - for managing a given tool are handled by the methods offered by the - class. + The goal of this class is to provide the methods and attributes necessary + for managing the life-cycles of a registered set of tools. The start_, stop_, send_, and wait_ prefixed methods represent all the necessary interfaces for managing the life-cycle of a tool. The cleanup() @@ -336,6 +334,7 @@ class ToolMeister: Meister is running remotely, then it will use a local temporary directory to write it's data, and will send that data to the Tool Data Sink during the "send" phase. + """ @staticmethod @@ -409,7 +408,7 @@ def __init__(self, pbench_bin, params, redis_server, logger): "stop": {"curr": "running", "next": "idle", "action": self.stop_tools}, } self._valid_actions = frozenset( - ["end", "init", "send", "start", "stop", "terminate"] + ["end", "init", "send", "start", "stop", "sysinfo", "terminate"] ) for key in self._state_trans.keys(): assert ( @@ -419,7 +418,7 @@ def __init__(self, pbench_bin, params, redis_server, logger): "INTERNAL ERROR: invalid state transition 'next' entry for" f" '{key}', '{self._state_trans[key]['next']}'" ) - self._message_keys = frozenset(["directory", "group", "action"]) + self._message_keys = frozenset(["action", "args", "directory", "group"]) # The current 'directory' into which the tools are collected; not set # until a 'start tools' is executed, cleared when a 'send tools' # completes. @@ -431,7 +430,7 @@ def __init__(self, pbench_bin, params, redis_server, logger): # tools for storing their collected data. self._tool_dir = None # The temporary directory to use for capturing all tool data. - self._tmp_dir = os.environ.get("_PBENCH_TOOL_MEISTER_TMP", "/var/tmp") + self._tmp_dir = os.environ["pbench_tmp"] # FIXME: run all the "--install" commands for the tools to ensure # they are successful before declaring that we are ready. @@ -468,33 +467,32 @@ def _get_data(self): # FIXME: Add connection drop error handling, retry loop # re-establishing a connection. self.logger.exception("Error fetching 'next' data off channel") + continue else: self.logger.debug("next success") + msg = None try: json_str = payload["data"].decode("utf-8") - data = json.loads(json_str) + tmp_data = json.loads(json_str) except Exception: - self.logger.warning("data payload in message not JSON, %r", json_str) - data = None + msg = f"data payload in message not JSON, {json_str!r}" else: - keys = frozenset(data.keys()) + keys = frozenset(tmp_data.keys()) if keys != self._message_keys: - self.logger.warning( - "unrecognized keys in data of payload in message, %r", json_str, + msg = ( + f"unrecognized keys in data of payload in message, {json_str!r}" ) - data = None - elif data["action"] not in self._valid_actions: - self.logger.warning( - "unrecognized action in data of payload in message, %r", - json_str, - ) - data = None - elif data["group"] is not None and data["group"] != self._group: - self.logger.warning( - "unrecognized group in data of payload in message, %r", - json_str, - ) - data = None + elif tmp_data["action"] not in self._valid_actions: + msg = f"unrecognized action in data of payload in message, {json_str!r}" + elif tmp_data["group"] is not None and tmp_data["group"] != self._group: + msg = f"unrecognized group in data of payload in message, {json_str!r}" + else: + data = tmp_data + finally: + if msg is not None: + assert data is None + self.logger.warning(msg) + self._send_client_status(msg) return data["action"], data def wait_for_command(self): @@ -507,24 +505,26 @@ def wait_for_command(self): """ self.logger.debug("%s: wait_for_command %s", self._hostname, self.state) action, data = self._get_data() - done = False - while not done: + action_method = None + while not action_method: if action == "terminate": self.logger.debug("%s: msg - %r", self._hostname, data) raise Terminate() if action == "send": - self.logger.debug("%s: msg - %r", self._hostname, data) - return self.send_tools, data + action_method = self.send_tools + continue + if action == "sysinfo": + action_method = self.sysinfo + continue state_trans_rec = self._state_trans[action] if state_trans_rec["curr"] != self.state: - self.logger.info( - "ignoring unexpected data, %r, in state '%s'", data, self.state - ) + msg = f"ignoring unexpected data, {data!r}, in state '{self.state}'" + self.logger.info(msg) + self._send_client_status(msg) action, data = self._get_data() continue - done = True - action_method = state_trans_rec["action"] - self.state = state_trans_rec["next"] + action_method = state_trans_rec["action"] + self.state = state_trans_rec["next"] self.logger.debug("%s: msg - %r", self._hostname, data) return action_method, data @@ -797,52 +797,20 @@ def stop_tools(self, data): self._send_client_status("success") return failures - def send_tools(self, data): - """send_tools - send any collected tool data to the tool data sink. - - The 'action' and 'group' values of the payload have already been - validated before this "send tools" action is invoked. - - This method only proceeds if the 'directory' entry value of the - payload matches what was previously provided to a "start tools" - action. - """ - - if len(set(self._tools.keys()) - set(self.persist_tools)) == 0: - self._send_client_status("success") - return 0 + def _send_directory(self, directory, uri, ctx): + """_send_directory - tar up the given directory and send via PUT to the + URL constructed from the "uri" fragment, using the provided context. - directory = data["directory"] - try: - tool_dir = self.directories[directory] - except KeyError: - self.logger.error( - "INTERNAL ERROR - send tools action encountered for a" - " directory, '%s', that is different from any previous" - " start tools directory, %r", - directory, - self.directories.keys(), - ) - self._send_client_status("internal-error") - return False + The directory argument is a Path object who last element is a + directory with a name that is the same as the self.hostname. - if self._hostname == self._controller: - del self.directories[directory] - self.logger.info( - "%s: send_tools (no-op) %s %s", self._hostname, self._group, tool_dir - ) - # Note that we don't have a directory to send when a Tool - # Meister runs on the same host as the controller. - self._send_client_status("success") - return 0 + The uri and ctx arguments are used to form the final URL as defined by: - assert tool_dir.name == self._hostname, ( - f"Logic Bomb! Final path component of the tool directory is" - f" '{tool_dir.name}', not our host name '{self._hostname}'" - ) + f"http://{self._controller}:8080/{uri}/{ctx}/{self._hostname}" + """ failures = 0 - parent_dir = tool_dir.parent + parent_dir = directory.parent tar_file = parent_dir / f"{self._hostname}.tar.xz" o_file = parent_dir / f"{self._hostname}.tar.out" e_file = parent_dir / f"{self._hostname}.tar.err" @@ -857,22 +825,20 @@ def send_tools(self, data): stderr=efp, ) except Exception: - self.logger.exception("Failed to create tools tar ball '%s'", tar_file) + self.logger.exception("Failed to create tar ball '%s'", tar_file) + failures += 1 else: try: if cp.returncode != 0: self.logger.error( - "Failed to create tools tar ball; return code: %d", - cp.returncode, + "Failed to create tar ball; return code: %d", cp.returncode, ) failures += 1 else: try: tar_md5 = md5sum(tar_file) except Exception: - self.logger.exception( - "Failed to read tools tar ball, '%s'", tar_file - ) + self.logger.exception("Failed to read tar ball, '%s'", tar_file) failures += 1 else: try: @@ -893,17 +859,15 @@ def send_tools(self, data): ) self.logger.debug( - "%s: starting send_tools %s %s", + "%s: starting send_data group=%s, directory=%s", self._hostname, self._group, self._directory, ) headers = {"md5sum": tar_md5} - directory_bytes = data["directory"].encode("utf-8") - tool_data_ctx = hashlib.md5(directory_bytes).hexdigest() url = ( - f"http://{self._controller}:8080/tool-data" - f"/{tool_data_ctx}/{self._hostname}" + f"http://{self._controller}:8080/{uri}" + f"/{ctx}/{self._hostname}" ) sent = False retries = 200 @@ -949,16 +913,16 @@ def send_tools(self, data): parent_dir, ) failures += 1 - else: - del self.directories[directory] self.logger.info( - "%s: send_tools completed %s %s", + "%s: PUT %s completed %s %s", self._hostname, + uri, self._group, - tool_dir, + directory, ) except Exception: self.logger.exception("Unexpected error encountered") + failures += 1 finally: # We always remove the created tar file regardless of success or # failure. The above code should take care of removing the @@ -971,15 +935,76 @@ def send_tools(self, data): except OSError as exc: if exc.errno != errno.ENOENT: self.logger.warning( - "error removing tools tar ball, '%s': %s", tar_file, exc + "error removing tar ball, '%s': %s", tar_file, exc ) except Exception as exc: self.logger.warning( - "unexpected error removing tools tar ball, '%s': %s", tar_file, exc + "unexpected error removing tar ball, '%s': %s", tar_file, exc ) + return failures + + def send_tools(self, data): + """send_tools - send any collected tool data to the tool data sink. + + The 'action' and 'group' values of the payload have already been + validated before this "send tools" action is invoked. + + This method only proceeds if the 'directory' entry value of the + payload matches what was previously provided to a "start tools" + action. + """ + + if self.state in ("running", "startup"): + # The "send tool data" action is only allowed when the Tool + # Meister has left the startup state (received the first "init" at + # least, and is not running any tools. It is a no-op if a "send" + # is issued "send" before any tools were started. + msg = f"send action received in state '{self.state}'" + self._send_client_status(msg) + return 1 + + if len(set(self._tools.keys()) - set(self.persist_tools)) == 0: + self._send_client_status("success") + return 0 + + directory = data["directory"] + try: + tool_dir = self.directories[directory] + except KeyError: + self.logger.error( + "INTERNAL ERROR - send tools action encountered for a" + " directory, '%s', that is different from any previous" + " start tools directory, %r", + directory, + self.directories.keys(), + ) + self._send_client_status("internal-error") + return 1 + + if self._hostname == self._controller: + del self.directories[directory] + self.logger.info( + "%s: send_tools (no-op) %s %s", self._hostname, self._group, tool_dir + ) + # Note that we don't have a directory to send when a Tool + # Meister runs on the same host as the controller. + self._send_client_status("success") + return 0 + + assert tool_dir.name == self._hostname, ( + f"Logic Bomb! Final path component of the tool directory is" + f" '{tool_dir.name}', not our host name '{self._hostname}'" + ) + + directory_bytes = data["directory"].encode("utf-8") + tool_data_ctx = hashlib.md5(directory_bytes).hexdigest() + failures = self._send_directory(tool_dir, "tool-data", tool_data_ctx) + + if failures == 0: + del self.directories[directory] self._send_client_status( - "success" if failures == 0 else "failures sending tool data" + "success" if failures == 0 else f"{failures} failures sending tool data" ) return failures @@ -1014,6 +1039,103 @@ def end_tools(self, data): self._send_client_status("success") return failures + def sysinfo(self, data): + """sysinfo - collect all the sysinfo data for this host.""" + if self.state in ("running", "idle"): + # The "gather system information" action is only allowed when the + # Tool Meister first starts ("startup"), and when it is ready for + # shutting down ("shutdown"). + msg = f"sysinfo action received in state '{self.state}'" + self._send_client_status(msg) + return 1 + + args = data["args"] + if not args: + self._send_client_status("No sysinfo arguments given") + return 1 + + self.logger.debug("sysinfo: %r", args) + # FIXME - Should we perform more checking to validate the argument? + sysinfo = args[0] + # FIXME - We need the label for this registered host + label = "" + + if self._hostname == self._controller: + try: + sysinfo_dir = Path(data["directory"]).resolve(strict=True) + except Exception: + self.logger.exception( + "Failed to access provided sysinfo directory, %s", data["directory"] + ) + self._send_client_status("internal-error") + return 1 + else: + try: + sysinfo_dir = Path( + tempfile.mkdtemp( + dir=self._tmp_dir, prefix=f"tm.{self._group}.{os.getpid()}." + ) + ).resolve(strict=True) + except Exception: + self.logger.exception( + "Failed to create temporary directory for sysinfo operation" + ) + self._send_client_status("internal-error") + return 1 + + global sysinfo_dump + command = [ + sysinfo_dump, + str(sysinfo_dir), + sysinfo, + label, + ] + + self.logger.info("pbench-sysinfo-dump -- %s", " ".join(command)) + + o_file = sysinfo_dir / "tm-sysinfo.out" + e_file = sysinfo_dir / "tm-sysinfo.err" + try: + with o_file.open("w") as ofp, e_file.open("w") as efp: + cp = subprocess.run( + command, cwd=sysinfo_dir, stdin=None, stdout=ofp, stderr=efp, + ) + except Exception as exc: + msg = f"Failed to collect system information: {exc}" + self.logger.exception(msg) + self._send_client_status(msg) + return 1 + else: + if cp.returncode != 0: + msg = f"failed to collect system information; return code: {cp.returncode}" + self.logger.error(msg) + self._send_client_status(msg) + return 1 + + if self._hostname == self._controller: + self.logger.info( + "%s: sysinfo send (no-op) %s %s", + self._hostname, + self._group, + sysinfo_dir, + ) + # Note that we don't have a directory to send when a Tool + # Meister runs on the same host as the controller. + self._send_client_status("success") + return 0 + + directory_bytes = data["directory"].encode("utf-8") + sysinfo_data_ctx = hashlib.md5(directory_bytes).hexdigest() + failures = self._send_directory( + sysinfo_dir / self._hostname, "sysinfo-data", sysinfo_data_ctx + ) + + self._send_client_status( + "success" if failures == 0 else f"{failures} failures sending sysinfo data" + ) + + return failures + def main(argv): """Main program for the Tool Meister. @@ -1046,6 +1168,12 @@ def main(argv): print("External 'tar' executable not found.", file=sys.stderr) return 2 + global sysinfo_dump + sysinfo_dump = find_executable("pbench-sysinfo-dump") + if sysinfo_dump is None: + print("External 'pbench-sysinfo-dump' executable not found.", file=sys.stderr) + return 2 + logger = logging.getLogger(PROG) fh = logging.FileHandler(f"{param_key}.log") if os.environ.get("_PBENCH_UNIT_TESTS"):