diff --git a/agent/Makefile b/agent/Makefile index 076fc2e72f..ba72c29290 100644 --- a/agent/Makefile +++ b/agent/Makefile @@ -50,7 +50,6 @@ util-scripts = \ pbench-register-tool \ pbench-register-tool-set \ pbench-register-tool-trigger \ - pbench-remote-sysinfo-dump \ pbench-send-tools \ pbench-start-tools \ pbench-stop-tools \ @@ -136,7 +135,7 @@ tool-scripts = \ prometheus-metrics \ qemu-migrate \ rabbit \ - README \ + README.md \ sar \ strace \ sysfs \ diff --git a/agent/bench-scripts/pbench-cyclictest b/agent/bench-scripts/pbench-cyclictest index b8d8992999..bc3f457254 100755 --- a/agent/bench-scripts/pbench-cyclictest +++ b/agent/bench-scripts/pbench-cyclictest @@ -141,8 +141,6 @@ printf "\n" >>$benchmark_summary_txt_file printf "
\n# these results generated with:\n# uperf %s\n\n" "$scrtip_name $orig_cmd" >$benchmark_summary_html_file
 printf "\n" >>$benchmark_summary_html_file
 
-pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg
-
 # Start the tool meisters on each registered local/remote host
 pbench-tool-meister-start "${tool_group}"
 if [[ ${?} != 0 ]]; then
@@ -150,6 +148,8 @@ if [[ ${?} != 0 ]]; then
 	exit 1
 fi
 
+pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg
+
 pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir beg
 
 trap "interrupt" INT QUIT TERM
@@ -198,13 +198,14 @@ if [ $stress ]; then
 fi
 pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir end
 
+pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end
+
 # Stop the tool meisters on each registered local/remote host
 pbench-tool-meister-stop
 if [[ ${?} != 0 ]]; then
 	error_log "[${script_name}]: failed to stop the tool meisters."
 fi
 
-pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end
 cat $benchmark_summary_txt_file
 
 # signal to move/copy-results that this run is done.
diff --git a/agent/bench-scripts/pbench-dbench b/agent/bench-scripts/pbench-dbench
index 80d9e2f5b1..371cefcab8 100755
--- a/agent/bench-scripts/pbench-dbench
+++ b/agent/bench-scripts/pbench-dbench
@@ -327,8 +327,6 @@ let total_iterations=count
 count=1
 max_key_length=0
 
-pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg
-
 # Start the tool meisters on each registered local/remote host
 pbench-tool-meister-start "${tool_group}"
 if [[ ${?} != 0 ]]; then
@@ -336,6 +334,8 @@ if [[ ${?} != 0 ]]; then
 	exit 1
 fi
 
+pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg
+
 pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir beg
 
 trap "interrupt" INT QUIT TERM
@@ -555,12 +555,12 @@ printf -- "
\n" >>$benchmark_summary_html_file pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir end +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end + # Stop the tool meisters on each registered local/remote host pbench-tool-meister-stop if [[ ${?} != 0 ]]; then error_log "[${script_name}]: failed to stop the tool meisters." fi -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end - rmdir $benchmark_run_dir/.running diff --git a/agent/bench-scripts/pbench-fio b/agent/bench-scripts/pbench-fio index ea007b5572..a0f7c01778 100755 --- a/agent/bench-scripts/pbench-fio +++ b/agent/bench-scripts/pbench-fio @@ -877,8 +877,6 @@ fi benchmark_iterations="${benchmark_run_dir}/.iterations" > ${benchmark_iterations} -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg - # Start the tool meisters on each registered local/remote host pbench-tool-meister-start "${tool_group}" if [[ ${?} != 0 ]]; then @@ -886,6 +884,8 @@ if [[ ${?} != 0 ]]; then exit 1 fi +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg + pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir beg trap "interrupt" INT QUIT TERM @@ -899,12 +899,12 @@ $benchmark_run_dir/generate-benchmark-summary.cmd pbench-end-tools --group=$tool_group --dir=$benchmark_run_dir pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir end +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end + # Stop the tool meisters on each registered local/remote host pbench-tool-meister-stop if [[ ${?} != 0 ]]; then error_log "[${script_name}]: failed to stop the tool meisters." fi -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end - rmdir $benchmark_run_dir/.running diff --git a/agent/bench-scripts/pbench-iozone b/agent/bench-scripts/pbench-iozone index 96a4b57cd4..571b094ae4 100755 --- a/agent/bench-scripts/pbench-iozone +++ b/agent/bench-scripts/pbench-iozone @@ -193,8 +193,6 @@ mkdir -p $benchmark_run_dir/.running # now that the benchmark_run_dir directory exists, we can initialize the iterations file > ${benchmark_iterations} -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg - # Start the tool meisters on each registered local/remote host pbench-tool-meister-start "${tool_group}" if [[ ${?} != 0 ]]; then @@ -202,6 +200,8 @@ if [[ ${?} != 0 ]]; then exit 1 fi +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg + pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir beg trap "interrupt" INT QUIT TERM @@ -232,12 +232,12 @@ ln -s ${sample} $benchmark_run_dir/$iteration/reference-result pbench-end-tools --group=$tool_group --dir=$benchmark_run_dir pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir end +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end + # Stop the tool meisters on each registered local/remote host pbench-tool-meister-stop if [[ ${?} != 0 ]]; then error_log "[${script_name}]: failed to stop the tool meisters." fi -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end - rmdir $benchmark_run_dir/.running diff --git a/agent/bench-scripts/pbench-linpack b/agent/bench-scripts/pbench-linpack index 4ac84801fb..16bd23b302 100755 --- a/agent/bench-scripts/pbench-linpack +++ b/agent/bench-scripts/pbench-linpack @@ -135,8 +135,6 @@ benchmark_results_dir="$benchmark_run_dir/$iteration/${sample}" result_file=$benchmark_results_dir/result.txt mkdir -p $benchmark_results_dir -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg - # Start the tool meisters on each registered local/remote host pbench-tool-meister-start "${tool_group}" if [[ ${?} != 0 ]]; then @@ -144,6 +142,8 @@ if [[ ${?} != 0 ]]; then exit 1 fi +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg + pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir beg trap "interrupt" INT QUIT TERM @@ -161,12 +161,12 @@ ln -s ${sample} $benchmark_run_dir/$iteration/reference-result pbench-end-tools --group=$tool_group --dir=$benchmark_run_dir pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir end +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end + # Stop the tool meisters on each registered local/remote host pbench-tool-meister-stop if [[ ${?} != 0 ]]; then error_log "[${script_name}]: failed to stop the tool meisters." fi -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end - rmdir $benchmark_run_dir/.running diff --git a/agent/bench-scripts/pbench-migrate b/agent/bench-scripts/pbench-migrate index 0ccde7d008..2a8afd3047 100755 --- a/agent/bench-scripts/pbench-migrate +++ b/agent/bench-scripts/pbench-migrate @@ -348,8 +348,6 @@ if [ $loops -eq 0 ]; then loops=1 fi -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg - # Start the tool meisters on each registered local/remote host pbench-tool-meister-start "${tool_group}" if [[ ${?} != 0 ]]; then @@ -357,6 +355,8 @@ if [[ ${?} != 0 ]]; then exit 1 fi +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg + pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir beg trap "interrupt" INT QUIT TERM @@ -418,12 +418,12 @@ wait pbench-end-tools --group=$tool_group --dir=$benchmark_run_dir pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir end +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end + # Stop the tool meisters on each registered local/remote host pbench-tool-meister-stop if [[ ${?} != 0 ]]; then error_log "[${script_name}]: failed to stop the tool meisters." fi -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end - rmdir $benchmark_run_dir/.running diff --git a/agent/bench-scripts/pbench-netperf b/agent/bench-scripts/pbench-netperf index ede408bcd8..d13e0ff58c 100755 --- a/agent/bench-scripts/pbench-netperf +++ b/agent/bench-scripts/pbench-netperf @@ -439,8 +439,6 @@ printf "\n" >>$benchmark_summary_html_file # disable firewall on local system systemctl stop firewalld -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg - # Start the tool meisters on each registered local/remote host pbench-tool-meister-start "${tool_group}" if [[ ${?} != 0 ]]; then @@ -448,6 +446,8 @@ if [[ ${?} != 0 ]]; then exit 1 fi +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg + pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir beg trap "interrupt" INT QUIT TERM @@ -641,12 +641,12 @@ printf -- "\n" >>$benchmark_summary_html_file pbench-end-tools --group=$tool_group --dir=$benchmark_run_dir pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir end +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end + # Stop the tool meisters on each registered local/remote host pbench-tool-meister-stop if [[ ${?} != 0 ]]; then error_log "[${script_name}]: failed to stop the tool meisters." fi -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end - rmdir $benchmark_run_dir/.running diff --git a/agent/bench-scripts/pbench-specjbb2005 b/agent/bench-scripts/pbench-specjbb2005 index 9137dd6a46..a223747b67 100755 --- a/agent/bench-scripts/pbench-specjbb2005 +++ b/agent/bench-scripts/pbench-specjbb2005 @@ -267,9 +267,6 @@ benchmark_iterations="${benchmark_run_dir}/.iterations" pbench-register-tool-trigger --group=$tool_group --start-trigger="Timing Measurement began" --stop-trigger="Timing Measurement ended" -# Collect system information before we begin, and record the beginning. -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg - # Start the tool meisters on each registered local/remote host pbench-tool-meister-start "${tool_group}" if [[ ${?} != 0 ]]; then @@ -277,6 +274,9 @@ if [[ ${?} != 0 ]]; then exit 1 fi +# Collect system information before we begin, and record the beginning. +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg + pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir beg trap "interrupt" INT QUIT TERM @@ -392,12 +392,12 @@ popd > /dev/null 2>&1 pbench-end-tools --group=$tool_group --dir=$benchmark_run_dir pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir end +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end + # Stop the tool meisters on each registered local/remote host pbench-tool-meister-stop if [[ ${?} != 0 ]]; then error_log "[${script_name}]: failed to stop the tool meisters." fi -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end - rmdir $benchmark_run_dir/.running diff --git a/agent/bench-scripts/pbench-trafficgen b/agent/bench-scripts/pbench-trafficgen index 5932e95e53..ff135a0d26 100755 --- a/agent/bench-scripts/pbench-trafficgen +++ b/agent/bench-scripts/pbench-trafficgen @@ -997,8 +997,6 @@ echo "Total number of benchmark iterations: $total_iterations" export benchmark=${benchmark_name} -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg - # Start the tool meisters on each registered local/remote host pbench-tool-meister-start "${tool_group}" if [[ ${?} != 0 ]]; then @@ -1006,6 +1004,8 @@ if [[ ${?} != 0 ]]; then exit 1 fi +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg + pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir beg trap "interrupt" INT QUIT TERM @@ -1308,12 +1308,12 @@ fi pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir end +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end + # Stop the tool meisters on each registered local/remote host pbench-tool-meister-stop if [[ ${?} != 0 ]]; then error_log "[${script_name}]: failed to stop the tool meisters." fi -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg - rmdir $benchmark_run_dir/.running diff --git a/agent/bench-scripts/pbench-uperf b/agent/bench-scripts/pbench-uperf index f19a15f160..4bbdf0b3da 100755 --- a/agent/bench-scripts/pbench-uperf +++ b/agent/bench-scripts/pbench-uperf @@ -499,8 +499,6 @@ for protocol in `echo $protocols | sed -e s/,/" "/g`; do done done -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg - # Start the tool meisters on each registered local/remote host pbench-tool-meister-start "${tool_group}" if [[ ${?} != 0 ]]; then @@ -508,6 +506,8 @@ if [[ ${?} != 0 ]]; then exit 1 fi +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg + pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir beg trap "interrupt" INT QUIT TERM @@ -746,12 +746,12 @@ $benchmark_run_dir/generate-benchmark-summary.cmd pbench-end-tools --group=$tool_group --dir=$benchmark_run_dir pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir end +pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end + # Stop the tool meisters on each registered local/remote host pbench-tool-meister-stop if [[ ${?} != 0 ]]; then error_log "[${script_name}]: failed to stop the tool meisters." fi -pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end - rmdir $benchmark_run_dir/.running diff --git a/agent/bench-scripts/pbench-user-benchmark b/agent/bench-scripts/pbench-user-benchmark index 4f6c3d503e..56ea16041c 100755 --- a/agent/bench-scripts/pbench-user-benchmark +++ b/agent/bench-scripts/pbench-user-benchmark @@ -355,17 +355,17 @@ for rdir in ${result_dirs}; do ln -s "$(basename "${rdir}")" "$(dirname "${rdir}")/reference-result" done +${script_path}/postprocess/user-benchmark-wrapper "${benchmark_run_dir}" "${total_duration}" + pbench-metadata-log --group="${tool_group}" --dir="${benchmark_run_dir}" end +pbench-collect-sysinfo --group="${tool_group}" --dir="${benchmark_run_dir}" --sysinfo="${sysinfo}" end + # Stop the tool meisters on each registered local/remote host pbench-tool-meister-stop if [[ ${?} != 0 ]]; then error_log "[${script_name}]: failed to stop the tool meisters." fi -${script_path}/postprocess/user-benchmark-wrapper "${benchmark_run_dir}" "${total_duration}" - -pbench-collect-sysinfo --group="${tool_group}" --dir="${benchmark_run_dir}" --sysinfo="${sysinfo}" end - rmdir ${benchmark_run_dir}/.running exit 0 diff --git a/agent/bench-scripts/tests/pbench-trafficgen/test-22.txt b/agent/bench-scripts/tests/pbench-trafficgen/test-22.txt index b5559fd83b..539c3389e5 100644 --- a/agent/bench-scripts/tests/pbench-trafficgen/test-22.txt +++ b/agent/bench-scripts/tests/pbench-trafficgen/test-22.txt @@ -32172,7 +32172,7 @@ Iteration 3-revunidirectional-74B-1000flows-0.0pct_drop complete (3 of 3), with --- pbench.log file contents +++ test-execution.log file contents /var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-collect-sysinfo --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent/trafficgen_test-22_1970.01.01T00.00.00 --sysinfo=none beg -/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-collect-sysinfo --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent/trafficgen_test-22_1970.01.01T00.00.00 --sysinfo=none beg +/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-collect-sysinfo --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent/trafficgen_test-22_1970.01.01T00.00.00 --sysinfo=none end /var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-collect-sysinfo --sysinfo=none --check /var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-metadata-log --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent/trafficgen_test-22_1970.01.01T00.00.00 beg /var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-metadata-log --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent/trafficgen_test-22_1970.01.01T00.00.00 end diff --git a/agent/bench-scripts/tests/pbench-trafficgen/test-39.txt b/agent/bench-scripts/tests/pbench-trafficgen/test-39.txt index 6977fe3be9..113dd85e9c 100644 --- a/agent/bench-scripts/tests/pbench-trafficgen/test-39.txt +++ b/agent/bench-scripts/tests/pbench-trafficgen/test-39.txt @@ -114,7 +114,7 @@ Iteration 3-revunidirectional-74B-1000flows-0.0pct_drop complete (3 of 3), with /var/tmp/pbench-test-bench/opt/pbench-agent/bench-scripts/postprocess/trafficgen-postprocess /var/tmp/pbench-test-bench/pbench-agent/3-revunidirectional-74B-1000flows-0.0pct_drop/sample1 3-revunidirectional-74B-1000flows-0.0pct_drop 0.0 unknown default /var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/cat /proc/cmdline /var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-collect-sysinfo --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent --sysinfo=none beg -/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-collect-sysinfo --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent --sysinfo=none beg +/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-collect-sysinfo --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent --sysinfo=none end /var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-collect-sysinfo --sysinfo=none --check /var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-end-tools --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent /var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-init-tools --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent diff --git a/agent/bench-scripts/tests/pbench-trafficgen/test-40.txt b/agent/bench-scripts/tests/pbench-trafficgen/test-40.txt index 189b4891e1..67e64295f0 100644 --- a/agent/bench-scripts/tests/pbench-trafficgen/test-40.txt +++ b/agent/bench-scripts/tests/pbench-trafficgen/test-40.txt @@ -111,7 +111,7 @@ Iteration 3-revunidirectional-74B-1000flows-0.0pct_drop complete (3 of 3), with /var/tmp/pbench-test-bench/opt/pbench-agent/bench-scripts/postprocess/trafficgen-postprocess /var/tmp/pbench-test-bench/pbench-agent/3-revunidirectional-74B-1000flows-0.0pct_drop/sample1 3-revunidirectional-74B-1000flows-0.0pct_drop 0.0 unknown default /var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/cat /proc/cmdline /var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-collect-sysinfo --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent --sysinfo=none beg -/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-collect-sysinfo --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent --sysinfo=none beg +/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-collect-sysinfo --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent --sysinfo=none end /var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-collect-sysinfo --sysinfo=none --check /var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-end-tools --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent /var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-init-tools --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent diff --git a/agent/rpm/pbench-agent.spec.j2 b/agent/rpm/pbench-agent.spec.j2 index 82cd668790..2255ada0e7 100644 --- a/agent/rpm/pbench-agent.spec.j2 +++ b/agent/rpm/pbench-agent.spec.j2 @@ -122,7 +122,7 @@ fi /%{installdir}/tool-scripts %attr(664,root,root) /%{installdir}/tool-scripts/meta.json %attr(664,root,root) /%{installdir}/tool-scripts/perf.README -%attr(664,root,root) /%{installdir}/tool-scripts/README +%attr(664,root,root) /%{installdir}/tool-scripts/README.md /%{installdir}/bench-scripts %attr(664,root,root) /%{installdir}/bench-scripts/pbench-fio.md %attr(664,root,root) /%{installdir}/bench-scripts/pbench-uperf.md diff --git a/agent/util-scripts/gold/pbench-collect-sysinfo/test-55.txt b/agent/util-scripts/gold/pbench-collect-sysinfo/test-55.txt index de73fd51e8..2789b81f4e 100644 --- a/agent/util-scripts/gold/pbench-collect-sysinfo/test-55.txt +++ b/agent/util-scripts/gold/pbench-collect-sysinfo/test-55.txt @@ -16,7 +16,8 @@ Collecting system information +++ pbench.log file contents [debug][1900-01-01T00:00:00.000000] tool_opts: "" [info][1900-01-01T00:00:00.000000] "mpstat" tool is now registered for host "testhost.example.com" in group "default" +[info][1900-01-01T00:00:00.000000] Collecting system information --- pbench.log file contents +++ test-execution.log file contents -/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end block,libvirt,kernel_config,security_mitigations,sos,topology +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-tool-meister-client default /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end sysinfo block,libvirt,kernel_config,security_mitigations,sos,topology --- test-execution.log file contents diff --git a/agent/util-scripts/gold/test-client-tool-meister/test-53.txt b/agent/util-scripts/gold/test-client-tool-meister/test-53.txt index f463a43d05..97ef0b7372 100644 --- a/agent/util-scripts/gold/test-client-tool-meister/test-53.txt +++ b/agent/util-scripts/gold/test-client-tool-meister/test-53.txt @@ -1,6 +1,8 @@ +++ Running test-53 test-client-tool-meister "mpstat" tool is now registered for host "testhost.example.com" in group "default" "iostat" tool is now registered for host "testhost.example.com" in group "default" +Collecting system information +Collecting system information --- Finished test-53 test-client-tool-meister (status=0) +++ pbench tree state /var/tmp/pbench-test-utils/pbench @@ -51,6 +53,17 @@ /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-default/testhost.example.com/tm-mpstat-start.out /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-default/testhost.example.com/tm-mpstat-stop.err /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-default/testhost.example.com/tm-mpstat-stop.out +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/testhost.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/testhost.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/tm-sysinfo.err +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/tm-sysinfo.out +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/testhost.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/testhost.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/tm-sysinfo.err +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/tm-sysinfo.out /var/tmp/pbench-test-utils/pbench/mock-run/tm /var/tmp/pbench-test-utils/pbench/mock-run/tm/pbench-tool-data-sink.err /var/tmp/pbench-test-utils/pbench/mock-run/tm/pbench-tool-data-sink.log @@ -79,6 +92,8 @@ [info][1900-01-01T00:00:00.000000] "mpstat" tool is now registered for host "testhost.example.com" in group "default" [debug][1900-01-01T00:00:00.000000] tool_opts: "--interval=42 --options=forty-two" [info][1900-01-01T00:00:00.000000] "iostat" tool is now registered for host "testhost.example.com" in group "default" +[info][1900-01-01T00:00:00.000000] Collecting system information +[info][1900-01-01T00:00:00.000000] Collecting system information --- pbench.log file contents +++ mock-run/tm/pbench-tool-data-sink.err file contents Bottle v#.##.## server starting up (using DataSinkWsgiServer(handler_class=.DataSinkWsgiRequestHandler'>))... @@ -116,6 +131,8 @@ port 17001 --- mock-run/tm/tm-default-testhost.example.com.err file contents +++ mock-run/tm/tm-default-testhost.example.com.log file contents INFO pbench-tool-meister main -- params_key (tm-default-testhost.example.com): b'{"benchmark_run_dir": "/var/tmp/pbench-test-utils/pbench/mock-run", "channel": "tool-meister-chan", "controller": "testhost.example.com", "group": "default", "hostname": "testhost.example.com", "tools": {"iostat": "--interval=42 --options=forty-two", "mpstat": "--interval=42 --options=forty-two"}}' +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg block,security_mitigations,sos +INFO pbench-tool-meister sysinfo -- testhost.example.com: sysinfo send (no-op) default /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg INFO pbench-tool-meister start -- iostat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/iostat --start --dir=/var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-default/testhost.example.com --interval=42 --options=forty-two INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-default/testhost.example.com --interval=42 --options=forty-two INFO pbench-tool-meister stop -- iostat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/iostat --stop --dir=/var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-default/testhost.example.com --interval=42 --options=forty-two @@ -130,12 +147,16 @@ INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils INFO pbench-tool-meister wait -- waiting for stop iostat INFO pbench-tool-meister wait -- waiting for stop mpstat INFO pbench-tool-meister send_tools -- testhost.example.com: send_tools (no-op) default /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-default/testhost.example.com +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end block,security_mitigations,sos +INFO pbench-tool-meister sysinfo -- testhost.example.com: sysinfo send (no-op) default /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end INFO pbench-tool-meister main -- terminating INFO pbench-tool-meister main -- Remove pid file ... (tm-default-testhost.example.com.pid) --- mock-run/tm/tm-default-testhost.example.com.log file contents +++ mock-run/tm/tm-default-testhost.example.com.out file contents --- mock-run/tm/tm-default-testhost.example.com.out file contents +++ test-execution.log file contents +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg block,security_mitigations,sos +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end block,security_mitigations,sos /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pidof -x iostat /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pidof -x iostat /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pidof -x mpstat diff --git a/agent/util-scripts/gold/test-client-tool-meister/test-56.txt b/agent/util-scripts/gold/test-client-tool-meister/test-56.txt index 672183191b..d950acd94f 100644 --- a/agent/util-scripts/gold/test-client-tool-meister/test-56.txt +++ b/agent/util-scripts/gold/test-client-tool-meister/test-56.txt @@ -4,6 +4,8 @@ "mpstat" tool is now registered for host "remote_a.example.com" in group "lite" "mpstat" tool is now registered for host "remote_b.example.com" in group "lite" "mpstat" tool is now registered for host "remote_c.example.com" in group "lite" +Collecting system information +Collecting system information --- Finished test-56 test-client-tool-meister (status=0) +++ pbench tree state /var/tmp/pbench-test-utils/pbench @@ -126,6 +128,29 @@ /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/testhost.example.com/tm-mpstat-start.out /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/testhost.example.com/tm-mpstat-stop.err /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/testhost.example.com/tm-mpstat-stop.out +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/remote_a.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/remote_a.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/remote_b.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/remote_b.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/remote_c.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/remote_c.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/testhost.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/testhost.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/tm-sysinfo.err +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/tm-sysinfo.out +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/remote_a.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/remote_a.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/remote_b.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/remote_b.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/remote_c.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/remote_c.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/testhost.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/testhost.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/tm-sysinfo.err +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/tm-sysinfo.out /var/tmp/pbench-test-utils/pbench/mock-run/tm /var/tmp/pbench-test-utils/pbench/mock-run/tm/pbench-tool-data-sink.err /var/tmp/pbench-test-utils/pbench/mock-run/tm/pbench-tool-data-sink.log @@ -184,6 +209,8 @@ [info][1900-01-01T00:00:00.000000] "mpstat" tool is now registered for host "remote_b.example.com" in group "lite" [debug][1900-01-01T00:00:00.000000] tool_opts: "--interval=42 --options=forty-two" [info][1900-01-01T00:00:00.000000] "mpstat" tool is now registered for host "remote_c.example.com" in group "lite" +[info][1900-01-01T00:00:00.000000] Collecting system information +[info][1900-01-01T00:00:00.000000] Collecting system information --- pbench.log file contents +++ mock-run/tm/pbench-tool-data-sink.err file contents Bottle v#.##.## server starting up (using DataSinkWsgiServer(handler_class=.DataSinkWsgiRequestHandler'>))... @@ -193,6 +220,12 @@ Hit Ctrl-C to quit. --- mock-run/tm/pbench-tool-data-sink.err file contents +++ mock-run/tm/pbench-tool-data-sink.log file contents INFO pbench-tool-data-sink execute -- Terminate bottle server +INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /sysinfo-data/27c00bc325171c4893ef3862b4340952/remote_a.example.com HTTP/1.1" 200 0 +INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /sysinfo-data/27c00bc325171c4893ef3862b4340952/remote_b.example.com HTTP/1.1" 200 0 +INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /sysinfo-data/27c00bc325171c4893ef3862b4340952/remote_c.example.com HTTP/1.1" 200 0 +INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /sysinfo-data/9df83db2144bfc399f1f1343b788a63c/remote_a.example.com HTTP/1.1" 200 0 +INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /sysinfo-data/9df83db2144bfc399f1f1343b788a63c/remote_b.example.com HTTP/1.1" 200 0 +INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /sysinfo-data/9df83db2144bfc399f1f1343b788a63c/remote_c.example.com HTTP/1.1" 200 0 INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /tool-data/4a200d14fbab61d26581ad8e0129a73d/remote_a.example.com HTTP/1.1" 200 0 INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /tool-data/4a200d14fbab61d26581ad8e0129a73d/remote_b.example.com HTTP/1.1" 200 0 INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /tool-data/4a200d14fbab61d26581ad8e0129a73d/remote_c.example.com HTTP/1.1" 200 0 @@ -227,14 +260,18 @@ port 17001 --- mock-run/tm/tm-lite-remote_a.example.com.err file contents +++ mock-run/tm/tm-lite-remote_a.example.com.log file contents INFO pbench-tool-meister main -- params_key (tm-lite-remote_a.example.com): b'{"benchmark_run_dir": "/var/tmp/pbench-test-utils/pbench/mock-run", "channel": "tool-meister-chan", "controller": "localhost", "group": "lite", "hostname": "remote_a.example.com", "tools": {"mpstat": "--interval=42 --options=forty-two"}}' -INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two -INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +INFO pbench-tool-meister _send_directory -- remote_a.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com +INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two INFO pbench-tool-meister wait -- waiting for stop mpstat -INFO pbench-tool-meister send_tools -- remote_a.example.com: send_tools completed lite /var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com -INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two -INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister _send_directory -- remote_a.example.com: PUT tool-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com +INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two INFO pbench-tool-meister wait -- waiting for stop mpstat -INFO pbench-tool-meister send_tools -- remote_a.example.com: send_tools completed lite /var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com +INFO pbench-tool-meister _send_directory -- remote_a.example.com: PUT tool-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +INFO pbench-tool-meister _send_directory -- remote_a.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com INFO pbench-tool-meister main -- terminating INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-remote_a.example.com.pid) --- mock-run/tm/tm-lite-remote_a.example.com.log file contents @@ -244,14 +281,18 @@ INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-remote_a.example.c --- mock-run/tm/tm-lite-remote_b.example.com.err file contents +++ mock-run/tm/tm-lite-remote_b.example.com.log file contents INFO pbench-tool-meister main -- params_key (tm-lite-remote_b.example.com): b'{"benchmark_run_dir": "/var/tmp/pbench-test-utils/pbench/mock-run", "channel": "tool-meister-chan", "controller": "localhost", "group": "lite", "hostname": "remote_b.example.com", "tools": {"mpstat": "--interval=42 --options=forty-two"}}' -INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two -INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +INFO pbench-tool-meister _send_directory -- remote_b.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com +INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two INFO pbench-tool-meister wait -- waiting for stop mpstat -INFO pbench-tool-meister send_tools -- remote_b.example.com: send_tools completed lite /var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com -INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two -INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister _send_directory -- remote_b.example.com: PUT tool-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com +INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two INFO pbench-tool-meister wait -- waiting for stop mpstat -INFO pbench-tool-meister send_tools -- remote_b.example.com: send_tools completed lite /var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com +INFO pbench-tool-meister _send_directory -- remote_b.example.com: PUT tool-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +INFO pbench-tool-meister _send_directory -- remote_b.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com INFO pbench-tool-meister main -- terminating INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-remote_b.example.com.pid) --- mock-run/tm/tm-lite-remote_b.example.com.log file contents @@ -261,14 +302,18 @@ INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-remote_b.example.c --- mock-run/tm/tm-lite-remote_c.example.com.err file contents +++ mock-run/tm/tm-lite-remote_c.example.com.log file contents INFO pbench-tool-meister main -- params_key (tm-lite-remote_c.example.com): b'{"benchmark_run_dir": "/var/tmp/pbench-test-utils/pbench/mock-run", "channel": "tool-meister-chan", "controller": "localhost", "group": "lite", "hostname": "remote_c.example.com", "tools": {"mpstat": "--interval=42 --options=forty-two"}}' -INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two -INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +INFO pbench-tool-meister _send_directory -- remote_c.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com +INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two INFO pbench-tool-meister wait -- waiting for stop mpstat -INFO pbench-tool-meister send_tools -- remote_c.example.com: send_tools completed lite /var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com -INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two -INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister _send_directory -- remote_c.example.com: PUT tool-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com +INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two INFO pbench-tool-meister wait -- waiting for stop mpstat -INFO pbench-tool-meister send_tools -- remote_c.example.com: send_tools completed lite /var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com +INFO pbench-tool-meister _send_directory -- remote_c.example.com: PUT tool-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +INFO pbench-tool-meister _send_directory -- remote_c.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com INFO pbench-tool-meister main -- terminating INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-remote_c.example.com.pid) --- mock-run/tm/tm-lite-remote_c.example.com.log file contents @@ -278,6 +323,8 @@ INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-remote_c.example.c --- mock-run/tm/tm-lite-testhost.example.com.err file contents +++ mock-run/tm/tm-lite-testhost.example.com.log file contents INFO pbench-tool-meister main -- params_key (tm-lite-testhost.example.com): b'{"benchmark_run_dir": "/var/tmp/pbench-test-utils/pbench/mock-run", "channel": "tool-meister-chan", "controller": "testhost.example.com", "group": "lite", "hostname": "testhost.example.com", "tools": {"iostat": "--interval=42 --options=forty-two", "mpstat": "--interval=42 --options=forty-two"}}' +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg block,security_mitigations,sos +INFO pbench-tool-meister sysinfo -- testhost.example.com: sysinfo send (no-op) lite /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg INFO pbench-tool-meister start -- iostat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/iostat --start --dir=/var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/testhost.example.com --interval=42 --options=forty-two INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/testhost.example.com --interval=42 --options=forty-two INFO pbench-tool-meister stop -- iostat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/iostat --stop --dir=/var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/testhost.example.com --interval=42 --options=forty-two @@ -292,6 +339,8 @@ INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils INFO pbench-tool-meister wait -- waiting for stop iostat INFO pbench-tool-meister wait -- waiting for stop mpstat INFO pbench-tool-meister send_tools -- testhost.example.com: send_tools (no-op) lite /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/testhost.example.com +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end block,security_mitigations,sos +INFO pbench-tool-meister sysinfo -- testhost.example.com: sysinfo send (no-op) lite /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end INFO pbench-tool-meister main -- terminating INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-testhost.example.com.pid) --- mock-run/tm/tm-lite-testhost.example.com.log file contents @@ -304,6 +353,14 @@ INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-testhost.example.c /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/mpstat -P ALL --options=forty-two 42 /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/mpstat -P ALL --options=forty-two 42 /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/mpstat -P ALL --options=forty-two 42 +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg block,security_mitigations,sos +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end block,security_mitigations,sos +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pidof -x iostat /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pidof -x iostat /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pidof -x mpstat diff --git a/agent/util-scripts/gold/test-client-tool-meister/test-57.txt b/agent/util-scripts/gold/test-client-tool-meister/test-57.txt index 30a72a0f2a..10e1741427 100644 --- a/agent/util-scripts/gold/test-client-tool-meister/test-57.txt +++ b/agent/util-scripts/gold/test-client-tool-meister/test-57.txt @@ -4,6 +4,8 @@ "mpstat" tool is now registered for host "remote_a.example.com" in group "lite" "mpstat" tool is now registered for host "remote_b.example.com" in group "lite" "mpstat" tool is now registered for host "remote_c.example.com" in group "lite" +Collecting system information +Collecting system information --- Finished test-57 test-client-tool-meister (status=0) +++ pbench tree state /var/tmp/pbench-test-utils/pbench @@ -126,6 +128,29 @@ /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/testhost.example.com/tm-mpstat-start.out /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/testhost.example.com/tm-mpstat-stop.err /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/testhost.example.com/tm-mpstat-stop.out +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/remote_a.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/remote_a.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/remote_b.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/remote_b.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/remote_c.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/remote_c.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/testhost.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/testhost.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/tm-sysinfo.err +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg/tm-sysinfo.out +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/remote_a.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/remote_a.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/remote_b.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/remote_b.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/remote_c.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/remote_c.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/testhost.example.com +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/testhost.example.com/contents.lis +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/tm-sysinfo.err +/var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end/tm-sysinfo.out /var/tmp/pbench-test-utils/pbench/mock-run/tm /var/tmp/pbench-test-utils/pbench/mock-run/tm/pbench-tool-data-sink.err /var/tmp/pbench-test-utils/pbench/mock-run/tm/pbench-tool-data-sink.log @@ -184,6 +209,8 @@ [info][1900-01-01T00:00:00.000000] "mpstat" tool is now registered for host "remote_b.example.com" in group "lite" [debug][1900-01-01T00:00:00.000000] tool_opts: "--interval=42 --options=forty-two" [info][1900-01-01T00:00:00.000000] "mpstat" tool is now registered for host "remote_c.example.com" in group "lite" +[info][1900-01-01T00:00:00.000000] Collecting system information +[info][1900-01-01T00:00:00.000000] Collecting system information --- pbench.log file contents +++ mock-run/tm/pbench-tool-data-sink.err file contents Bottle v#.##.## server starting up (using DataSinkWsgiServer(handler_class=.DataSinkWsgiRequestHandler'>))... @@ -193,6 +220,12 @@ Hit Ctrl-C to quit. --- mock-run/tm/pbench-tool-data-sink.err file contents +++ mock-run/tm/pbench-tool-data-sink.log file contents INFO pbench-tool-data-sink execute -- Terminate bottle server +INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /sysinfo-data/27c00bc325171c4893ef3862b4340952/remote_a.example.com HTTP/1.1" 200 0 +INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /sysinfo-data/27c00bc325171c4893ef3862b4340952/remote_b.example.com HTTP/1.1" 200 0 +INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /sysinfo-data/27c00bc325171c4893ef3862b4340952/remote_c.example.com HTTP/1.1" 200 0 +INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /sysinfo-data/9df83db2144bfc399f1f1343b788a63c/remote_a.example.com HTTP/1.1" 200 0 +INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /sysinfo-data/9df83db2144bfc399f1f1343b788a63c/remote_b.example.com HTTP/1.1" 200 0 +INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /sysinfo-data/9df83db2144bfc399f1f1343b788a63c/remote_c.example.com HTTP/1.1" 200 0 INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /tool-data/4a200d14fbab61d26581ad8e0129a73d/remote_a.example.com HTTP/1.1" 200 0 INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /tool-data/4a200d14fbab61d26581ad8e0129a73d/remote_b.example.com HTTP/1.1" 200 0 INFO pbench-tool-data-sink log_request -- 127.0.0.1 - - "PUT /tool-data/4a200d14fbab61d26581ad8e0129a73d/remote_c.example.com HTTP/1.1" 200 0 @@ -227,14 +260,18 @@ port 17001 --- mock-run/tm/tm-lite-remote_a.example.com.err file contents +++ mock-run/tm/tm-lite-remote_a.example.com.log file contents INFO pbench-tool-meister main -- params_key (tm-lite-remote_a.example.com): b'{"benchmark_run_dir": "/var/tmp/pbench-test-utils/pbench/mock-run", "channel": "tool-meister-chan", "controller": "localhost", "group": "lite", "hostname": "remote_a.example.com", "tools": {"mpstat": "--interval=42 --options=forty-two"}}' -INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two -INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +INFO pbench-tool-meister _send_directory -- remote_a.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com +INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two INFO pbench-tool-meister wait -- waiting for stop mpstat -INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two -INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com --interval=42 --options=forty-two INFO pbench-tool-meister wait -- waiting for stop mpstat -INFO pbench-tool-meister send_tools -- remote_a.example.com: send_tools completed lite /var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com -INFO pbench-tool-meister send_tools -- remote_a.example.com: send_tools completed lite /var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com +INFO pbench-tool-meister _send_directory -- remote_a.example.com: PUT tool-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com +INFO pbench-tool-meister _send_directory -- remote_a.example.com: PUT tool-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +INFO pbench-tool-meister _send_directory -- remote_a.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_a.example.com INFO pbench-tool-meister main -- terminating INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-remote_a.example.com.pid) --- mock-run/tm/tm-lite-remote_a.example.com.log file contents @@ -244,14 +281,18 @@ INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-remote_a.example.c --- mock-run/tm/tm-lite-remote_b.example.com.err file contents +++ mock-run/tm/tm-lite-remote_b.example.com.log file contents INFO pbench-tool-meister main -- params_key (tm-lite-remote_b.example.com): b'{"benchmark_run_dir": "/var/tmp/pbench-test-utils/pbench/mock-run", "channel": "tool-meister-chan", "controller": "localhost", "group": "lite", "hostname": "remote_b.example.com", "tools": {"mpstat": "--interval=42 --options=forty-two"}}' -INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two -INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +INFO pbench-tool-meister _send_directory -- remote_b.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com +INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two INFO pbench-tool-meister wait -- waiting for stop mpstat -INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two -INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com --interval=42 --options=forty-two INFO pbench-tool-meister wait -- waiting for stop mpstat -INFO pbench-tool-meister send_tools -- remote_b.example.com: send_tools completed lite /var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com -INFO pbench-tool-meister send_tools -- remote_b.example.com: send_tools completed lite /var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com +INFO pbench-tool-meister _send_directory -- remote_b.example.com: PUT tool-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com +INFO pbench-tool-meister _send_directory -- remote_b.example.com: PUT tool-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +INFO pbench-tool-meister _send_directory -- remote_b.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_b.example.com INFO pbench-tool-meister main -- terminating INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-remote_b.example.com.pid) --- mock-run/tm/tm-lite-remote_b.example.com.log file contents @@ -261,14 +302,18 @@ INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-remote_b.example.c --- mock-run/tm/tm-lite-remote_c.example.com.err file contents +++ mock-run/tm/tm-lite-remote_c.example.com.log file contents INFO pbench-tool-meister main -- params_key (tm-lite-remote_c.example.com): b'{"benchmark_run_dir": "/var/tmp/pbench-test-utils/pbench/mock-run", "channel": "tool-meister-chan", "controller": "localhost", "group": "lite", "hostname": "remote_c.example.com", "tools": {"mpstat": "--interval=42 --options=forty-two"}}' -INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two -INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +INFO pbench-tool-meister _send_directory -- remote_c.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com +INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two INFO pbench-tool-meister wait -- waiting for stop mpstat -INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two -INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two +INFO pbench-tool-meister stop -- mpstat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --stop --dir=/var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com --interval=42 --options=forty-two INFO pbench-tool-meister wait -- waiting for stop mpstat -INFO pbench-tool-meister send_tools -- remote_c.example.com: send_tools completed lite /var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com -INFO pbench-tool-meister send_tools -- remote_c.example.com: send_tools completed lite /var/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com +INFO pbench-tool-meister _send_directory -- remote_c.example.com: PUT tool-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com +INFO pbench-tool-meister _send_directory -- remote_c.example.com: PUT tool-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +INFO pbench-tool-meister _send_directory -- remote_c.example.com: PUT sysinfo-data completed lite /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn/remote_c.example.com INFO pbench-tool-meister main -- terminating INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-remote_c.example.com.pid) --- mock-run/tm/tm-lite-remote_c.example.com.log file contents @@ -278,6 +323,8 @@ INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-remote_c.example.c --- mock-run/tm/tm-lite-testhost.example.com.err file contents +++ mock-run/tm/tm-lite-testhost.example.com.log file contents INFO pbench-tool-meister main -- params_key (tm-lite-testhost.example.com): b'{"benchmark_run_dir": "/var/tmp/pbench-test-utils/pbench/mock-run", "channel": "tool-meister-chan", "controller": "testhost.example.com", "group": "lite", "hostname": "testhost.example.com", "tools": {"iostat": "--interval=42 --options=forty-two", "mpstat": "--interval=42 --options=forty-two"}}' +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg block,security_mitigations,sos +INFO pbench-tool-meister sysinfo -- testhost.example.com: sysinfo send (no-op) lite /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg INFO pbench-tool-meister start -- iostat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/iostat --start --dir=/var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/testhost.example.com --interval=42 --options=forty-two INFO pbench-tool-meister start -- mpstat: start_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/mpstat --start --dir=/var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/testhost.example.com --interval=42 --options=forty-two INFO pbench-tool-meister stop -- iostat: stop_tool -- /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/iostat --stop --dir=/var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/testhost.example.com --interval=42 --options=forty-two @@ -292,6 +339,8 @@ INFO pbench-tool-meister wait -- waiting for stop iostat INFO pbench-tool-meister wait -- waiting for stop mpstat INFO pbench-tool-meister send_tools -- testhost.example.com: send_tools (no-op) lite /var/tmp/pbench-test-utils/pbench/mock-run/0-iter-zero/sample42/tools-lite/testhost.example.com INFO pbench-tool-meister send_tools -- testhost.example.com: send_tools (no-op) lite /var/tmp/pbench-test-utils/pbench/mock-run/1-iter-one/sample42/tools-lite/testhost.example.com +INFO pbench-tool-meister sysinfo -- pbench-sysinfo-dump -- /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end block,security_mitigations,sos +INFO pbench-tool-meister sysinfo -- testhost.example.com: sysinfo send (no-op) lite /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end INFO pbench-tool-meister main -- terminating INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-testhost.example.com.pid) --- mock-run/tm/tm-lite-testhost.example.com.log file contents @@ -304,6 +353,14 @@ INFO pbench-tool-meister main -- Remove pid file ... (tm-lite-testhost.example.c /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/mpstat -P ALL --options=forty-two 42 /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/mpstat -P ALL --options=forty-two 42 /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/mpstat -P ALL --options=forty-two 42 +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/beg block,security_mitigations,sos +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/mock-run/sysinfo/end block,security_mitigations,sos +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pbench-sysinfo-dump /var/tmp/pbench-test-utils/pbench/tmp/tm.lite.NNNNN.nnnnnnnn block,security_mitigations,sos /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pidof -x iostat /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pidof -x iostat /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/pidof -x mpstat diff --git a/agent/util-scripts/pbench-collect-sysinfo b/agent/util-scripts/pbench-collect-sysinfo index 8aa3716301..698356bc44 100755 --- a/agent/util-scripts/pbench-collect-sysinfo +++ b/agent/util-scripts/pbench-collect-sysinfo @@ -1,5 +1,5 @@ #!/bin/bash -# -*- mode: shell-script; indent-tabs-mode: t; sh-basic-offset: 8; sh-indentation: 8; sh-indent-for-case-alt: + -*- +# -*- mode: shell-script; indent-tabs-mode: t; sh-basic-offset: 8; sh-indentation: 8; tab-width: 8 -*- script_path=`dirname $0` script_name=`basename $0` @@ -156,9 +156,6 @@ if [[ ! -d ${dir} ]]; then exit 1 fi -# always notify the user, as collection can sometimes take a while -echo "Collecting system information" - # Ensure we have a tools group directory to work with tool_group_dir="$(verify_tool_group "${group}")" if [[ ${?} -ne 0 || -z "${tool_group_dir}" ]]; then @@ -177,55 +174,10 @@ if [[ ! -d ${sysinfo_path} ]]; then exit 1 fi +# Always notify the user, as collection can sometimes take a while +msg="Collecting system information" +echo "${msg}"; log "${msg}" -function gather_remote_sysinfo_data { - local remote_host=${1} - local remote_label=${2} - debug_log "[${script_name}]running sysinfo-dump on ${remote_host}" - cmd="ssh ${ssh_opts} -n ${remote_host} pbench-remote-sysinfo-dump ${sysinfo_path} \"${sysinfo}\" ${remote_label}" - debug_log "[${script_name}] ${cmd} > ${sysinfo_path}/${remote_label}${remote_host}.tar 2> ${sysinfo_path}/${remote_label}${remote_host}.err" - ${cmd} > ${sysinfo_path}/${remote_label}${remote_host}.tar 2> ${sysinfo_path}/${remote_label}${remote_host}.err - if [[ ${?} -ne 0 ]]; then - error_log "Remote sysinfo-dump failed" - error_log "$(cat ${sysinfo_path}/${remote_label}${remote_host}.err)" - else - debug_log "[${script_name}]completed: deleting sysinfo-dump data on ${remote_host}" - debug_log "$(cat ${sysinfo_path}/${remote_label}${remote_host}.err)" - pushd "${sysinfo_path}" >/dev/null - tar xf ${remote_label}${remote_host}.tar - tar_status=${?} - popd >/dev/null - if [[ ${tar_status} -ne 0 ]]; then - error_log "Remote tar ball was bad: ${sysinfo_path}/${remote_label}${remote_host}.tar" - fi - fi - rm -f ${sysinfo_path}/${remote_label}${remote_host}.tar - rm -f ${sysinfo_path}/${remote_label}${remote_host}.err -} - - -## Collect the sysinfo-dumps in parallel - -for dirent in $(/bin/ls -1 ${tool_group_dir}); do - if [[ "${dirent}" == "__trigger__" ]]; then - # Ignore trigger files - continue - elif [[ ! -d ${tool_group_dir}/${dirent} ]]; then - # Skip spurious files of ${tool_group_dir} - warn_log "[$script_name] \"${this_tool_file}\" is a file in \"${tool_group_dir}\"; that should not happen. Please consider deleting it." - continue - fi - hostdir="${dirent}" - label="$(cat ${tool_group_dir}/${hostdir}/__label__ 2> /dev/null)" - if [[ "${hostdir}" == "localhost" || "${hostdir}" == "${_pbench_hostname}" || "${hostdir}" == "${_pbench_full_hostname}" ]]; then - # We found local tools specified in the tools group directory, - # so we should collect sysinfo data locally. - pbench-sysinfo-dump "${sysinfo_path}" "${sysinfo}" "${label}" & - else - # We have a remote directory. - gather_remote_sysinfo_data ${hostdir} ${label} & - fi -done -wait - -exit 0 +# Direct the Tool Meister's and Tool Data Sink to collect the system information. +pbench-tool-meister-client "${group}" "${sysinfo_path}" "sysinfo" "${sysinfo}" +exit ${?} diff --git a/agent/util-scripts/pbench-remote-sysinfo-dump b/agent/util-scripts/pbench-remote-sysinfo-dump deleted file mode 100755 index 9081dfedda..0000000000 --- a/agent/util-scripts/pbench-remote-sysinfo-dump +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -script_path=`dirname $0` -script_name=`basename $0` -pbench_bin="`cd ${script_path}/..; /bin/pwd`" - -# source the base script -. "$pbench_bin"/base - -# This is a wrapper script to allow sysinfo-dump to be invoked remotely -# sending on stdout the contents of the sysinfo tar ball, capturing all -# other output and emitting that on stderr. - -sysinfo_path=$1 -sysinfo=$2 -label=$3 - -mkdir -p $sysinfo_path -if [ ! -d $sysinfo_path ]; then - echo "Failed to create $sysinfo_path" >&2 - exit 1 -fi - -pbench-sysinfo-dump $sysinfo_path "$sysinfo" $label -if [ $? -ne 0 ]; then - echo "\"pbench-sysinfo-dump $sysinfo_path $label\" failed!" >&2 - exit 1 -fi - -cd $sysinfo_path >&2 -echo "pbench-sysinfo-dump data size: $(du -sm $sysinfo_path | awk '{print $1}') MB" >&2 -tar cf - * -exit_status=$? -rm -rf $sysinfo_path >&2 -exit $exit_status diff --git a/agent/util-scripts/pbench-tool-meister-client b/agent/util-scripts/pbench-tool-meister-client index fa52b2c142..8259b8b78b 100755 --- a/agent/util-scripts/pbench-tool-meister-client +++ b/agent/util-scripts/pbench-tool-meister-client @@ -25,7 +25,7 @@ tm_channel = "tool-meister-chan" cl_channel = "tool-meister-client" # List of allowed actions -allowed_actions = ("end", "init", "send", "start", "stop", "kill") +allowed_actions = ("end", "init", "send", "start", "stop", "kill", "sysinfo") def main(argv): @@ -68,6 +68,10 @@ def main(argv): # tool meisters, locally and remotely, and ensuring they are all # properly shut down. return 0 + try: + args = argv[4:] + except IndexError: + args = [] logger.debug("constructing Redis() object") try: @@ -141,7 +145,7 @@ def main(argv): # The caller of tool-meister-client must be sure the directory argument # is accessible by the tool-data-sink. logger.debug("publish %s", tm_channel) - msg = dict(action=action, group=group, directory=directory) + msg = dict(action=action, group=group, directory=directory, args=args) try: num_present = redis_server.publish(tm_channel, json.dumps(msg)) except Exception: diff --git a/agent/util-scripts/pbench-tool-meister-start b/agent/util-scripts/pbench-tool-meister-start index c269ab2b21..1ec1b6b3f5 100755 --- a/agent/util-scripts/pbench-tool-meister-start +++ b/agent/util-scripts/pbench-tool-meister-start @@ -60,7 +60,7 @@ channel = "tool-meister-chan" REDIS_MAX_WAIT = 60 -class ToolGroup(object): +class ToolGroup: tg_prefix = "tools-v1" def __init__(self, group): diff --git a/agent/util-scripts/pbench-tool-meister-stop b/agent/util-scripts/pbench-tool-meister-stop index 7987022604..5f8088a2a6 100755 --- a/agent/util-scripts/pbench-tool-meister-stop +++ b/agent/util-scripts/pbench-tool-meister-stop @@ -101,7 +101,7 @@ def main(argv): ret_val = 0 logger.debug("terminating %d pids at localhost:%d", expected_pids, redis_port) - terminate_msg = dict(action="terminate", group=None, directory=None) + terminate_msg = dict(action="terminate", group=None, directory=None, args=None) try: num_present = redis_server.publish( channel, json.dumps(terminate_msg, sort_keys=True) diff --git a/agent/util-scripts/test-bin/pbench-sysinfo-dump b/agent/util-scripts/test-bin/pbench-sysinfo-dump deleted file mode 120000 index 7773fb5be0..0000000000 --- a/agent/util-scripts/test-bin/pbench-sysinfo-dump +++ /dev/null @@ -1 +0,0 @@ -mpstat \ No newline at end of file diff --git a/agent/util-scripts/test-bin/pbench-sysinfo-dump b/agent/util-scripts/test-bin/pbench-sysinfo-dump new file mode 100755 index 0000000000..878c4a9de4 --- /dev/null +++ b/agent/util-scripts/test-bin/pbench-sysinfo-dump @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 + +import sys +import os + +from pathlib import Path + + +with open(os.environ["_testlog"], "a") as ofp: + args = " ".join(sys.argv) + ofp.write(f"{args}\n") + +# Mock the behavior of the real pbench-sysinfo-dump + +tgtdir = Path(sys.argv[1]) +sysinfo = sys.argv[2] +try: + label = sys.argv[3] +except IndexError: + label = "" +hostname = os.environ["_pbench_full_hostname"] +hostdir = f"{label}:{hostname}" if label else hostname + +(tgtdir / hostdir).mkdir() + +(tgtdir / hostdir / "contents.lis").write_text(sysinfo) +sys.exit(0) diff --git a/agent/util-scripts/test-bin/ssh b/agent/util-scripts/test-bin/ssh index 2108546374..d4e449358a 100755 --- a/agent/util-scripts/test-bin/ssh +++ b/agent/util-scripts/test-bin/ssh @@ -26,7 +26,7 @@ if [[ "${1}" == "hostname" && "${2}" == "-s" ]]; then exit_code=0 elif [[ "$(basename -- "${1}")" == "pbench-tool-meister-remote" ]]; then _dir=$(dirname ${0}) - _tool_bin=${_dir}/mpstat ${1} localhost "${3}" "${4}" + _pbench_full_hostname="${remote}" _pbench_hostname="${remote}" _tool_bin=${_dir}/mpstat ${1} localhost "${3}" "${4}" exit_code=$? fi diff --git a/agent/util-scripts/test-bin/test-client-tool-meister b/agent/util-scripts/test-bin/test-client-tool-meister index cdcaf1f609..1f1721242b 100755 --- a/agent/util-scripts/test-bin/test-client-tool-meister +++ b/agent/util-scripts/test-bin/test-client-tool-meister @@ -68,6 +68,13 @@ if [[ ${status} -ne 0 ]]; then exit 1 fi +_timeout pbench-collect-sysinfo --group="${group}" --dir="${benchmark_run_dir}" --sysinfo="block,security_mitigations,sos" beg +status=${?} +if [[ ${status} -ne 0 ]]; then + printf -- "ERROR - \"pbench-collect-sysinfo\" failed to execute successfully (exit code: %s)\n" "${status}" >&2 + exit 1 +fi + _timeout pbench-init-tools --group="${group}" --dir="${benchmark_run_dir}" status=${?} if [[ ${status} -ne 0 ]]; then @@ -127,6 +134,13 @@ if [[ ${status} -ne 0 ]]; then exit 1 fi +_timeout pbench-collect-sysinfo --group="${group}" --dir="${benchmark_run_dir}" --sysinfo="block,security_mitigations,sos" end +status=${?} +if [[ ${status} -ne 0 ]]; then + printf -- "ERROR - \"pbench-collect-sysinfo\" failed to execute successfully (exit code: %s)\n" "${status}" >&2 + exit 1 +fi + _timeout pbench-tool-meister-stop status=${?} if [[ ${status} -ne 0 ]]; then diff --git a/agent/util-scripts/unittests b/agent/util-scripts/unittests index 4a84a4c240..c8a4e79f2b 100755 --- a/agent/util-scripts/unittests +++ b/agent/util-scripts/unittests @@ -530,7 +530,7 @@ declare -A pre_hooks=( [test-46]='mkdir ${_testdir}/tmp; printf -- "# bad list\none.example.com\ntwo.example.com,labelTwo\n\nthree.example.com,labelThree,junk\n" > ${_testdir}/tmp/remotes.lis' [test-47]='mkdir ${_testdir}/tmp; printf -- "# good list with no labels\none.example.com\ntwo.example.com\nthree.example.com\n" > ${_testdir}/tmp/remotes.lis' [test-48]='mkdir ${_testdir}/tmp; printf -- "%s\n" "30%" > ${_testdir}/tmp/foo.txt' - [test-55]='pbench-register-tool --name=mpstat --remote=localhost > /dev/null; mkdir ${_testdir}/mock-run' + [test-55]='pbench-register-tool --name=mpstat --remote=localhost > /dev/null; mkdir ${_testdir}/mock-run; ln -s mock-pbench-tool-meister-client ${_testopt}/unittest-scripts/pbench-tool-meister-client' [test-61]='ln -s mock-pbench-tool-meister-client ${_testopt}/unittest-scripts/pbench-tool-meister-client' [test-62]='ln -s mock-pbench-tool-meister-client ${_testopt}/unittest-scripts/pbench-tool-meister-client; mkdir -p ${_testdir}/mock-run/tools-default' ) @@ -551,6 +551,7 @@ declare -A post_hooks=( [test-07]='cat ${_testdir}/42-iter/sample42/tools-foobar/testhost.example.com/postprocess.log >> ${_testout} 2>&1' [test-19]='rm ${_testopt}/unittest-scripts/pbench-tool-meister-client' [test-53]='sort_testlog' + [test-55]='rm ${_testopt}/unittest-scripts/pbench-tool-meister-client' [test-56]='sort_testlog; sort_tdslog' [test-57]='sort_testlog; sort_tdslog' [test-61]='rm ${_testopt}/unittest-scripts/pbench-tool-meister-client' diff --git a/lib/pbench/agent/tool_data_sink.py b/lib/pbench/agent/tool_data_sink.py index 4b162c95d6..62f68a9759 100644 --- a/lib/pbench/agent/tool_data_sink.py +++ b/lib/pbench/agent/tool_data_sink.py @@ -297,7 +297,7 @@ def __init__(self, redis_server, channel, benchmark_run_dir, tool_group, logger) # Initialize internal state self._hostname = os.environ["_pbench_full_hostname"] self.state = None - self.tool_data_ctx = None + self.data_ctx = None self.directory = None self.tool_metadata = toolmetadata.ToolMetadata("redis", redis_server, logger) self._data = None @@ -307,10 +307,17 @@ def __init__(self, redis_server, channel, benchmark_run_dir, tool_group, logger) self._cv = Condition(lock=self._lock) # Setup the Bottle server route and the WSGI server instance. self.route( - "/tool-data//", + "/tool-data//", method="PUT", callback=self.put_document, ) + self.route( + "/sysinfo-data//", + method="PUT", + callback=self.put_document, + ) + # The list of states where we expect Tool Meisters to send data to us. + self._data_states = frozenset(("send", "sysinfo")) self._server = DataSinkWsgiServer(host="0.0.0.0", port=8080, logger=logger) # Setup the Redis server channel subscription logger.debug("pubsub") @@ -494,10 +501,13 @@ def _wait_for_all_data(self): """wait_for_all_data - block the caller until all of the registered tool meisters have sent their data. - Waiting is a no-op for all states except 'send'. In the 'send' state, - we are expecting to hear from all registered tool meisters. + Waiting is a no-op for all states except the "data states" (see + self._data_states). In a "data" state, we are expecting to hear from + all registered tool meisters. """ - assert self.state == "send", f"expected state 'send' not '{self.state}'" + assert ( + self.state in self._data_states + ), f"expected state to be one of {self._data_states!r} not {self.state!r}" assert self._tm_tracking is not None, "Logic bomb! self._tm_tracking is None" done = False @@ -597,7 +607,7 @@ def state_change(self, data): # data we receive to that directory, but expect them to provide the # opaque context in the URL for the PUT method. directory_bytes = directory_str.encode("utf-8") - self.tool_data_ctx = hashlib.md5(directory_bytes).hexdigest() + self.data_ctx = hashlib.md5(directory_bytes).hexdigest() # Transition to "send" state should reset self._tm_tracking with self._lock: @@ -626,7 +636,7 @@ def state_change(self, data): elif self.state == "end": if self._prom_server: self._prom_server.terminate() - elif self.state == "send": + elif self.state in self._data_states: self._change_tm_tracking("dormant", "waiting") # The Tool Data Sink cannot send success until all the Tool # Meisters have sent their collected data, so wait for all the @@ -678,7 +688,7 @@ def _send_client_status(self, status): ret_val = 0 return ret_val - def put_document(self, tool_data_ctx, hostname): + def put_document(self, data_ctx, hostname): """put_document - PUT callback method for Bottle web server end point The put_document method is called by threads serving web requests. @@ -689,16 +699,16 @@ def put_document(self, tool_data_ctx, hostname): """ with self._lock: - if self.state != "send": + if self.state not in self._data_states: # FIXME: Don't we have a race condition if the tool meisters # process their messages first? Seems like we need to send to # the Tool Data Sink first, and then send to all the tool # meisters. abort(400, f"Can't accept PUT requests in state '{self.state}'") - if self.tool_data_ctx != tool_data_ctx: + if self.data_ctx != data_ctx: # Tool Data Sink and this Tool Meister are out of sink as to # what data is expected. - abort(400, f"Unexpected tool data context, '{tool_data_ctx}'") + abort(400, f"Unexpected data context, '{data_ctx}'") # Fetch the Tool Meister tracking record for this host and verify # it is in the expected waiting state. tm_tracker = self._tm_tracking[hostname] @@ -735,11 +745,11 @@ def put_document(self, tool_data_ctx, hostname): target_dir = self.directory if not target_dir.is_dir(): self.logger.error("ERROR - directory, '%s', does not exist", target_dir) - abort(500, f"Invalid URL, path {target_dir} does not exist") - host_tool_data_tb_name = target_dir / f"{hostname}.tar.xz" - if host_tool_data_tb_name.exists(): - abort(409, f"{host_tool_data_tb_name} already uploaded") - host_tool_data_tb_md5 = Path(f"{host_tool_data_tb_name}.md5") + abort(500, "INTERNAL ERROR") + host_data_tb_name = target_dir / f"{hostname}.tar.xz" + if host_data_tb_name.exists(): + abort(409, f"{host_data_tb_name} already uploaded") + host_data_tb_md5 = Path(f"{host_data_tb_name}.md5") with tempfile.NamedTemporaryFile(mode="wb", dir=target_dir) as ofp: total_bytes = 0 @@ -766,45 +776,45 @@ def put_document(self, tool_data_ctx, hostname): # First write the .md5 try: - with host_tool_data_tb_md5.open("w") as md5fp: - md5fp.write(f"{exp_md5} {host_tool_data_tb_name.name}\n") + with host_data_tb_md5.open("w") as md5fp: + md5fp.write(f"{exp_md5} {host_data_tb_name.name}\n") except Exception: try: - os.remove(host_tool_data_tb_md5) + os.remove(host_data_tb_md5) except Exception as exc: self.logger.warning( "Failed to remove .md5 %s when trying to clean up: %s", - host_tool_data_tb_md5, + host_data_tb_md5, exc, ) self.logger.exception( - "Failed to write .md5 file, '%s'", host_tool_data_tb_md5 + "Failed to write .md5 file, '%s'", host_data_tb_md5 ) raise # Then create the final filename link to the temporary file. try: - os.link(ofp.name, host_tool_data_tb_name) + os.link(ofp.name, host_data_tb_name) except Exception: try: - os.remove(host_tool_data_tb_md5) + os.remove(host_data_tb_md5) except Exception as exc: self.logger.warning( "Failed to remove .md5 %s when trying to clean up: %s", - host_tool_data_tb_md5, + host_data_tb_md5, exc, ) self.logger.exception( "Failed to rename tar ball '%s' to '%s'", ofp.name, - host_tool_data_tb_md5, + host_data_tb_md5, ) raise else: self.logger.debug( "Successfully wrote %s (%s.md5)", - host_tool_data_tb_name, - host_tool_data_tb_name, + host_data_tb_name, + host_data_tb_name, ) # Now unpack that tar ball @@ -814,34 +824,32 @@ def put_document(self, tool_data_ctx, hostname): # Invoke tar directly for efficiency. with o_file.open("w") as ofp, e_file.open("w") as efp: cp = subprocess.run( - [tar_path, "-xf", host_tool_data_tb_name], + [tar_path, "-xf", host_data_tb_name], cwd=target_dir, stdin=None, stdout=ofp, stderr=efp, ) except Exception: - self.logger.exception( - "Failed to extract tools tar ball, '%s'", host_tool_data_tb_name - ) - raise + self.logger.exception("Failed to extract tar ball, '%s'", host_data_tb_name) + abort(500, "INTERNAL ERROR") else: if cp.returncode != 0: self.logger.error( - "Failed to create tools tar ball; return code: %d", cp.returncode + "Failed to create tar ball; return code: %d", cp.returncode ) abort(500, "INTERNAL ERROR") else: - self.logger.debug("Successfully unpacked %s", host_tool_data_tb_name) + self.logger.debug("Successfully unpacked %s", host_data_tb_name) try: o_file.unlink() e_file.unlink() - host_tool_data_tb_md5.unlink() - host_tool_data_tb_name.unlink() + host_data_tb_md5.unlink() + host_data_tb_name.unlink() except Exception: self.logger.exception( "Error removing unpacked tar ball '%s' and it's .md5", - host_tool_data_tb_name, + host_data_tb_name, ) # Tell the waiting "watcher" thread engaging in a "state" change diff --git a/lib/pbench/agent/tool_meister.py b/lib/pbench/agent/tool_meister.py index 858fae637d..cb752dabb2 100644 --- a/lib/pbench/agent/tool_meister.py +++ b/lib/pbench/agent/tool_meister.py @@ -32,10 +32,6 @@ and the redis key to fetch its configuration from for its operation. [1] https://redis.io/ - -$ sudo dnf install python3-redis -$ sudo pip3 install python-daemon -$ sudo pip3 install python-pidfile """ import errno @@ -65,6 +61,9 @@ # Path to external tar executable. tar_path = None +# Path to external pbench-sysinfo-dump executable. +sysinfo_dump = None + # FIXME: The client response channel should be in a shared constants module. client_channel = "tool-meister-client" @@ -155,7 +154,7 @@ def stop(self): return 0 -class Tool(object): +class Tool: """Encapsulates all the state needed to manage a tool running as a background process. @@ -280,12 +279,11 @@ class ToolMeisterError(Exception): pass -class ToolMeister(object): +class ToolMeister: """Encapsulate tool life-cycle - The goal of this class is to make sure all necessary state and behaviors - for managing a given tool are handled by the methods offered by the - class. + The goal of this class is to provide the methods and attributes necessary + for managing the life-cycles of a registered set of tools. The start_, stop_, send_, and wait_ prefixed methods represent all the necessary interfaces for managing the life-cycle of a tool. The cleanup() @@ -336,6 +334,7 @@ class ToolMeister(object): Meister is running remotely, then it will use a local temporary directory to write it's data, and will send that data to the Tool Data Sink during the "send" phase. + """ @staticmethod @@ -409,7 +408,7 @@ def __init__(self, pbench_bin, params, redis_server, logger): "stop": {"curr": "running", "next": "idle", "action": self.stop_tools}, } self._valid_actions = frozenset( - ["end", "init", "send", "start", "stop", "terminate"] + ["end", "init", "send", "start", "stop", "sysinfo", "terminate"] ) for key in self._state_trans.keys(): assert ( @@ -419,7 +418,7 @@ def __init__(self, pbench_bin, params, redis_server, logger): "INTERNAL ERROR: invalid state transition 'next' entry for" f" '{key}', '{self._state_trans[key]['next']}'" ) - self._message_keys = frozenset(["directory", "group", "action"]) + self._message_keys = frozenset(["action", "args", "directory", "group"]) # The current 'directory' into which the tools are collected; not set # until a 'start tools' is executed, cleared when a 'send tools' # completes. @@ -431,7 +430,7 @@ def __init__(self, pbench_bin, params, redis_server, logger): # tools for storing their collected data. self._tool_dir = None # The temporary directory to use for capturing all tool data. - self._tmp_dir = os.environ.get("_PBENCH_TOOL_MEISTER_TMP", "/var/tmp") + self._tmp_dir = os.environ["pbench_tmp"] # FIXME: run all the "--install" commands for the tools to ensure # they are successful before declaring that we are ready. @@ -468,33 +467,32 @@ def _get_data(self): # FIXME: Add connection drop error handling, retry loop # re-establishing a connection. self.logger.exception("Error fetching 'next' data off channel") + continue else: self.logger.debug("next success") + msg = None try: json_str = payload["data"].decode("utf-8") - data = json.loads(json_str) + tmp_data = json.loads(json_str) except Exception: - self.logger.warning("data payload in message not JSON, %r", json_str) - data = None + msg = f"data payload in message not JSON, {json_str!r}" else: - keys = frozenset(data.keys()) + keys = frozenset(tmp_data.keys()) if keys != self._message_keys: - self.logger.warning( - "unrecognized keys in data of payload in message, %r", json_str, + msg = ( + f"unrecognized keys in data of payload in message, {json_str!r}" ) - data = None - elif data["action"] not in self._valid_actions: - self.logger.warning( - "unrecognized action in data of payload in message, %r", - json_str, - ) - data = None - elif data["group"] is not None and data["group"] != self._group: - self.logger.warning( - "unrecognized group in data of payload in message, %r", - json_str, - ) - data = None + elif tmp_data["action"] not in self._valid_actions: + msg = f"unrecognized action in data of payload in message, {json_str!r}" + elif tmp_data["group"] is not None and tmp_data["group"] != self._group: + msg = f"unrecognized group in data of payload in message, {json_str!r}" + else: + data = tmp_data + finally: + if msg is not None: + assert data is None + self.logger.warning(msg) + self._send_client_status(msg) return data["action"], data def wait_for_command(self): @@ -507,24 +505,26 @@ def wait_for_command(self): """ self.logger.debug("%s: wait_for_command %s", self._hostname, self.state) action, data = self._get_data() - done = False - while not done: + action_method = None + while not action_method: if action == "terminate": self.logger.debug("%s: msg - %r", self._hostname, data) raise Terminate() if action == "send": - self.logger.debug("%s: msg - %r", self._hostname, data) - return self.send_tools, data + action_method = self.send_tools + continue + if action == "sysinfo": + action_method = self.sysinfo + continue state_trans_rec = self._state_trans[action] if state_trans_rec["curr"] != self.state: - self.logger.info( - "ignoring unexpected data, %r, in state '%s'", data, self.state - ) + msg = f"ignoring unexpected data, {data!r}, in state '{self.state}'" + self.logger.info(msg) + self._send_client_status(msg) action, data = self._get_data() continue - done = True - action_method = state_trans_rec["action"] - self.state = state_trans_rec["next"] + action_method = state_trans_rec["action"] + self.state = state_trans_rec["next"] self.logger.debug("%s: msg - %r", self._hostname, data) return action_method, data @@ -797,52 +797,20 @@ def stop_tools(self, data): self._send_client_status("success") return failures - def send_tools(self, data): - """send_tools - send any collected tool data to the tool data sink. - - The 'action' and 'group' values of the payload have already been - validated before this "send tools" action is invoked. - - This method only proceeds if the 'directory' entry value of the - payload matches what was previously provided to a "start tools" - action. - """ - - if len(set(self._tools.keys()) - set(self.persist_tools)) == 0: - self._send_client_status("success") - return 0 + def _send_directory(self, directory, uri, ctx): + """_send_directory - tar up the given directory and send via PUT to the + URL constructed from the "uri" fragment, using the provided context. - directory = data["directory"] - try: - tool_dir = self.directories[directory] - except KeyError: - self.logger.error( - "INTERNAL ERROR - send tools action encountered for a" - " directory, '%s', that is different from any previous" - " start tools directory, %r", - directory, - self.directories.keys(), - ) - self._send_client_status("internal-error") - return False + The directory argument is a Path object who last element is a + directory with a name that is the same as the self.hostname. - if self._hostname == self._controller: - del self.directories[directory] - self.logger.info( - "%s: send_tools (no-op) %s %s", self._hostname, self._group, tool_dir - ) - # Note that we don't have a directory to send when a Tool - # Meister runs on the same host as the controller. - self._send_client_status("success") - return 0 + The uri and ctx arguments are used to form the final URL as defined by: - assert tool_dir.name == self._hostname, ( - f"Logic Bomb! Final path component of the tool directory is" - f" '{tool_dir.name}', not our host name '{self._hostname}'" - ) + f"http://{self._controller}:8080/{uri}/{ctx}/{self._hostname}" + """ failures = 0 - parent_dir = tool_dir.parent + parent_dir = directory.parent tar_file = parent_dir / f"{self._hostname}.tar.xz" o_file = parent_dir / f"{self._hostname}.tar.out" e_file = parent_dir / f"{self._hostname}.tar.err" @@ -857,22 +825,20 @@ def send_tools(self, data): stderr=efp, ) except Exception: - self.logger.exception("Failed to create tools tar ball '%s'", tar_file) + self.logger.exception("Failed to create tar ball '%s'", tar_file) + failures += 1 else: try: if cp.returncode != 0: self.logger.error( - "Failed to create tools tar ball; return code: %d", - cp.returncode, + "Failed to create tar ball; return code: %d", cp.returncode, ) failures += 1 else: try: tar_md5 = md5sum(tar_file) except Exception: - self.logger.exception( - "Failed to read tools tar ball, '%s'", tar_file - ) + self.logger.exception("Failed to read tar ball, '%s'", tar_file) failures += 1 else: try: @@ -893,17 +859,15 @@ def send_tools(self, data): ) self.logger.debug( - "%s: starting send_tools %s %s", + "%s: starting send_data group=%s, directory=%s", self._hostname, self._group, self._directory, ) headers = {"md5sum": tar_md5} - directory_bytes = data["directory"].encode("utf-8") - tool_data_ctx = hashlib.md5(directory_bytes).hexdigest() url = ( - f"http://{self._controller}:8080/tool-data" - f"/{tool_data_ctx}/{self._hostname}" + f"http://{self._controller}:8080/{uri}" + f"/{ctx}/{self._hostname}" ) sent = False retries = 200 @@ -949,16 +913,16 @@ def send_tools(self, data): parent_dir, ) failures += 1 - else: - del self.directories[directory] self.logger.info( - "%s: send_tools completed %s %s", + "%s: PUT %s completed %s %s", self._hostname, + uri, self._group, - tool_dir, + directory, ) except Exception: self.logger.exception("Unexpected error encountered") + failures += 1 finally: # We always remove the created tar file regardless of success or # failure. The above code should take care of removing the @@ -971,15 +935,76 @@ def send_tools(self, data): except OSError as exc: if exc.errno != errno.ENOENT: self.logger.warning( - "error removing tools tar ball, '%s': %s", tar_file, exc + "error removing tar ball, '%s': %s", tar_file, exc ) except Exception as exc: self.logger.warning( - "unexpected error removing tools tar ball, '%s': %s", tar_file, exc + "unexpected error removing tar ball, '%s': %s", tar_file, exc ) + return failures + + def send_tools(self, data): + """send_tools - send any collected tool data to the tool data sink. + + The 'action' and 'group' values of the payload have already been + validated before this "send tools" action is invoked. + + This method only proceeds if the 'directory' entry value of the + payload matches what was previously provided to a "start tools" + action. + """ + + if self.state in ("running", "startup"): + # The "send tool data" action is only allowed when the Tool + # Meister has left the startup state (received the first "init" at + # least, and is not running any tools. It is a no-op if a "send" + # is issued "send" before any tools were started. + msg = f"send action received in state '{self.state}'" + self._send_client_status(msg) + return 1 + + if len(set(self._tools.keys()) - set(self.persist_tools)) == 0: + self._send_client_status("success") + return 0 + + directory = data["directory"] + try: + tool_dir = self.directories[directory] + except KeyError: + self.logger.error( + "INTERNAL ERROR - send tools action encountered for a" + " directory, '%s', that is different from any previous" + " start tools directory, %r", + directory, + self.directories.keys(), + ) + self._send_client_status("internal-error") + return 1 + + if self._hostname == self._controller: + del self.directories[directory] + self.logger.info( + "%s: send_tools (no-op) %s %s", self._hostname, self._group, tool_dir + ) + # Note that we don't have a directory to send when a Tool + # Meister runs on the same host as the controller. + self._send_client_status("success") + return 0 + + assert tool_dir.name == self._hostname, ( + f"Logic Bomb! Final path component of the tool directory is" + f" '{tool_dir.name}', not our host name '{self._hostname}'" + ) + + directory_bytes = data["directory"].encode("utf-8") + tool_data_ctx = hashlib.md5(directory_bytes).hexdigest() + failures = self._send_directory(tool_dir, "tool-data", tool_data_ctx) + + if failures == 0: + del self.directories[directory] self._send_client_status( - "success" if failures == 0 else "failures sending tool data" + "success" if failures == 0 else f"{failures} failures sending tool data" ) return failures @@ -1014,6 +1039,103 @@ def end_tools(self, data): self._send_client_status("success") return failures + def sysinfo(self, data): + """sysinfo - collect all the sysinfo data for this host.""" + if self.state in ("running", "idle"): + # The "gather system information" action is only allowed when the + # Tool Meister first starts ("startup"), and when it is ready for + # shutting down ("shutdown"). + msg = f"sysinfo action received in state '{self.state}'" + self._send_client_status(msg) + return 1 + + args = data["args"] + if not args: + self._send_client_status("No sysinfo arguments given") + return 1 + + self.logger.debug("sysinfo: %r", args) + # FIXME - Should we perform more checking to validate the argument? + sysinfo = args[0] + # FIXME - We need the label for this registered host + label = "" + + if self._hostname == self._controller: + try: + sysinfo_dir = Path(data["directory"]).resolve(strict=True) + except Exception: + self.logger.exception( + "Failed to access provided sysinfo directory, %s", data["directory"] + ) + self._send_client_status("internal-error") + return 1 + else: + try: + sysinfo_dir = Path( + tempfile.mkdtemp( + dir=self._tmp_dir, prefix=f"tm.{self._group}.{os.getpid()}." + ) + ).resolve(strict=True) + except Exception: + self.logger.exception( + "Failed to create temporary directory for sysinfo operation" + ) + self._send_client_status("internal-error") + return 1 + + global sysinfo_dump + command = [ + sysinfo_dump, + str(sysinfo_dir), + sysinfo, + label, + ] + + self.logger.info("pbench-sysinfo-dump -- %s", " ".join(command)) + + o_file = sysinfo_dir / "tm-sysinfo.out" + e_file = sysinfo_dir / "tm-sysinfo.err" + try: + with o_file.open("w") as ofp, e_file.open("w") as efp: + cp = subprocess.run( + command, cwd=sysinfo_dir, stdin=None, stdout=ofp, stderr=efp, + ) + except Exception as exc: + msg = f"Failed to collect system information: {exc}" + self.logger.exception(msg) + self._send_client_status(msg) + return 1 + else: + if cp.returncode != 0: + msg = f"failed to collect system information; return code: {cp.returncode}" + self.logger.error(msg) + self._send_client_status(msg) + return 1 + + if self._hostname == self._controller: + self.logger.info( + "%s: sysinfo send (no-op) %s %s", + self._hostname, + self._group, + sysinfo_dir, + ) + # Note that we don't have a directory to send when a Tool + # Meister runs on the same host as the controller. + self._send_client_status("success") + return 0 + + directory_bytes = data["directory"].encode("utf-8") + sysinfo_data_ctx = hashlib.md5(directory_bytes).hexdigest() + failures = self._send_directory( + sysinfo_dir / self._hostname, "sysinfo-data", sysinfo_data_ctx + ) + + self._send_client_status( + "success" if failures == 0 else f"{failures} failures sending sysinfo data" + ) + + return failures + def main(argv): """Main program for the Tool Meister. @@ -1046,6 +1168,12 @@ def main(argv): print("External 'tar' executable not found.", file=sys.stderr) return 2 + global sysinfo_dump + sysinfo_dump = find_executable("pbench-sysinfo-dump") + if sysinfo_dump is None: + print("External 'pbench-sysinfo-dump' executable not found.", file=sys.stderr) + return 2 + logger = logging.getLogger(PROG) fh = logging.FileHandler(f"{param_key}.log") if os.environ.get("_PBENCH_UNIT_TESTS"): diff --git a/lib/pbench/agent/toolmetadata.py b/lib/pbench/agent/toolmetadata.py index 6f32c6ead9..78d6473fa1 100644 --- a/lib/pbench/agent/toolmetadata.py +++ b/lib/pbench/agent/toolmetadata.py @@ -69,7 +69,7 @@ def __dataCheck(self): if not self.data: self.data == self.__getInitialData() if not self.data: - self.logger.error(f"Unable to access data through {self.mode}") + self.logger.error("Unable to access data through %s", self.mode) return False return True diff --git a/server/bin/pbench-server-prep-shim-002.py b/server/bin/pbench-server-prep-shim-002.py index 84111270a4..25c394630c 100755 --- a/server/bin/pbench-server-prep-shim-002.py +++ b/server/bin/pbench-server-prep-shim-002.py @@ -27,7 +27,7 @@ _NAME_ = "pbench-server-prep-shim-002" -class Results(object): +class Results: def __init__( self, nstatus="", ntotal=0, ntbs=0, nquarantined=0, ndups=0, nerrs=0, ):