From 78a3b6ba71b68286d99a673b4b2f15f1d0c352aa Mon Sep 17 00:00:00 2001 From: Nick Dokos Date: Wed, 22 Feb 2017 13:32:53 -0500 Subject: [PATCH 1/3] Pbench agent base script: do not recalculate $date $date is set by the base script which is sourced by every benchmark script, tool script and util script. It is used by benchmark scripts to create the results directory for the run, but because it is recalculated every time the base script is sourced, other components were getting a different value and e.g. could not find the results directory for the run. We now check whether $date has a value already and if so, we just leave it alone. It is only set the first time that the base script is sourced. --- agent/base | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/agent/base b/agent/base index bd2fdfb5b5..87eb53f733 100755 --- a/agent/base +++ b/agent/base @@ -83,7 +83,9 @@ function debug_log { # Some standard global vars - try the config file first and fall back on hardwired defaults # which are valid today. if [[ -z "$_PBENCH_BENCH_TESTS" ]]; then - date=`date --utc "+%F_%H:%M:%S"` + if [ -z "$date" ] ;then + export date=`date --utc "+%F_%H:%M:%S"` + fi hostname=`hostname -s` full_hostname=`hostname` else From 1d3508add59319dd4db1da170f52e7ee00466d2a Mon Sep 17 00:00:00 2001 From: Nick Dokos Date: Thu, 20 Oct 2016 12:20:31 -0400 Subject: [PATCH 2/3] All pbench benchmark scripts record their iterations When a benchmark script starts a new iteration, it records it in a temporary file. This file will be used to add an iterations entry into the metadata log, to speed up processing on the server (in particular, indexing into ES). --- agent/bench-scripts/gold/pbench-dbench/test-02.txt | 1 + agent/bench-scripts/gold/pbench-fio/test-04.txt | 1 + agent/bench-scripts/gold/pbench-fio/test-05.txt | 1 + agent/bench-scripts/gold/pbench-fio/test-06.txt | 1 + agent/bench-scripts/gold/pbench-moongen/test-03.txt | 1 + agent/bench-scripts/gold/pbench-uperf/test-00.txt | 1 + agent/bench-scripts/gold/pbench-uperf/test-01.txt | 1 + agent/bench-scripts/pbench-cyclictest | 5 ++++- agent/bench-scripts/pbench-dbench | 6 +++++- agent/bench-scripts/pbench-fio | 13 +++++++++---- agent/bench-scripts/pbench-iozone | 7 +++++-- agent/bench-scripts/pbench-linpack | 6 +++++- agent/bench-scripts/pbench-migrate | 7 ++++++- agent/bench-scripts/pbench-moongen | 7 ++++++- agent/bench-scripts/pbench-netperf | 6 +++++- agent/bench-scripts/pbench-specjbb2005 | 6 +++++- agent/bench-scripts/pbench-uperf | 6 +++++- agent/bench-scripts/pbench-user-benchmark | 6 ++++-- 18 files changed, 66 insertions(+), 16 deletions(-) diff --git a/agent/bench-scripts/gold/pbench-dbench/test-02.txt b/agent/bench-scripts/gold/pbench-dbench/test-02.txt index 39532cebef..e9fc725190 100644 --- a/agent/bench-scripts/gold/pbench-dbench/test-02.txt +++ b/agent/bench-scripts/gold/pbench-dbench/test-02.txt @@ -94,6 +94,7 @@ Iteration 2-48thread complete (2 of 2), with 1 pass and 0 failures /var/tmp/pbench-test-bench/pbench/dbench__1900-01-01_00:00:00/summary-result.txt /var/tmp/pbench-test-bench/pbench/pbench.log /var/tmp/pbench-test-bench/pbench/tmp +/var/tmp/pbench-test-bench/pbench/tmp/dbench__1900-01-01_00:00:00.iterations /var/tmp/pbench-test-bench/pbench/tools-default /var/tmp/pbench-test-bench/pbench/tools-default/mpstat /var/tmp/pbench-test-bench/pbench/tools-default/sar diff --git a/agent/bench-scripts/gold/pbench-fio/test-04.txt b/agent/bench-scripts/gold/pbench-fio/test-04.txt index 30ba1d0442..6690f2a113 100644 --- a/agent/bench-scripts/gold/pbench-fio/test-04.txt +++ b/agent/bench-scripts/gold/pbench-fio/test-04.txt @@ -1105,6 +1105,7 @@ Iteration 6-randrw-1024KiB complete (6 of 6), with 1 pass and 6 failures /var/tmp/pbench-test-bench/pbench/fio__1900-01-01_00:00:00/summary-result.txt /var/tmp/pbench-test-bench/pbench/pbench.log /var/tmp/pbench-test-bench/pbench/tmp +/var/tmp/pbench-test-bench/pbench/tmp/fio__1900-01-01_00:00:00.iterations /var/tmp/pbench-test-bench/pbench/tools-default /var/tmp/pbench-test-bench/pbench/tools-default/mpstat /var/tmp/pbench-test-bench/pbench/tools-default/sar diff --git a/agent/bench-scripts/gold/pbench-fio/test-05.txt b/agent/bench-scripts/gold/pbench-fio/test-05.txt index 0bf1cdcaf6..b266d45c1a 100644 --- a/agent/bench-scripts/gold/pbench-fio/test-05.txt +++ b/agent/bench-scripts/gold/pbench-fio/test-05.txt @@ -561,6 +561,7 @@ Iteration 3-throughput-1024KiB complete (3 of 3), with 1 pass and 6 failures /var/tmp/pbench-test-bench/pbench/fio__1900-01-01_00:00:00/summary-result.txt /var/tmp/pbench-test-bench/pbench/pbench.log /var/tmp/pbench-test-bench/pbench/tmp +/var/tmp/pbench-test-bench/pbench/tmp/fio__1900-01-01_00:00:00.iterations /var/tmp/pbench-test-bench/pbench/tools-default /var/tmp/pbench-test-bench/pbench/tools-default/mpstat /var/tmp/pbench-test-bench/pbench/tools-default/sar diff --git a/agent/bench-scripts/gold/pbench-fio/test-06.txt b/agent/bench-scripts/gold/pbench-fio/test-06.txt index 7018e84d86..94f00026d0 100644 --- a/agent/bench-scripts/gold/pbench-fio/test-06.txt +++ b/agent/bench-scripts/gold/pbench-fio/test-06.txt @@ -562,6 +562,7 @@ Iteration 3-throughput-1024KiB complete (3 of 3), with 1 pass and 6 failures /var/tmp/pbench-test-bench/pbench/fio__1900-01-01_00:00:00/summary-result.txt /var/tmp/pbench-test-bench/pbench/pbench.log /var/tmp/pbench-test-bench/pbench/tmp +/var/tmp/pbench-test-bench/pbench/tmp/fio__1900-01-01_00:00:00.iterations /var/tmp/pbench-test-bench/pbench/tools-default /var/tmp/pbench-test-bench/pbench/tools-default/mpstat /var/tmp/pbench-test-bench/pbench/tools-default/sar diff --git a/agent/bench-scripts/gold/pbench-moongen/test-03.txt b/agent/bench-scripts/gold/pbench-moongen/test-03.txt index 2edfafcc82..701439de0d 100644 --- a/agent/bench-scripts/gold/pbench-moongen/test-03.txt +++ b/agent/bench-scripts/gold/pbench-moongen/test-03.txt @@ -76,6 +76,7 @@ Iteration 4-throughput-unidirec-1024B-1024flows-1pct_drop complete (4 of 4), wit /var/tmp/pbench-test-bench/pbench/moongen__1900-01-01_00:00:00/pbench-moongen.cmd /var/tmp/pbench-test-bench/pbench/pbench.log /var/tmp/pbench-test-bench/pbench/tmp +/var/tmp/pbench-test-bench/pbench/tmp/moongen__1900-01-01_00:00:00.iterations /var/tmp/pbench-test-bench/pbench/tools-default /var/tmp/pbench-test-bench/pbench/tools-default/mpstat /var/tmp/pbench-test-bench/pbench/tools-default/sar diff --git a/agent/bench-scripts/gold/pbench-uperf/test-00.txt b/agent/bench-scripts/gold/pbench-uperf/test-00.txt index cde9233204..0e90e53522 100644 --- a/agent/bench-scripts/gold/pbench-uperf/test-00.txt +++ b/agent/bench-scripts/gold/pbench-uperf/test-00.txt @@ -17,6 +17,7 @@ Iteration 2-tcp_stream-64B-1i complete (2 of 2), with 1 pass and 0 failures /var/tmp/pbench-test-bench/pbench /var/tmp/pbench-test-bench/pbench/pbench.log /var/tmp/pbench-test-bench/pbench/tmp +/var/tmp/pbench-test-bench/pbench/tmp/uperf_test-00_1900-01-01_00:00:00.iterations /var/tmp/pbench-test-bench/pbench/tools-default /var/tmp/pbench-test-bench/pbench/tools-default/mpstat /var/tmp/pbench-test-bench/pbench/tools-default/sar diff --git a/agent/bench-scripts/gold/pbench-uperf/test-01.txt b/agent/bench-scripts/gold/pbench-uperf/test-01.txt index 67c077e4c5..414ddf4584 100644 --- a/agent/bench-scripts/gold/pbench-uperf/test-01.txt +++ b/agent/bench-scripts/gold/pbench-uperf/test-01.txt @@ -25,6 +25,7 @@ Iteration 2-tcp_stream-64B-1i complete (2 of 2), with 1 pass and 0 failures /var/tmp/pbench-test-bench/pbench /var/tmp/pbench-test-bench/pbench/pbench.log /var/tmp/pbench-test-bench/pbench/tmp +/var/tmp/pbench-test-bench/pbench/tmp/uperf_test-01_1900-01-01_00:00:00.iterations /var/tmp/pbench-test-bench/pbench/tools-default /var/tmp/pbench-test-bench/pbench/tools-default/mpstat /var/tmp/pbench-test-bench/pbench/tools-default/sar diff --git a/agent/bench-scripts/pbench-cyclictest b/agent/bench-scripts/pbench-cyclictest index 2b9e5824a9..09af24efe2 100755 --- a/agent/bench-scripts/pbench-cyclictest +++ b/agent/bench-scripts/pbench-cyclictest @@ -104,7 +104,9 @@ if [[ $? -ne 0 ]]; then exit 1 fi -benchmark_run_dir="$pbench_run/${benchmark}_${config}_$date" +benchmark_fullname="${benchmark}_${config}_${date}" +benchmark_run_dir="$pbench_run/${benchmark_fullname}" +benchmark_iterations="$pbench_tmp/${benchmark_fullname}.iterations" benchmark_summary_txt_file="$benchmark_run_dir/$benchmark-summary.txt" benchmark_summary_html_file="$benchmark_run_dir/$benchmark-summary.html" if [ ! -z "$cpu" ]; then @@ -137,6 +139,7 @@ for runtime in `echo $runtimes | sed -e s/,/" "/g`; do if [ ! -z "$cpu" ]; then iteration="$iteration-cpu_$cpu" fi + echo $iteration >> $benchmark_iterations benchmark_results_dir="$benchmark_run_dir/$iteration/reference-result/" result_file=$benchmark_results_dir/result.txt benchmark_cmd_file=$benchmark_results_dir/cyclictest.cmd diff --git a/agent/bench-scripts/pbench-dbench b/agent/bench-scripts/pbench-dbench index a36cd48e79..78296aa976 100755 --- a/agent/bench-scripts/pbench-dbench +++ b/agent/bench-scripts/pbench-dbench @@ -240,14 +240,17 @@ while true; do done if [[ -z "$benchmark_run_dir" ]]; then # We don't have an explicit run directory, construct one - benchmark_run_dir="$pbench_run/${benchmark}_${config}_$date" + benchmark_fullname="${benchmark}_${config}_${date}" + benchmark_run_dir="$pbench_run/${benchmark_fullname}" else # We have an explicit run directory provided by --run-dir, so warn # the user if they also used --config if [[ ! -z "$config" ]]; then warn_log "[$script_name] ignoring --config=\"$config\" in favor of --rundir=\"$benchmark_run_dir\"" fi + benchmark_fullname="$(basename $benchmark_run_dir)" fi +benchmark_iterations="$pbench_tmp/${benchmark_fullname}.iterations" if [ -z "$clients" ]; then clients=127.0.0.1 @@ -304,6 +307,7 @@ for thread_count in `echo $thread_counts | sed -e s/,/" "/g`; do echo "Starting iteration $iteration ($count of $total_iterations)" log "Starting iteration $iteration ($count of $total_iterations)" iteration="${count}-${thread_count}thread" + echo $iteration >> $benchmark_iterations iteration_dir="$benchmark_run_dir/$iteration" result_stddevpct=$maxstddevpct # this test case will get a "do-over" if the stddev is not low enough failures=0 diff --git a/agent/bench-scripts/pbench-fio b/agent/bench-scripts/pbench-fio index 0f37b05ef8..3618f57636 100755 --- a/agent/bench-scripts/pbench-fio +++ b/agent/bench-scripts/pbench-fio @@ -372,14 +372,17 @@ function fio_process_options() { esac done if [ "$postprocess_only" = "n" ]; then - benchmark_run_dir="$pbench_run/${benchmark}_${config}_$date" + benchmark_fullname="${benchmark}_${config}_${date}" + benchmark_run_dir="$pbench_run/${benchmark_fullname}" else if [ -z "$run_dir" ]; then - echo "I need a directory if postprocessing an existing result (--run-dir=)" - else - benchmark_run_dir="$run_dir" + err_log "I need a directory if postprocessing an existing result (--run-dir=)" + exit 1 fi + benchmark_fullname="$(basename $run_dir)" + benchmark_run_dir="$run_dir" fi + benchmark_iterations="$pbench_tmp/${benchmark_fullname}.iterations" verify_tool_group $tool_group } @@ -638,6 +641,8 @@ function fio_run_benchmark() { for block_size in `echo $block_sizes | sed -e s/,/" "/g`; do job_num=1 iteration="${count}-${test_type}-${block_size}KiB" + + echo $iteration >> $benchmark_iterations iteration_dir=$benchmark_run_dir/$iteration result_stddevpct=$maxstddevpct # this test case will get a "do-over" if the stddev is not low enough failures=0 diff --git a/agent/bench-scripts/pbench-iozone b/agent/bench-scripts/pbench-iozone index a14664feb8..795438614c 100755 --- a/agent/bench-scripts/pbench-iozone +++ b/agent/bench-scripts/pbench-iozone @@ -1,4 +1,5 @@ #!/bin/bash +# -*- mode: shell-script; indent-tabs-mode: t; sh-basic-offset: 8; sh-indentation: 8; sh-indent-for-case-alt: + -*- # pbench-iozone 0.01 initial version # TODO @@ -155,7 +156,9 @@ done verify_tool_group $tool_group -benchmark_run_dir="$pbench_run/${benchmark}_${config}_$date" +benchmark_fullname="${benchmark}_${config}_${date}" +benchmark_run_dir="$pbench_run/${benchmark_fullname}" +benchmark_iterations="$pbench_tmp/${benchmark_fullname}.iterations" benchmark_summary_file="$benchmark_run_dir/$benchmark-summary.txt" mkdir -p $benchmark_run_dir/.running @@ -163,7 +166,7 @@ export benchmark config pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir beg benchmark_results_dir="$benchmark_run_dir/$iteration/reference-result" - +echo $iteration >> $benchmark_iterations benchmark_tools_dir="$benchmark_results_dir/tools-$tool_group" mkdir -p $benchmark_results_dir $benchmark_tools_dir diff --git a/agent/bench-scripts/pbench-linpack b/agent/bench-scripts/pbench-linpack index 8dcba79bca..c5e46e0365 100755 --- a/agent/bench-scripts/pbench-linpack +++ b/agent/bench-scripts/pbench-linpack @@ -1,4 +1,5 @@ #!/bin/bash +# -*- mode: shell-script; indent-tabs-mode: t; sh-basic-offset: 8; sh-indentation: 8; sh-indent-for-case-alt: + -*- # This is a script to run the linpack benchmark @@ -106,7 +107,9 @@ else exit 1 fi -benchmark_run_dir="$pbench_run/${benchmark}_${config}_$date" +benchmark_fullname="${benchmark}_${config}_${date}" +benchmark_run_dir="$pbench_run/${benchmark_fullname}" +benchmark_iterations="$pbench_tmp/${benchmark_fullname}.iterations" benchmark_summary_txt_file="$benchmark_run_dir/$benchmark-summary.txt" benchmark_summary_html_file="$benchmark_run_dir/$benchmark-summary.html" @@ -118,6 +121,7 @@ result_file=$benchmark_results_dir/result.txt mkdir -p $benchmark_results_dir export benchmark config pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir beg +echo $iteration >> $benchmark_iterations ## Run the benchmark and start/stop perf analysis tools pbench-start-tools --group=$tool_group --iteration=$iteration --dir=$benchmark_results_dir pushd /usr/local/linpack_11.1.3/benchmarks/linpack diff --git a/agent/bench-scripts/pbench-migrate b/agent/bench-scripts/pbench-migrate index 94e91e2412..aef4c7d0b5 100755 --- a/agent/bench-scripts/pbench-migrate +++ b/agent/bench-scripts/pbench-migrate @@ -1,4 +1,5 @@ #!/bin/bash +# -*- mode: shell-script; indent-tabs-mode: t; sh-basic-offset: 8; sh-indentation: 8; sh-indent-for-case-alt: + -*- # This is a script to run the migrate benchmark # Author: Andrew Theurer @@ -309,11 +310,14 @@ function do_migrate() { if [ "$postprocess" = "only" ]; then # user provides directory for previous test + benchmark_fullname=$(basename $benchmark_data_dir) benchmark_run_dir="$benchmark_data_dir" else config="from_${src_host}_to_${dest_host}--$config" - benchmark_run_dir="$pbench_run/${benchmark}_${config}_$date" + benchmark_fullname="${benchmark}_${config}_${date}" + benchmark_run_dir="$pbench_run/${benchmark_fullname}" fi +benchmark_iterations="$pbench_tmp/${benchmark_fullname}.iterations" benchmark_summary_txt_file="$benchmark_run_dir/$benchmark-summary.txt" benchmark_summary_html_file="$benchmark_run_dir/$benchmark-summary.html" @@ -328,6 +332,7 @@ export benchmark config pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir beg for i in `seq 1 $loops`; do iteration="$count-from-$src_host-to-$dest_host" + echo $iteration >> $benchmark_iterations benchmark_results_dir="$benchmark_run_dir/$iteration/reference-result" if [ "$postprocess" != "only" ]; then mkdir -p $benchmark_results_dir diff --git a/agent/bench-scripts/pbench-moongen b/agent/bench-scripts/pbench-moongen index 005409b24a..aae74b8287 100755 --- a/agent/bench-scripts/pbench-moongen +++ b/agent/bench-scripts/pbench-moongen @@ -1,4 +1,5 @@ #!/bin/bash +# -*- mode: shell-script; indent-tabs-mode: t; sh-basic-offset: 8; sh-indentation: 8; sh-indent-for-case-alt: + -*- # This is a script to run the moongen benchmark # Author: Andrew Theurer @@ -305,14 +306,17 @@ while true; do done if [[ -z "$benchmark_run_dir" ]]; then # We don't have an explicit run directory, construct one - benchmark_run_dir="$pbench_run/${benchmark_name}_${config}_$date" + benchmark_fullname="${benchmark_name}_${config}_${date}" + benchmark_run_dir="$pbench_run/${benchmark_fullname}" else # We have an explicit run directory provided by --run-dir, so warn # the user if they also used --config if [[ ! -z "$config" ]]; then warn_log "[$script_name] ignoring --config=\"$config\" in favor of --rundir=\"$benchmark_run_dir\"" fi + benchmark_fullname=$(basename $benchmark_run_dir) fi +benchmark_iterations="$pbench_tmp/${benchmark_fullname}.iterations" mkdir -p $benchmark_run_dir/.running # save a copy of the command, in case the test needs to be reproduced or post-processed again echo "$script_name $pbench_cmd" >$benchmark_run_dir/$script_name.cmd @@ -355,6 +359,7 @@ for rate in `echo $rates | sed -e s/,/" "/g`; do else iteration="${count}-${test_type}-${traffic_direction}-${frame_size}B-${nr_flow}flows-${max_drop_pct}pct_drop" fi + echo $iteration >> $benchmark_iterations iteration_dir="$benchmark_run_dir/$iteration" result_stddevpct=$maxstddevpct # this test case will get a "do-over" if the stddev is not low enough failures=0 diff --git a/agent/bench-scripts/pbench-netperf b/agent/bench-scripts/pbench-netperf index cf6a6b4a10..7454e791ad 100755 --- a/agent/bench-scripts/pbench-netperf +++ b/agent/bench-scripts/pbench-netperf @@ -1,4 +1,5 @@ #!/bin/bash +# -*- mode: shell-script; indent-tabs-mode: t; sh-basic-offset: 8; sh-indentation: 8; sh-indent-for-case-alt: + -*- # This is a script to run the netperf benchmark # Note: This script is under development, and since netperf and uperf have very similar @@ -51,7 +52,9 @@ spacing=25 # Defaults -benchmark_run_dir="$pbench_run/${benchmark}_${config}_$date" +benchmark_fullname="${benchmark}_${config}_${date}" +benchmark_run_dir="$pbench_run/${benchmark_fullname}" +benchmark_iterations="$pbench_tmp/${benchmark_fullname}.iterations" protocols="tcp,udp" test_types="stream,maerts,bidirec,rr" message_sizes="1,64,1024,16384" # in bytes @@ -410,6 +413,7 @@ for protocol in `echo $protocols | sed -e s/,/" "/g`; do for instance in `echo $instances | sed -e s/,/" "/g`; do if [ $count -ge $start_iteration_num ]; then iteration="${count}-${protocol}_${test_type}-${message_size}B-${instance}i" + echo $iteration >> $benchmark_iterations iteration_dir="$benchmark_run_dir/$iteration" result_stddevpct=$maxstddevpct # this test case will get a "do-over" if the stddev is not low enough failures=0 diff --git a/agent/bench-scripts/pbench-specjbb2005 b/agent/bench-scripts/pbench-specjbb2005 index b60e5af776..24b98dc560 100755 --- a/agent/bench-scripts/pbench-specjbb2005 +++ b/agent/bench-scripts/pbench-specjbb2005 @@ -1,4 +1,5 @@ #!/bin/bash +# -*- mode: shell-script; indent-tabs-mode: t; sh-basic-offset: 8; sh-indentation: 8; sh-indent-for-case-alt: + -*- # This is a script to run the specjbb2005 benchmark # Author: Andrew Theurer @@ -158,7 +159,9 @@ if [ "$stop_warehouses" == "default" ]; then fi ## Ensure the right version of the benchmark is installed -benchmark_run_dir="$pbench_run/${benchmark}_${config}_$date" +benchmark_fullname="${benchmark}_${config}_${date}" +benchmark_run_dir="$pbench_run/${benchmark_fullname}" +benchmark_iterations="$pbench_tmp/${benchmark_fullname}.iterations" benchmark_summary_txt_file="$benchmark_run_dir/$benchmark-summary.txt" benchmark_summary_html_file="$benchmark_run_dir/$benchmark-summary.html" benchmark_results_dir="$benchmark_run_dir" @@ -214,6 +217,7 @@ for i in `seq $start_warehouses $inc_warehouses $stop_warehouses`; do # for each of the test iterations (each test with a specific number of warehouses), copy the raw result data for that iteration in iteraration directory grep "test$i" SPECjbb.raw >$i/reference-result/SPECjbb.raw new_iteration_name="$iteration-warehouses:$i" + echo $new_iteration_name >> $benchmark_iterations mv $iteration $new_iteration_name pbench-postprocess-tools --group=$tool_group --iteration=$new_iteration_name --dir=$benchmark_results_dir/$new_iteration_name/reference-result let iteration=$iteration+1 diff --git a/agent/bench-scripts/pbench-uperf b/agent/bench-scripts/pbench-uperf index effff39cf1..780110c8ba 100755 --- a/agent/bench-scripts/pbench-uperf +++ b/agent/bench-scripts/pbench-uperf @@ -367,14 +367,17 @@ while true; do done if [[ -z "$benchmark_run_dir" ]]; then # We don't have an explicit run directory, construct one - benchmark_run_dir="$pbench_run/${benchmark}_${config}_$date" + benchmark_fullname="${benchmark}_${config}_${date}" + benchmark_run_dir="$pbench_run/${benchmark_fullname}" else # We have an explicit run directory provided by --run-dir, so warn # the user if they also used --config if [[ ! -z "$config" ]]; then warn_log "[$script_name] ignoring --config=\"$config\" in favor of --rundir=\"$benchmark_run_dir\"" fi + benchmark_fullname=$(basename $benchmark_run_dir) fi +benchmark_iterations="$pbench_tmp/${benchmark_fullname}.iterations" # Verify the tool group exists verify_tool_group $tool_group @@ -472,6 +475,7 @@ for protocol in `echo $protocols | sed -e s/,/" "/g`; do echo "Starting iteration $iteration ($count of $total_iterations)" log "Starting iteration $iteration ($count of $total_iterations)" iteration="${count}-${protocol}_${test_type}-${message_size}B-${instance}i" + echo $iteration >> $benchmark_iterations iteration_dir="$benchmark_run_dir/$iteration" result_stddevpct=$maxstddevpct # this test case will get a "do-over" if the stddev is not low enough failures=0 diff --git a/agent/bench-scripts/pbench-user-benchmark b/agent/bench-scripts/pbench-user-benchmark index e625367cdc..195956f391 100755 --- a/agent/bench-scripts/pbench-user-benchmark +++ b/agent/bench-scripts/pbench-user-benchmark @@ -75,14 +75,15 @@ done verify_tool_group $tool_group iteration="1" # there can be only 1 iteration for user-benchmark benchmark_bin="$@" -benchmark_run_dir="$pbench_run/${benchmark}_${config}_$date" +benchmark_fullname="${benchmark}_${config}_${date}" +benchmark_run_dir="$pbench_run/${benchmark_fullname}" +benchmark_iterations="$pbench_tmp/${benchmark_fullname}.iterations" # make the run dir available to the user script export benchmark_run_dir benchmark_results_dir="$benchmark_run_dir/$iteration/reference-result" mkdir -p $benchmark_results_dir/.running export benchmark_results_dir=$benchmark_results_dir - # For now, since we only ever run one iteration here, don't bother # collecting sysinfo data before and after, just do it after. # pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir beg @@ -100,6 +101,7 @@ trap "pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir int" INT pbench-start-tools --group=$tool_group --iteration=$iteration --dir=$benchmark_results_dir echo "Running $benchmark_bin" log "[$script_name] Running $benchmark_bin" +echo $iteration >> $benchmark_iterations $benchmark_bin 2>&1 | tee $benchmark_results_dir/result.txt pbench-stop-tools --group=$tool_group --iteration=$iteration --dir=$benchmark_results_dir pbench-postprocess-tools --group=$tool_group --iteration=$iteration --dir=$benchmark_results_dir From fc479c457d0017a53afdc3112a64e2bef150a62f Mon Sep 17 00:00:00 2001 From: Nick Dokos Date: Thu, 20 Oct 2016 12:25:37 -0400 Subject: [PATCH 3/3] pbench-metadata-log changes to record iterations The benchmark scripts record their iterations in a temp file. pbench-metadata-log uses that to add an iterations entry to the metadata log. pbench-add-metalog-option is a generic script that is used for that purpose, but it can add an arbitrary option to an arbitrary section of the metadata log. Modify unit tests and add a new one to test this functionality. --- agent/bench-scripts/test-bin/ssh | 12 +- .../gold/pbench-metadata-log/test-01.txt | 6 - .../gold/pbench-metadata-log/test-02.txt | 1 + .../gold/pbench-metadata-log/test-03.txt | 3 + .../gold/pbench-metadata-log/test-04.txt | 1 + .../gold/pbench-metadata-log/test-21.txt | 76 ++++++++++++ agent/util-scripts/pbench-add-metalog-option | 33 +++++ agent/util-scripts/pbench-metadata-log | 114 ++++++------------ .../test-21/tmp/pbench.iterations | 3 + .../test-21/tools-default/iostat | 1 + .../test-21/tools-default/mpstat | 1 + .../test-21/tools-default/perf | 5 + .../test-21/tools-default/pidstat | 1 + .../test-21/tools-default/proc-interrupts | 1 + .../test-21/tools-default/proc-vmstat | 1 + .../test-21/tools-default/sar | 1 + .../test-21/tools-default/turbostat | 1 + agent/util-scripts/unittests | 9 +- 18 files changed, 181 insertions(+), 89 deletions(-) create mode 100644 agent/util-scripts/gold/pbench-metadata-log/test-21.txt create mode 100755 agent/util-scripts/pbench-add-metalog-option create mode 100644 agent/util-scripts/samples/pbench-metadata-log/test-21/tmp/pbench.iterations create mode 100644 agent/util-scripts/samples/pbench-metadata-log/test-21/tools-default/iostat create mode 100644 agent/util-scripts/samples/pbench-metadata-log/test-21/tools-default/mpstat create mode 100644 agent/util-scripts/samples/pbench-metadata-log/test-21/tools-default/perf create mode 100644 agent/util-scripts/samples/pbench-metadata-log/test-21/tools-default/pidstat create mode 100644 agent/util-scripts/samples/pbench-metadata-log/test-21/tools-default/proc-interrupts create mode 100644 agent/util-scripts/samples/pbench-metadata-log/test-21/tools-default/proc-vmstat create mode 100644 agent/util-scripts/samples/pbench-metadata-log/test-21/tools-default/sar create mode 100644 agent/util-scripts/samples/pbench-metadata-log/test-21/tools-default/turbostat diff --git a/agent/bench-scripts/test-bin/ssh b/agent/bench-scripts/test-bin/ssh index 5970ea4657..d887bce146 100755 --- a/agent/bench-scripts/test-bin/ssh +++ b/agent/bench-scripts/test-bin/ssh @@ -2,10 +2,6 @@ echo "$0 $*" >> $_testlog -if [[ "$4" == "netstat" && "$5" == "-tlpn" ]]; then - echo "tcp 0 0 0.0.0.0:21000 0.0.0.0:* LISTEN 5830/uperf" -fi - while true ;do case $1 in -o) @@ -21,6 +17,14 @@ while true ;do done remote=$1 +if [[ "$2" == "netstat" && "$3" == "-tlnp" ]]; then + echo "tcp 0 0 0.0.0.0:21000 0.0.0.0:* LISTEN 5830/uperf" +fi + +if [[ "$2" == "hostname" && "$3" == "-s" ]]; then + echo "$remote" +fi + if [[ "$remote" == "fubar" ]] ;then exit 255 else diff --git a/agent/util-scripts/gold/pbench-metadata-log/test-01.txt b/agent/util-scripts/gold/pbench-metadata-log/test-01.txt index 8327c6de53..b38183f6f9 100644 --- a/agent/util-scripts/gold/pbench-metadata-log/test-01.txt +++ b/agent/util-scripts/gold/pbench-metadata-log/test-01.txt @@ -15,12 +15,6 @@ config: date: 1900-01-01_00:00:00 rpm-version: -[tools] -hosts: testhost - -[tools/testhost] -mpstat: --interval="10" - [run] controller: testhost.example.com --- pbench tree state diff --git a/agent/util-scripts/gold/pbench-metadata-log/test-02.txt b/agent/util-scripts/gold/pbench-metadata-log/test-02.txt index 75490958ea..7ecd9b7256 100644 --- a/agent/util-scripts/gold/pbench-metadata-log/test-02.txt +++ b/agent/util-scripts/gold/pbench-metadata-log/test-02.txt @@ -39,6 +39,7 @@ rpm-version: [tools] hosts: testhost +group: default [tools/testhost] iostat: --interval=3 diff --git a/agent/util-scripts/gold/pbench-metadata-log/test-03.txt b/agent/util-scripts/gold/pbench-metadata-log/test-03.txt index d8d5a50299..66650f43a3 100644 --- a/agent/util-scripts/gold/pbench-metadata-log/test-03.txt +++ b/agent/util-scripts/gold/pbench-metadata-log/test-03.txt @@ -48,6 +48,7 @@ rpm-version: [tools] hosts: testhost dhcp31-174 perf104 +group: default [tools/testhost] iostat: --interval=3 @@ -74,6 +75,8 @@ grep: /var/tmp/pbench-test-utils/pbench/pbench.log: No such file or directory --- pbench.log file contents +++ test-execution.log file contents /var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/yum list installed pbench-agent +/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/ssh -o StrictHostKeyChecking=no -n dhcp31-174 hostname -s +/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/ssh -o StrictHostKeyChecking=no -n perf104 hostname -s /var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/ssh -o StrictHostKeyChecking=no dhcp31-174 /bin/ls /var/tmp/pbench-test-utils/pbench/tools-default /var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/ssh -o StrictHostKeyChecking=no perf104 /bin/ls /var/tmp/pbench-test-utils/pbench/tools-default --- test-execution.log file contents diff --git a/agent/util-scripts/gold/pbench-metadata-log/test-04.txt b/agent/util-scripts/gold/pbench-metadata-log/test-04.txt index dc93f9dccd..a582ae43f4 100644 --- a/agent/util-scripts/gold/pbench-metadata-log/test-04.txt +++ b/agent/util-scripts/gold/pbench-metadata-log/test-04.txt @@ -43,6 +43,7 @@ rpm-version: [tools] hosts: testhost +group: default [tools/testhost] iostat: --interval=3 diff --git a/agent/util-scripts/gold/pbench-metadata-log/test-21.txt b/agent/util-scripts/gold/pbench-metadata-log/test-21.txt new file mode 100644 index 0000000000..16afeb41c9 --- /dev/null +++ b/agent/util-scripts/gold/pbench-metadata-log/test-21.txt @@ -0,0 +1,76 @@ ++++ Running test-21 pbench-metadata-log +--- Finished test-21 pbench-metadata-log (status=0} ++++ pbench tree state +/var/tmp/pbench-test-utils/pbench +/var/tmp/pbench-test-utils/pbench/metadata.log +/var/tmp/pbench-test-utils/pbench/tmp +/var/tmp/pbench-test-utils/pbench/tmp/pbench.iterations +/var/tmp/pbench-test-utils/pbench/tools-default +/var/tmp/pbench-test-utils/pbench/tools-default/iostat +/var/tmp/pbench-test-utils/pbench/tools-default/mpstat +/var/tmp/pbench-test-utils/pbench/tools-default/perf +/var/tmp/pbench-test-utils/pbench/tools-default/pidstat +/var/tmp/pbench-test-utils/pbench/tools-default/proc-interrupts +/var/tmp/pbench-test-utils/pbench/tools-default/proc-vmstat +/var/tmp/pbench-test-utils/pbench/tools-default/sar +/var/tmp/pbench-test-utils/pbench/tools-default/turbostat +/var/tmp/pbench-test-utils/pbench/tools-default/iostat: +--interval=3 +/var/tmp/pbench-test-utils/pbench/tools-default/mpstat: +--interval=3 +/var/tmp/pbench-test-utils/pbench/tools-default/perf: +--record-opts=-a -freq=100 +--record-opts=--call-graph +--record-opts=--event=branch-misses +--record-opts=--event=cache-misses +--record-opts=--event=instructions +/var/tmp/pbench-test-utils/pbench/tools-default/pidstat: +--interval=3 +/var/tmp/pbench-test-utils/pbench/tools-default/proc-interrupts: +--interval=3 +/var/tmp/pbench-test-utils/pbench/tools-default/proc-vmstat: +--interval=3 +/var/tmp/pbench-test-utils/pbench/tools-default/sar: +--interval=3 +/var/tmp/pbench-test-utils/pbench/tools-default/turbostat: +--interval=3 +/var/tmp/pbench-test-utils/pbench/tmp/pbench.iterations: +iter1 +iter2 +iter3 +--- pbench tree state ++++ metadata.log file contents +/var/tmp/pbench-test-utils/pbench/metadata.log:[pbench] +/var/tmp/pbench-test-utils/pbench/metadata.log:name = pbench +/var/tmp/pbench-test-utils/pbench/metadata.log:script = +/var/tmp/pbench-test-utils/pbench/metadata.log:config = +/var/tmp/pbench-test-utils/pbench/metadata.log:date = 1900-01-01_00:00:00 +/var/tmp/pbench-test-utils/pbench/metadata.log:rpm-version = +/var/tmp/pbench-test-utils/pbench/metadata.log:iterations = iter1, iter2, iter3 +/var/tmp/pbench-test-utils/pbench/metadata.log: +/var/tmp/pbench-test-utils/pbench/metadata.log:[tools] +/var/tmp/pbench-test-utils/pbench/metadata.log:hosts = testhost +/var/tmp/pbench-test-utils/pbench/metadata.log:group = default +/var/tmp/pbench-test-utils/pbench/metadata.log: +/var/tmp/pbench-test-utils/pbench/metadata.log:[tools/testhost] +/var/tmp/pbench-test-utils/pbench/metadata.log:iostat = --interval=3 +/var/tmp/pbench-test-utils/pbench/metadata.log:mpstat = --interval=3 +/var/tmp/pbench-test-utils/pbench/metadata.log:perf = --record-opts=-a -freq=100 --record-opts=--call-graph --record-opts=--event=branch-misses --record-opts=--event=cache-misses --record-opts=--event=instructions +/var/tmp/pbench-test-utils/pbench/metadata.log:pidstat = --interval=3 +/var/tmp/pbench-test-utils/pbench/metadata.log:proc-interrupts = --interval=3 +/var/tmp/pbench-test-utils/pbench/metadata.log:proc-vmstat = --interval=3 +/var/tmp/pbench-test-utils/pbench/metadata.log:sar = --interval=3 +/var/tmp/pbench-test-utils/pbench/metadata.log:turbostat = --interval=3 +/var/tmp/pbench-test-utils/pbench/metadata.log: +/var/tmp/pbench-test-utils/pbench/metadata.log:[run] +/var/tmp/pbench-test-utils/pbench/metadata.log:controller = testhost.example.com +/var/tmp/pbench-test-utils/pbench/metadata.log:start_run = 1900-01-01T00:00:00.000000 +/var/tmp/pbench-test-utils/pbench/metadata.log:end_run = 1900-01-01T00:00:00.000000 +/var/tmp/pbench-test-utils/pbench/metadata.log: +--- metadata.log file contents ++++ pbench.log file contents +grep: /var/tmp/pbench-test-utils/pbench/pbench.log: No such file or directory +--- pbench.log file contents ++++ test-execution.log file contents +/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/yum list installed pbench-agent +--- test-execution.log file contents diff --git a/agent/util-scripts/pbench-add-metalog-option b/agent/util-scripts/pbench-add-metalog-option new file mode 100755 index 0000000000..3b4b4139e7 --- /dev/null +++ b/agent/util-scripts/pbench-add-metalog-option @@ -0,0 +1,33 @@ +#! /usr/bin/env python2 + +# Usage: pbench-add-metalog-option