From 967b9b852151c6d1834dc8c1b7462c911eeadfef Mon Sep 17 00:00:00 2001 From: Peter Portante Date: Tue, 22 Sep 2020 19:01:30 -0400 Subject: [PATCH 1/5] Backport black and flake8 version lock See PR #1768, #d382a8da2b8bdac4e88fe0f6da24ec31fc6d1051. --- .gitignore | 2 ++ test-requirements.txt | 2 +- tox.ini | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 2f78cd0c8a..653764fa87 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,5 @@ dist *.egg-info .tox .eggs +Pipfile +Pipfile.lock diff --git a/test-requirements.txt b/test-requirements.txt index de63c055b0..fba1b3739c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,2 +1,2 @@ -flake8 +flake8==3.8.3 pre-commit diff --git a/tox.ini b/tox.ini index 8748a2edce..bfa787ab9f 100644 --- a/tox.ini +++ b/tox.ini @@ -42,7 +42,7 @@ commands = black --check . flake8 . deps = - black - flake8 + black==19.10b0 + flake8==3.8.3 skip_install = true usedevelop = false From d953463575f99a6d20789d484aba3f1af27f9834 Mon Sep 17 00:00:00 2001 From: Peter Portante Date: Tue, 22 Sep 2020 19:40:18 -0400 Subject: [PATCH 2/5] Stablize the cpuacct-datalog test This is a back-port of #bf382c23f2ddd1a1a97a6c035fbff2cc6f3c9313, from PR #1733. --- agent/tool-scripts/datalog/cpuacct-datalog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/tool-scripts/datalog/cpuacct-datalog b/agent/tool-scripts/datalog/cpuacct-datalog index 786b321a86..5d4fb4cf97 100755 --- a/agent/tool-scripts/datalog/cpuacct-datalog +++ b/agent/tool-scripts/datalog/cpuacct-datalog @@ -20,7 +20,7 @@ if [[ ${?} -ne 0 ]]; then exit 1 fi -dirs="$(find -mindepth 1 -type d)" +dirs="$(find -mindepth 1 -type d | sort)" if [[ -z "${dirs}" ]]; then printf -- "%s: CPU accounting appears disabled, no sub-directories found\n" "${PROG}" >&2 exit 1 From 36a394821500e30d9103614d53067aa7d099aff2 Mon Sep 17 00:00:00 2001 From: Peter Portante Date: Tue, 22 Sep 2020 21:17:59 -0400 Subject: [PATCH 3/5] Stabilize server test-27 This is a back-port of #47edfa80d9 (PR #1734). --- server/bin/pbench-cull-unpacked-tarballs.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/server/bin/pbench-cull-unpacked-tarballs.py b/server/bin/pbench-cull-unpacked-tarballs.py index e255af9435..4deb7de3d6 100755 --- a/server/bin/pbench-cull-unpacked-tarballs.py +++ b/server/bin/pbench-cull-unpacked-tarballs.py @@ -419,9 +419,13 @@ def main(options): actions_taken = [] errors = 0 start = pbench._time() - for tb_incoming_dir, controller_name in gen_list_unpacked_aged( - incoming_p, archive_p, curr_dt, max_unpacked_age - ): + + gen = gen_list_unpacked_aged(incoming_p, archive_p, curr_dt, max_unpacked_age) + if config._unittests: + # force the generator and sort the list + gen = sorted(list(gen)) + + for tb_incoming_dir, controller_name in gen: act_set = remove_unpacked( tb_incoming_dir, controller_name, @@ -457,7 +461,7 @@ def main(options): ) if total > 0: print("\nActions Taken:", file=tfp) - for act_set in actions_taken: + for act_set in sorted(actions_taken, key=lambda a: a.name): print( f" - {act_set.name} ({act_set.errors:d} errors," f" {act_set.duration():0.2f} secs)", From f822773b0b8681e434db5c1f987b03900adceb0f Mon Sep 17 00:00:00 2001 From: Peter Portante Date: Tue, 22 Sep 2020 19:53:46 -0400 Subject: [PATCH 4/5] Ensure tool option order is preserved When `pbench-register-tool` accepts command line arguments to add as tool options, the order of those options received needs to be preserved. The code needs to use indexed arrays properly to preserve order. It was trying to use associative arrays as indexed. We also add a unit test for `pbench-start-tools`, using the `tcpdump` command with two arguments registered. --- .../gold/pbench-register-tool/test-52.txt | 17 ++++++++++ .../gold/pbench-start-tools/test-51.txt | 33 +++++++++++++++++++ agent/util-scripts/pbench-register-tool | 6 ++-- .../test-51/pbench/tools-default/tcpdump | 2 ++ agent/util-scripts/unittests | 4 +++ 5 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 agent/util-scripts/gold/pbench-register-tool/test-52.txt create mode 100644 agent/util-scripts/gold/pbench-start-tools/test-51.txt create mode 100644 agent/util-scripts/samples/pbench-start-tools/test-51/pbench/tools-default/tcpdump diff --git a/agent/util-scripts/gold/pbench-register-tool/test-52.txt b/agent/util-scripts/gold/pbench-register-tool/test-52.txt new file mode 100644 index 0000000000..e3b841c099 --- /dev/null +++ b/agent/util-scripts/gold/pbench-register-tool/test-52.txt @@ -0,0 +1,17 @@ ++++ Running test-52 pbench-register-tool --name=tcpdump --group=default --no-install -- --interface=br1 --packets=1000 +tcpdump tool is now registered in group default +--- Finished test-52 pbench-register-tool (status=0) ++++ pbench tree state +/var/tmp/pbench-test-utils/pbench +/var/tmp/pbench-test-utils/pbench/pbench.log +/var/tmp/pbench-test-utils/pbench/tmp +/var/tmp/pbench-test-utils/pbench/tools-default +/var/tmp/pbench-test-utils/pbench/tools-default/tcpdump +=== /var/tmp/pbench-test-utils/pbench/tools-default/tcpdump: +--interface=br1 +--packets=1000 +--- pbench tree state ++++ pbench.log file contents +[debug][1900-01-01T00:00:00.000000] tool_opts: "--interface=br1 --packets=1000" +[info][1900-01-01T00:00:00.000000] tcpdump tool is now registered in group default +--- pbench.log file contents diff --git a/agent/util-scripts/gold/pbench-start-tools/test-51.txt b/agent/util-scripts/gold/pbench-start-tools/test-51.txt new file mode 100644 index 0000000000..fc3a1af6a0 --- /dev/null +++ b/agent/util-scripts/gold/pbench-start-tools/test-51.txt @@ -0,0 +1,33 @@ ++++ Running test-51 pbench-start-tools --dir=/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample --group=default +--- Finished test-51 pbench-start-tools (status=0) ++++ pbench tree state +/var/tmp/pbench-test-utils/pbench +/var/tmp/pbench-test-utils/pbench/mock-run +/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration +/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample +/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample/tools-default +/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample/tools-default/.screen.d +/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample/tools-default/.screen.d/default-tcpdump +/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample/tools-default/.screen.d/default-tcpdump/command +/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample/tools-default/tcpdump +/var/tmp/pbench-test-utils/pbench/pbench.log +/var/tmp/pbench-test-utils/pbench/tmp +/var/tmp/pbench-test-utils/pbench/tools-default +/var/tmp/pbench-test-utils/pbench/tools-default/tcpdump +=== /var/tmp/pbench-test-utils/pbench/tools-default/tcpdump: +--interface=br1 +--packets=1000 +--- pbench tree state ++++ pbench.log file contents +[debug][1900-01-01T00:00:00.000000] [pbench-start-tools]started: --dir=/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample --group=default +[debug][1900-01-01T00:00:00.000000] [pbench-kill-tools]started: --dir /var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample --group default +[debug][1900-01-01T00:00:00.000000] [pbench-kill-tools] /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/tcpdump --kill --dir=/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample/tools-default --interface=br1 --packets=1000 +[debug][1900-01-01T00:00:00.000000] [pbench-kill-tools]completed: +[debug][1900-01-01T00:00:00.000000] [pbench-start-tools] /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/tcpdump --start --dir=/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample/tools-default --interface=br1 --packets=1000 +[debug][1900-01-01T00:00:00.000000] [pbench-start-tools] "#!/bin/bash screen -dm -L -S "pbench-tool-default-tcpdump" /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/tcpdump --start --dir=/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample/tools-default --interface=br1" +[debug][1900-01-01T00:00:00.000000] [pbench-start-tools]completed: +--- pbench.log file contents ++++ test-execution.log file contents +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/screen -ls +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/screen -dm -L -S pbench-tool-default-tcpdump /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/tcpdump --start --dir=/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample/tools-default --interface=br1 +--- test-execution.log file contents diff --git a/agent/util-scripts/pbench-register-tool b/agent/util-scripts/pbench-register-tool index 79da8e469a..4472801058 100755 --- a/agent/util-scripts/pbench-register-tool +++ b/agent/util-scripts/pbench-register-tool @@ -139,7 +139,7 @@ if [[ ${remotes_arg::1} == "@" ]]; then usage >&2 exit 1 fi - declare -A remotes_A + declare -a remotes_A declare -A labels_A idx=0 lineno=0 @@ -216,7 +216,7 @@ fi # The remainder of ${@} is now tool-specific options. The best way to not # mess up arguments seems to be using an array. idx=0 -declare -A tool_opts +declare -a tool_opts while [[ -n "${1}" ]]; do tool_opts[${idx}]="${1}" let idx=${idx}+1 @@ -240,7 +240,7 @@ local_interfaces="${local_ips} ${hostname} ${full_hostname} localhost" idx=0 local_hostname="" local_label="" -declare -A remotes +declare -a remotes for (( i=0; ${i} < ${#remotes_A[@]}; i++ )); do remote="${remotes_A[${i}]}" for local_interface in ${local_interfaces}; do diff --git a/agent/util-scripts/samples/pbench-start-tools/test-51/pbench/tools-default/tcpdump b/agent/util-scripts/samples/pbench-start-tools/test-51/pbench/tools-default/tcpdump new file mode 100644 index 0000000000..5d89bb256c --- /dev/null +++ b/agent/util-scripts/samples/pbench-start-tools/test-51/pbench/tools-default/tcpdump @@ -0,0 +1,2 @@ +--interface=br1 +--packets=1000 diff --git a/agent/util-scripts/unittests b/agent/util-scripts/unittests index 82b1c58aef..5e215d451b 100755 --- a/agent/util-scripts/unittests +++ b/agent/util-scripts/unittests @@ -353,6 +353,8 @@ declare -A tools=( [test-48]="test-add-metalog-%-option" [test-49]="pbench-copy-results" [test-50]="test-pbench-metadata-log" + [test-51]="pbench-start-tools" + [test-52]="pbench-register-tool" ) declare -A options=( @@ -421,6 +423,8 @@ declare -A options=( [test-46]="--name=mpstat --no-install --remotes=@${_testdir}/tmp/remotes.lis" [test-47]="--name=mpstat --no-install --remotes=@${_testdir}/tmp/remotes.lis --labels=labelOne,labelTwo" [test-49]="--prefix=foo/bar3% --user=ndk20%" + [test-51]="--dir=${_testdir}/mock-run/mock-iteration/mock-sample --group=default" + [test-52]="--name=tcpdump --group=default --no-install -- --interface=br1 --packets=1000" ) declare -A expected_status=( From beeff2446e36fd9cb1dba901b6b40436a201fbce Mon Sep 17 00:00:00 2001 From: Peter Portante Date: Tue, 22 Sep 2020 15:14:12 -0400 Subject: [PATCH 5/5] Fix tool argument handling Fixes issue #1751, where tools with multiple arguments only have the first argument passed to the tool command. --- .../util-scripts/gold/pbench-start-tools/test-51.txt | 4 ++-- agent/util-scripts/pbench-postprocess-tools | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/agent/util-scripts/gold/pbench-start-tools/test-51.txt b/agent/util-scripts/gold/pbench-start-tools/test-51.txt index fc3a1af6a0..ae38593f5f 100644 --- a/agent/util-scripts/gold/pbench-start-tools/test-51.txt +++ b/agent/util-scripts/gold/pbench-start-tools/test-51.txt @@ -24,10 +24,10 @@ [debug][1900-01-01T00:00:00.000000] [pbench-kill-tools] /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/tcpdump --kill --dir=/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample/tools-default --interface=br1 --packets=1000 [debug][1900-01-01T00:00:00.000000] [pbench-kill-tools]completed: [debug][1900-01-01T00:00:00.000000] [pbench-start-tools] /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/tcpdump --start --dir=/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample/tools-default --interface=br1 --packets=1000 -[debug][1900-01-01T00:00:00.000000] [pbench-start-tools] "#!/bin/bash screen -dm -L -S "pbench-tool-default-tcpdump" /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/tcpdump --start --dir=/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample/tools-default --interface=br1" +[debug][1900-01-01T00:00:00.000000] [pbench-start-tools] "#!/bin/bash screen -dm -L -S "pbench-tool-default-tcpdump" /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/tcpdump --start --dir=/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample/tools-default --interface=br1 --packets=1000 " [debug][1900-01-01T00:00:00.000000] [pbench-start-tools]completed: --- pbench.log file contents +++ test-execution.log file contents /var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/screen -ls -/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/screen -dm -L -S pbench-tool-default-tcpdump /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/tcpdump --start --dir=/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample/tools-default --interface=br1 +/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/screen -dm -L -S pbench-tool-default-tcpdump /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/tcpdump --start --dir=/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample/tools-default --interface=br1 --packets=1000 --- test-execution.log file contents diff --git a/agent/util-scripts/pbench-postprocess-tools b/agent/util-scripts/pbench-postprocess-tools index a1b5932fc8..2b786bbbef 100755 --- a/agent/util-scripts/pbench-postprocess-tools +++ b/agent/util-scripts/pbench-postprocess-tools @@ -28,13 +28,13 @@ debug_log "[$script_name]started: $@" opts=$(getopt -q -o d:g: --longoptions "dir:,group:" -n "getopt.sh" -- "$@"); if [ $? -ne 0 ]; then - printf "\n" - printf "$script_name: you specified an invalid option\n\n" - printf "The following are required:\n\n" + printf -- "\n" + printf -- "%s: you specified an invalid option\n\n" "${script_name}" + printf -- "The following are required:\n\n" printf -- "\t-g str --group=str, str = a tool group used in a benchmark\n" printf -- "\t (the default group is 'default')\n" - printf "\n" - printf -- "\t-d str --dir=str, str = a directory where the $script_name\n" + printf -- "\n" + printf -- "\t-d str --dir=str, str = a directory where the %s\n" "${script_name}" printf -- "\t will store and process data\n" exit 1 fi @@ -163,7 +163,7 @@ for this_tool_file in `/bin/ls $tool_group_dir`; do exit ${rc} fi screen_cmd="${screen_dir}/command" - printf -- "#!/bin/bash\n\nscreen -dm -L -S \"${screen_name}\" ${pbench_bin}/tool-scripts/${name} --${action} --dir=${tool_output_dir} ${tool_opts[@]}\n" > ${screen_cmd} + printf -- "#!/bin/bash\n\nscreen -dm -L -S \"%s\" %s/tool-scripts/%s --%s --dir=%s %s\n" "${screen_name}" "${pbench_bin}" "${name}" "${action}" "${tool_output_dir}" "$(echo ${tool_opts[@]})" > ${screen_cmd} chmod +x ${screen_cmd} debug_log "[${script_name}] \"$(cat ${screen_cmd} | tr '\n' ' ')\"" (cd ${screen_dir}; ${screen_cmd})