Fix tool argument handling (v0.69)

.gitignore

@@ -11,3 +11,5 @@ dist
 *.egg-info
 .tox
 .eggs
+Pipfile
+Pipfile.lock

agent/tool-scripts/datalog/cpuacct-datalog

@@ -20,7 +20,7 @@ if [[ ${?} -ne 0 ]]; then
 	exit 1
 fi
 
-dirs="$(find -mindepth 1 -type d)"
+dirs="$(find -mindepth 1 -type d | sort)"
 if [[ -z "${dirs}" ]]; then
 	printf -- "%s: CPU accounting appears disabled, no sub-directories found\n" "${PROG}" >&2
 	exit 1

agent/util-scripts/gold/pbench-register-tool/test-52.txt

@@ -0,0 +1,17 @@
++++ Running test-52 pbench-register-tool --name=tcpdump --group=default --no-install -- --interface=br1 --packets=1000
+tcpdump tool is now registered in group default
+--- Finished test-52 pbench-register-tool (status=0)
++++ pbench tree state
+/var/tmp/pbench-test-utils/pbench
+/var/tmp/pbench-test-utils/pbench/pbench.log
+/var/tmp/pbench-test-utils/pbench/tmp
+/var/tmp/pbench-test-utils/pbench/tools-default
+/var/tmp/pbench-test-utils/pbench/tools-default/tcpdump
+=== /var/tmp/pbench-test-utils/pbench/tools-default/tcpdump:
+--interface=br1
+--packets=1000
+--- pbench tree state
++++ pbench.log file contents
+[debug][1900-01-01T00:00:00.000000] tool_opts: "--interface=br1 --packets=1000"
+[info][1900-01-01T00:00:00.000000] tcpdump tool is now registered in group default
+--- pbench.log file contents

agent/util-scripts/gold/pbench-start-tools/test-51.txt

@@ -0,0 +1,33 @@
++++ Running test-51 pbench-start-tools --dir=/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample --group=default
+--- Finished test-51 pbench-start-tools (status=0)
++++ pbench tree state
+/var/tmp/pbench-test-utils/pbench
+/var/tmp/pbench-test-utils/pbench/mock-run
+/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration
+/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample
+/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample/tools-default
+/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample/tools-default/.screen.d
+/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample/tools-default/.screen.d/default-tcpdump
+/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample/tools-default/.screen.d/default-tcpdump/command
+/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample/tools-default/tcpdump
+/var/tmp/pbench-test-utils/pbench/pbench.log
+/var/tmp/pbench-test-utils/pbench/tmp
+/var/tmp/pbench-test-utils/pbench/tools-default
+/var/tmp/pbench-test-utils/pbench/tools-default/tcpdump
+=== /var/tmp/pbench-test-utils/pbench/tools-default/tcpdump:
+--interface=br1
+--packets=1000
+--- pbench tree state
++++ pbench.log file contents
+[debug][1900-01-01T00:00:00.000000] [pbench-start-tools]started: --dir=/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample --group=default
+[debug][1900-01-01T00:00:00.000000] [pbench-kill-tools]started: --dir /var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample --group default
+[debug][1900-01-01T00:00:00.000000] [pbench-kill-tools] /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/tcpdump --kill --dir=/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample/tools-default --interface=br1 --packets=1000
+[debug][1900-01-01T00:00:00.000000] [pbench-kill-tools]completed: 
+[debug][1900-01-01T00:00:00.000000] [pbench-start-tools] /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/tcpdump --start --dir=/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample/tools-default --interface=br1 --packets=1000
+[debug][1900-01-01T00:00:00.000000] [pbench-start-tools] "#!/bin/bash  screen -dm -L -S "pbench-tool-default-tcpdump" /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/tcpdump --start --dir=/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample/tools-default --interface=br1 --packets=1000 "
+[debug][1900-01-01T00:00:00.000000] [pbench-start-tools]completed: 
+--- pbench.log file contents
++++ test-execution.log file contents
+/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/screen -ls
+/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/screen -dm -L -S pbench-tool-default-tcpdump /var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/tcpdump --start --dir=/var/tmp/pbench-test-utils/pbench/mock-run/mock-iteration/mock-sample/tools-default --interface=br1 --packets=1000
+--- test-execution.log file contents

agent/util-scripts/pbench-postprocess-tools

@@ -28,13 +28,13 @@ debug_log "[$script_name]started: $@"
 
 opts=$(getopt -q -o d:g: --longoptions "dir:,group:" -n "getopt.sh" -- "$@");
 if [ $? -ne 0 ]; then
-	printf "\n"
-	printf "$script_name: you specified an invalid option\n\n"
-	printf "The following are required:\n\n"
+	printf -- "\n"
+	printf -- "%s: you specified an invalid option\n\n" "${script_name}"
+	printf -- "The following are required:\n\n"
 	printf -- "\t-g str --group=str, str = a tool group used in a benchmark\n"
 	printf -- "\t                          (the default group is 'default')\n"
-	printf "\n"
-	printf -- "\t-d str --dir=str, str = a directory where the $script_name\n"
+	printf -- "\n"
+	printf -- "\t-d str --dir=str, str = a directory where the %s\n" "${script_name}"
 	printf -- "\t                        will store and process data\n"
 	exit 1
 fi
@@ -163,7 +163,7 @@ for this_tool_file in `/bin/ls $tool_group_dir`; do
 				exit ${rc}
 			fi
 			screen_cmd="${screen_dir}/command"
-			printf -- "#!/bin/bash\n\nscreen -dm -L -S \"${screen_name}\" ${pbench_bin}/tool-scripts/${name} --${action} --dir=${tool_output_dir} ${tool_opts[@]}\n" > ${screen_cmd}
+			printf -- "#!/bin/bash\n\nscreen -dm -L -S \"%s\" %s/tool-scripts/%s --%s --dir=%s %s\n" "${screen_name}" "${pbench_bin}" "${name}" "${action}" "${tool_output_dir}" "$(echo ${tool_opts[@]})" > ${screen_cmd}
 			chmod +x ${screen_cmd}
 			debug_log "[${script_name}] \"$(cat ${screen_cmd} | tr '\n' ' ')\""
 			(cd ${screen_dir}; ${screen_cmd})

agent/util-scripts/pbench-register-tool

@@ -139,7 +139,7 @@ if [[ ${remotes_arg::1} == "@" ]]; then
 		usage >&2
 		exit 1
 	fi
-	declare -A remotes_A
+	declare -a remotes_A
 	declare -A labels_A
 	idx=0
 	lineno=0
@@ -216,7 +216,7 @@ fi
 # The remainder of ${@} is now tool-specific options. The best way to not
 # mess up arguments seems to be using an array.
 idx=0
-declare -A tool_opts
+declare -a tool_opts
 while [[ -n "${1}" ]]; do
 	tool_opts[${idx}]="${1}"
 	let idx=${idx}+1
@@ -240,7 +240,7 @@ local_interfaces="${local_ips} ${hostname} ${full_hostname} localhost"
 idx=0
 local_hostname=""
 local_label=""
-declare -A remotes
+declare -a remotes
 for (( i=0; ${i} < ${#remotes_A[@]}; i++ )); do
 	remote="${remotes_A[${i}]}"
 	for local_interface in ${local_interfaces}; do

agent/util-scripts/samples/pbench-start-tools/test-51/pbench/tools-default/tcpdump

@@ -0,0 +1,2 @@
+--interface=br1
+--packets=1000

agent/util-scripts/unittests

@@ -353,6 +353,8 @@ declare -A tools=(
     [test-48]="test-add-metalog-%-option"
     [test-49]="pbench-copy-results"
     [test-50]="test-pbench-metadata-log"
+    [test-51]="pbench-start-tools"
+    [test-52]="pbench-register-tool"
 )
 
 declare -A options=(
@@ -421,6 +423,8 @@ declare -A options=(
     [test-46]="--name=mpstat --no-install --remotes=@${_testdir}/tmp/remotes.lis"
     [test-47]="--name=mpstat --no-install --remotes=@${_testdir}/tmp/remotes.lis --labels=labelOne,labelTwo"
     [test-49]="--prefix=foo/bar3% --user=ndk20%"
+    [test-51]="--dir=${_testdir}/mock-run/mock-iteration/mock-sample --group=default"
+    [test-52]="--name=tcpdump --group=default --no-install -- --interface=br1 --packets=1000"
 )
 
 declare -A expected_status=(

server/bin/pbench-cull-unpacked-tarballs.py

@@ -419,9 +419,13 @@ def main(options):
     actions_taken = []
     errors = 0
     start = pbench._time()
-    for tb_incoming_dir, controller_name in gen_list_unpacked_aged(
-        incoming_p, archive_p, curr_dt, max_unpacked_age
-    ):
+
+    gen = gen_list_unpacked_aged(incoming_p, archive_p, curr_dt, max_unpacked_age)
+    if config._unittests:
+        # force the generator and sort the list
+        gen = sorted(list(gen))
+
+    for tb_incoming_dir, controller_name in gen:
         act_set = remove_unpacked(
             tb_incoming_dir,
             controller_name,
@@ -457,7 +461,7 @@ def main(options):
         )
         if total > 0:
             print("\nActions Taken:", file=tfp)
-        for act_set in actions_taken:
+        for act_set in sorted(actions_taken, key=lambda a: a.name):
             print(
                 f"  - {act_set.name} ({act_set.errors:d} errors,"
                 f" {act_set.duration():0.2f} secs)",

test-requirements.txt

@@ -1,2 +1,2 @@
-flake8
+flake8==3.8.3
 pre-commit

tox.ini

@@ -42,7 +42,7 @@ commands =
   black --check .
   flake8 .
 deps =
-  black
-  flake8
+  black==19.10b0
+  flake8==3.8.3
 skip_install = true
 usedevelop = false