Skip to content

Decompose pbench-move-results into "make" and "copy" components #1289

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
portante merged 2 commits into from
Aug 9, 2019
Merged

Decompose pbench-move-results into "make" and "copy" components #1289

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
58820c4
Add raw_size run variable for tar balls
portante Jul 30, 2019
df85bdb
Break up pbench-move-results into components
portante Jul 30, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions agent/util-scripts/gold/pbench-copy-results/test-31.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
+++ Running test-31 pbench-copy-results --prefix=foo/bar --user=ndk
tar --create --force-local "pbench-user-benchmark_ndk-test-1_2018.05.23T03.21.32" | xz -T0 > "/var/tmp/pbench-test-utils/pbench/tmp/pbench-copy-results.NNNNN/testhost/pbench-user-benchmark_ndk-test-1_2018.05.23T03.21.32.tar.xz"
tar --create --force-local "pbench-user-benchmark_ndk-test-1_2018.05.23T03.21.32" | xz -T0 > "/var/tmp/pbench-test-utils/pbench/tmp/pbench-copy-results.NNNNN/testhost.example.com/pbench-user-benchmark_ndk-test-1_2018.05.23T03.21.32.tar.xz"
--- Finished test-31 pbench-copy-results (status=0)
+++ pbench tree state
/var/tmp/pbench-test-utils/pbench
Expand Down Expand Up @@ -295,15 +295,15 @@ tar --create --force-local "pbench-user-benchmark_ndk-test-1_2018.05.23T03.21.32
/var/tmp/pbench-test-utils/pbench/tmp
--- pbench tree state
+++ pbench.log file contents
/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] preparing to copy 5 MB of data from /var/tmp/pbench-test-utils/pbench/pbench-user-benchmark_ndk-test-1_2018.05.23T03.21.32
/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] preparing to tar up ##### bytes of data from /var/tmp/pbench-test-utils/pbench/pbench-user-benchmark_ndk-test-1_2018.05.23T03.21.32
/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] successfully copied 1 runs, encountered 0 failures
--- pbench.log file contents
+++ test-execution.log file contents
/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/yum info installed pbench-agent
/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/yum info installed pbench-agent
/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/curl -s -A pbench-agent-unknown-:agent.example.com:nobody:/var/tmp/pbench-test-utils/opt/pbench-agent/util-scripts/pbench-copy-results -L http://pbench.example.com/pbench-results-host-info.versioned/pbench-results-host-info.URL002
/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/curl -s -A pbench-agent-unknown-unknown:agent.example.com:nobody:pbench-copy-results -L http://pbench.example.com/pbench-results-host-info.versioned/pbench-results-host-info.URL002
/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/ssh -q -i /var/tmp/pbench-test-utils/opt/pbench-agent/id_rsa -o StrictHostKeyChecking=no pbench@server.com exit
/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/scp -r -o StrictHostKeyChecking=no -i /var/tmp/pbench-test-utils/opt/pbench-agent/id_rsa -o StrictHostKeyChecking=no /var/tmp/pbench-test-utils/pbench/tmp/pbench-copy-results.NNNNN/testhost pbench@server.com:/foo/bar
/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/scp -r -o StrictHostKeyChecking=no -i /var/tmp/pbench-test-utils/opt/pbench-agent/id_rsa -o StrictHostKeyChecking=no /var/tmp/pbench-test-utils/pbench/tmp/pbench-copy-results.NNNNN/testhost.example.com pbench@server.com:/foo/bar
/var/tmp/pbench-test-utils/test-execution.log:pbench@server.com:/foo/bar/iostat:foo
/var/tmp/pbench-test-utils/test-execution.log:pbench@server.com:/foo/bar/mpstat:bar
/var/tmp/pbench-test-utils/test-execution.log:pbench@server.com:/foo/bar/perf:baz
Expand All @@ -312,7 +312,7 @@ tar --create --force-local "pbench-user-benchmark_ndk-test-1_2018.05.23T03.21.32
/var/tmp/pbench-test-utils/test-execution.log:pbench@server.com:/foo/bar/proc-vmstat:foobar
/var/tmp/pbench-test-utils/test-execution.log:pbench@server.com:/foo/bar/sar:foo
/var/tmp/pbench-test-utils/test-execution.log:pbench@server.com:/foo/bar/turbostat:bar
/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/ssh -i /var/tmp/pbench-test-utils/opt/pbench-agent/id_rsa -o StrictHostKeyChecking=no pbench@server.com cd /foo/bar/testhost; md5sum --check pbench-user-benchmark_ndk-test-1_2018.05.23T03.21.32.tar.xz.md5.check && mv pbench-user-benchmark_ndk-test-1_2018.05.23T03.21.32.tar.xz.md5.check pbench-user-benchmark_ndk-test-1_2018.05.23T03.21.32.tar.xz.md5
/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/ssh -i /var/tmp/pbench-test-utils/opt/pbench-agent/id_rsa -o StrictHostKeyChecking=no pbench@server.com cd /foo/bar/testhost.example.com; md5sum --check pbench-user-benchmark_ndk-test-1_2018.05.23T03.21.32.tar.xz.md5.check && mv pbench-user-benchmark_ndk-test-1_2018.05.23T03.21.32.tar.xz.md5.check pbench-user-benchmark_ndk-test-1_2018.05.23T03.21.32.tar.xz.md5
--- test-execution.log file contents
+++ metadata.log file contents
[pbench]
Expand Down Expand Up @@ -344,6 +344,7 @@ start_run = 2018-05-23T03:21:32.387628370
end_run = 2018-05-23T03:22:39.538437410
user = ndk
prefix = foo/bar
raw_size = #####

[iterations/1]
iteration_name = 1
Expand Down
10 changes: 5 additions & 5 deletions agent/util-scripts/gold/pbench-move-results/test-33.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
+++ setup pbench results dir time stamps
--- setup pbench results dir time stamps
+++ Running test-33 pbench-move-results
tar --create --force-local "pbench-user-benchmark_ndk-test-1_2019.01.01T12.00.42" | xz -T0 > "/var/tmp/pbench-test-utils/pbench/tmp/pbench-move-results.NNNNN/testhost/pbench-user-benchmark_ndk-test-1_2019.01.01T12.00.42.tar.xz"
tar --create --force-local "pbench-user-benchmark_ndk-test-1_2019.01.01T12.00.42" | xz -T0 > "/var/tmp/pbench-test-utils/pbench/tmp/pbench-move-results.NNNNN/testhost.example.com/pbench-user-benchmark_ndk-test-1_2019.01.01T12.00.42.tar.xz"
[warn][1900-01-01T00:00:00.000000] The run in directory /var/tmp/pbench-test-utils/pbench/pbench-user-benchmark_ndk-test-1_2019.01.01T12.00.43 has an unexpected metadata name, "pbench-user-benchmark__2019.01.01T12.00.43" - skipping
--- Finished test-33 pbench-move-results (status=0)
+++ pbench tree state
Expand All @@ -26,7 +26,7 @@ tar --create --force-local "pbench-user-benchmark_ndk-test-1_2019.01.01T12.00.42
--- pbench tree state
+++ pbench.log file contents
/var/tmp/pbench-test-utils/pbench/pbench.log:Fake log file contents.
/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] preparing to copy 1 MB of data from /var/tmp/pbench-test-utils/pbench/pbench-user-benchmark_ndk-test-1_2019.01.01T12.00.42
/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] preparing to tar up ##### bytes of data from /var/tmp/pbench-test-utils/pbench/pbench-user-benchmark_ndk-test-1_2019.01.01T12.00.42
/var/tmp/pbench-test-utils/pbench/pbench.log:[warn][1900-01-01T00:00:00.000000] The run in directory /var/tmp/pbench-test-utils/pbench/pbench-user-benchmark_ndk-test-1_2019.01.01T12.00.43 has an unexpected metadata name, "pbench-user-benchmark__2019.01.01T12.00.43" - skipping
/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] The pbench result pbench-user-benchmark_ndk-test-1_2019.01.01T12.00.44 does not appear to be a benchmark directory - skipping
/var/tmp/pbench-test-utils/pbench/pbench.log:[debug][1900-01-01T00:00:00.000000] The /var/tmp/pbench-test-utils/pbench/pbench-user-benchmark_ndk-test-1_2019.01.01T12.00.44/metadata.log file seems to be missing
Expand All @@ -35,9 +35,9 @@ tar --create --force-local "pbench-user-benchmark_ndk-test-1_2019.01.01T12.00.42
+++ test-execution.log file contents
/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/yum info installed pbench-agent
/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/yum info installed pbench-agent
/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/curl -s -A pbench-agent-unknown-:agent.example.com:nobody:/var/tmp/pbench-test-utils/opt/pbench-agent/util-scripts/pbench-move-results -L http://pbench.example.com/pbench-results-host-info.versioned/pbench-results-host-info.URL002
/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/curl -s -A pbench-agent-unknown-unknown:agent.example.com:nobody:pbench-move-results -L http://pbench.example.com/pbench-results-host-info.versioned/pbench-results-host-info.URL002
/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/ssh -q -i /var/tmp/pbench-test-utils/opt/pbench-agent/id_rsa -o StrictHostKeyChecking=no pbench@server.com exit
/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/scp -r -o StrictHostKeyChecking=no -i /var/tmp/pbench-test-utils/opt/pbench-agent/id_rsa -o StrictHostKeyChecking=no /var/tmp/pbench-test-utils/pbench/tmp/pbench-move-results.NNNNN/testhost pbench@server.com:/foo/bar
/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/scp -r -o StrictHostKeyChecking=no -i /var/tmp/pbench-test-utils/opt/pbench-agent/id_rsa -o StrictHostKeyChecking=no /var/tmp/pbench-test-utils/pbench/tmp/pbench-move-results.NNNNN/testhost.example.com pbench@server.com:/foo/bar
/var/tmp/pbench-test-utils/test-execution.log:pbench@server.com:/foo/bar/iostat:foo
/var/tmp/pbench-test-utils/test-execution.log:pbench@server.com:/foo/bar/mpstat:bar
/var/tmp/pbench-test-utils/test-execution.log:pbench@server.com:/foo/bar/perf:baz
Expand All @@ -46,5 +46,5 @@ tar --create --force-local "pbench-user-benchmark_ndk-test-1_2019.01.01T12.00.42
/var/tmp/pbench-test-utils/test-execution.log:pbench@server.com:/foo/bar/proc-vmstat:foobar
/var/tmp/pbench-test-utils/test-execution.log:pbench@server.com:/foo/bar/sar:foo
/var/tmp/pbench-test-utils/test-execution.log:pbench@server.com:/foo/bar/turbostat:bar
/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/ssh -i /var/tmp/pbench-test-utils/opt/pbench-agent/id_rsa -o StrictHostKeyChecking=no pbench@server.com cd /foo/bar/testhost; md5sum --check pbench-user-benchmark_ndk-test-1_2019.01.01T12.00.42.tar.xz.md5.check && mv pbench-user-benchmark_ndk-test-1_2019.01.01T12.00.42.tar.xz.md5.check pbench-user-benchmark_ndk-test-1_2019.01.01T12.00.42.tar.xz.md5
/var/tmp/pbench-test-utils/test-execution.log:/var/tmp/pbench-test-utils/opt/pbench-agent/unittest-scripts/ssh -i /var/tmp/pbench-test-utils/opt/pbench-agent/id_rsa -o StrictHostKeyChecking=no pbench@server.com cd /foo/bar/testhost.example.com; md5sum --check pbench-user-benchmark_ndk-test-1_2019.01.01T12.00.42.tar.xz.md5.check && mv pbench-user-benchmark_ndk-test-1_2019.01.01T12.00.42.tar.xz.md5.check pbench-user-benchmark_ndk-test-1_2019.01.01T12.00.42.tar.xz.md5
--- test-execution.log file contents
84 changes: 84 additions & 0 deletions agent/util-scripts/pbench-copy-result-tb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
#!/bin/bash
# -*- mode: shell-script; indent-tabs-mode: t; sh-basic-offset: 4; sh-indentation: 4; sh-indent-for-case-alt: + -*-

script_path="$(dirname ${0})"
script_name="$(basename ${0})"
pbench_bin="$(realpath -e ${script_path}/..)"

# source the base script
. "${pbench_bin}"/base

function usage() {
printf "usage:\n"
printf "${script_name} [--help] <full path to tar ball> <results host> <results path prefix>\n"
}

# Process options and arguments
opts=$(getopt -q -o h --longoptions "help" -- "${@}")
if [[ ${?} -ne 0 ]]; then
printf "\n${script_name}: you specified an invalid option\n\n" >&2
usage >&2
exit 1
fi

eval set -- "${opts}"
while true; do
case "${1}" in
-h|--help)
usage
exit 0
;;
--)
shift;
break;
;;
esac
done

tarball=$(realpath -e ${1})
if [[ ! -f "${tarball}" ]]; then
error_log "ERROR: tar ball does not exist, ${tarball}"
exit 1
fi
if [[ ! -f "${tarball}.md5.check" ]]; then
error_log "ERROR: tar ball's .md5.check does not exist, ${tarball}.md5.check"
exit 1
fi
controller_dir=$(dirname ${tarball})
cnt=$(find ${controller_dir} -type f 2> /dev/null | wc -l 2> /dev/null)
if [[ ${cnt} != 2 ]]; then
error_log "ERROR (internal): unexpected file count, ${cnt}, associated with tar ball, ${tarball}"
exit 1
fi

results_repo=${2}
results_path_prefix=${3}

if [[ ! -f "${pbench_bin}/id_rsa" ]]; then
error_log "ERROR: ${pbench_bin}/id_rsa required for moving results to archive host"
exit 1
fi

# Copy the directory with scp -r $tmp/$controller $remote: that will
# create the $controller subdirectory on the remote (if necessary) OR
# fail. If it does not fail, then check the MD5 sum and rename the
# foo.tar.xz.md5.check file to foo.tar.xz.md5. That's the signal that the
# agent has finished with this tarball.

scp -r ${scp_opts} -i ${pbench_bin}/id_rsa ${ssh_opts} ${controller_dir} ${results_repo}:${results_path_prefix}
if [[ ${?} -ne 0 ]]; then
error_log "ERROR: unable to copy results tarball, ${tarball}, to ${results_repo}:${results_path_prefix}"
exit 1
fi

# Verify the remotely copied bits are good
md5name=$(basename ${tarball}).md5
controller=$(basename ${controller_dir})
ssh -i ${pbench_bin}/id_rsa ${ssh_opts} ${results_repo} "cd ${results_path_prefix}/${controller}; md5sum --check ${md5name}.check && mv ${md5name}.check ${md5name}"
chk_res=${?}
if [[ ${chk_res} -ne 0 ]]; then
error_log "ERROR: remote copy failed, remote tarball MD5 does not match original"
exit 1
fi

exit 0
175 changes: 175 additions & 0 deletions agent/util-scripts/pbench-make-result-tb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,175 @@
#!/bin/bash
# -*- mode: shell-script; indent-tabs-mode: t; sh-basic-offset: 4; sh-indentation: 4; sh-indent-for-case-alt: + -*-

script_path="$(dirname ${0})"
script_name="$(basename ${0})"
pbench_bin="$(realpath -e ${script_path}/..)"

# source the base script
. "$pbench_bin"/base

function usage() {
printf "usage:\n"
printf "${script_name} --result-dir=<pbench results dir> --target-dir=<where to put tar ball> [--help] [--user=<user>] [--prefix=<path>] [--xz-single-threaded=<0|1>]\n"
}

# Process options and arguments
opts=$(getopt -q -o h --longoptions "user:,prefix:,result-dir:,target-dir:,xz-single-threaded:,help" -n "getopt.sh" -- "${@}")
sts=${?}
if [[ ${sts} -ne 0 ]]; then
printf "\n${script_name}: you specified an invalid option\n\n" >&2
usage >&2
exit 1
fi

result_dir=""
target_dir=""
user="${PBENCH_USER}"
prefix=""
xz_single_threaded=0
eval set -- "${opts}"
while true; do
case "${1}" in
--result-dir)
shift;
if [[ -n "${1}" ]]; then
result_dir="${1}"
shift;
fi
;;
--target-dir)
shift;
if [[ -n "${1}" ]]; then
target_dir="${1}"
shift;
fi
;;
--user)
shift;
if [[ -n "${1}" ]]; then
user="${1}"
shift;
fi
;;
--prefix)
shift;
if [[ -n "${1}" ]]; then
prefix="${1}"
shift;
fi
;;
--xz-single-threaded)
shift;
if [[ -n "${1}" ]]; then
xz_single_threaded="${1}"
shift;
fi
;;
-h|--help)
usage
exit 0
;;
--)
shift;
break;
;;
esac
done

if [[ ! -d "${result_dir}" ]]; then
error_log "Invalid result directory provided: \"${result_dir}\""
usage >&2
exit 1
fi

if [[ ! -d "${target_dir}" ]]; then
error_log "Invalid target directory provided: \"${target_dir}\""
exit 1
fi

# We can now start building the target tarball

# Move into pbench run collection directory
full_result_dir=$(realpath -e ${result_dir})
pushd $(dirname full_result_dir) > /dev/null 2>&1
if [[ ${?} -ne 0 ]]; then
exit 1
fi
pbench_run_name=$(basename ${result_dir})
if [[ -f "${pbench_run_name}.copied" ]]; then
debug_log "Already copied ${result_dir}"
exit 0
fi

if [[ -d "${pbench_run_name}/.running" ]]; then
# The benchmark is still running in this directory, skip it
debug_log "The benchmark is still running in ${pbench_run_name} - skipping"
debug_log "If that is not true, rmdir ${pbench_run_name}/.running, and try again"
exit 0
fi

mdlog=${pbench_run_name}/metadata.log
if [[ ! -e "${mdlog}" ]]; then
debug_log "The pbench result ${pbench_run_name} does not appear to be a benchmark directory - skipping"
debug_log "The ${pbench_run}/${pbench_run_name}/metadata.log file seems to be missing"
exit 0
fi

res_name=$(getconf.py --config ${mdlog} name pbench)
if [[ "${res_name}" != "${pbench_run_name}" ]]; then
warn_log "The run in directory ${pbench_run}/${pbench_run_name} has an unexpected metadata name, \"${res_name}\" - skipping"
exit 1
fi

if [[ -e pbench.log ]]; then
# FIXME: We should not copy this log file into a given result, as it may
# contain data not relevant to that results' execution.
#
# We have a pbench.log file, so make a copy of it in the current result
# directory so that any log datas from tools or benchmarks can be
# referenced later.
/bin/cp pbench.log ${pbench_run_name}/
fi

# if -u|--user was specified, store the specified user in metadata.log
if [[ ! -z "$user" ]] ;then
printf -- "$user" | pbench-add-metalog-option ${mdlog} run user
fi

# if -p|--prefix was specified, store the specified prefix in metadata.log
if [[ ! -z "$prefix" ]] ;then
printf -- "$prefix" | pbench-add-metalog-option ${mdlog} run prefix
fi

result_size=$(du -sb ${pbench_run_name} | awk '{print $1}')
debug_log "preparing to tar up ${result_size} bytes of data from ${full_result_dir}"
printf -- "$result_size" | pbench-add-metalog-option ${mdlog} run raw_size

tarball="${target_dir}/${pbench_run_name}.tar.xz"
if [[ "${xz_single_threaded}" == "1" ]]; then
tar_cmd="tar --create --force-local -xz \"${pbench_run_name}\""
else
tar_cmd="tar --create --force-local \"${pbench_run_name}\" | xz -T0"
fi
printf -- "${tar_cmd} > \"${tarball}\"\n" >&2
eval ${tar_cmd} > "${tarball}"
if [[ $? -ne 0 ]]; then
error_log "ERROR: tar ball creation failed for ${result_dir}, skipping"
rm -f "${tarball}"
exit 1
fi

tarballmd5="${tarball}.md5.check"
# We need to calculate the md5 sum in the temp directory
# in order to get the filename right.
pushd $(dirname ${tarball}) > /dev/null 2>&1
md5sum "$(basename ${tarball})" > "${tarballmd5}"
if [[ $? -ne 0 ]]; then
error_log "ERROR: md5sum failed for ${tarball}, skipping"
rm -f "${tarball}" "${tarballmd5}"
exit 1
fi

# We ran the gauntlet successfully!
printf -- "${tarball}\n"
exit 0
Loading