From fac83ef1d0a860b913a5fdb507bdf2de821b1468 Mon Sep 17 00:00:00 2001 From: Andy C Date: Wed, 16 Feb 2022 19:14:05 -0500 Subject: [PATCH] [refactor] Add test/tsv-lib.sh Still need to consolidate more --- benchmarks/common.sh | 1 + benchmarks/compute.sh | 12 +++++++----- benchmarks/ovm-build.sh | 11 +++++++---- metrics/source-code.sh | 9 ++++----- soil/worker.sh | 7 ++----- test/tsv-lib.sh | 32 ++++++++++++++++++++++++++++++++ test/unit.sh | 31 +------------------------------ 7 files changed, 54 insertions(+), 49 deletions(-) create mode 100644 test/tsv-lib.sh diff --git a/benchmarks/common.sh b/benchmarks/common.sh index 9d9911c43a..bae2a7a478 100644 --- a/benchmarks/common.sh +++ b/benchmarks/common.sh @@ -57,6 +57,7 @@ csv2html() { web/table/csv2html.py --css-class-pattern 'special ^osh' "$@" } +# TODO: Conslidate with test/tsv-lib.sh tsv2html() { web/table/csv2html.py --tsv "$@" } diff --git a/benchmarks/compute.sh b/benchmarks/compute.sh index 1504bd4f0d..46ad48770a 100755 --- a/benchmarks/compute.sh +++ b/benchmarks/compute.sh @@ -30,6 +30,7 @@ set -o pipefail set -o errexit source benchmarks/common.sh # filter-provenance +source test/tsv-lib.sh readonly BASE_DIR=_tmp/compute readonly OSH_CC=_bin/osh_eval.opt.stripped @@ -275,7 +276,12 @@ task-all() { mkdir -p $tmp_dir $raw_dir/$task_name # header - echo $'status\telapsed_secs\tuser_secs\tsys_secs\tmax_rss_KiB\tstdout_md5sum\thost_name\thost_hash\truntime_name\truntime_hash\ttask_name\targ1\targ2' > $times_tsv + tsv-row \ + status elapsed_secs user_secs sys_secs max_rss_KiB \ + stdout_md5sum \ + host_name host_hash \ + runtime_name runtime_hash \ + task_name arg1 arg2 > $times_tsv local task_id=0 @@ -418,10 +424,6 @@ stage1() { wc -l $times_tsv } -tsv2html() { - csv2html --tsv "$@" -} - print-report() { local in_dir=$1 diff --git a/benchmarks/ovm-build.sh b/benchmarks/ovm-build.sh index 1eb1302e8b..d9a614e1d8 100755 --- a/benchmarks/ovm-build.sh +++ b/benchmarks/ovm-build.sh @@ -36,6 +36,7 @@ set -o errexit source benchmarks/common.sh # for log, etc. source build/common.sh # for $CLANG +source test/tsv-lib.sh readonly BASE_DIR=_tmp/ovm-build readonly TAR_DIR=$PWD/_deps/ovm-build # Make it absolute @@ -96,7 +97,7 @@ extract-oil() { sizes-tsv() { # host_label matches the times.tsv file output by report.R - echo $'host_label\tnum_bytes\tpath' + tsv-row host_label num_bytes path local host=$(hostname) find "$@" -maxdepth 0 -printf "$host\t%s\t%p\n" } @@ -315,7 +316,6 @@ oil-historical-tasks() { } # action is 'configure', a target name, etc. -readonly HEADER=$'status\telapsed_secs\thost_name\thost_hash\tcompiler_path\tcompiler_hash\tsrc_dir\taction' readonly NUM_COLUMNS=7 # 5 from provenence, then tarball/target measure() { @@ -334,8 +334,11 @@ measure() { # TODO: the $times_out calculation is duplicated in build-task() - # Write Header of the CSV file that is appended to. - echo "$HEADER" > $times_out + # Write header of the TSV file that is appended to. + tsv-row \ + status elapsed_secs \ + host_name host_hash compiler_path compiler_hash \ + src_dir action > $times_out local t1=$BASE_DIR/oil-tasks.txt local t2=$BASE_DIR/other-shell-tasks.txt diff --git a/metrics/source-code.sh b/metrics/source-code.sh index c7c7c48a1c..17a0c4a503 100755 --- a/metrics/source-code.sh +++ b/metrics/source-code.sh @@ -9,6 +9,9 @@ set -o nounset set -o pipefail set -o errexit +source test/common.sh +source test/tsv-lib.sh + filter-py() { grep -E -v '__init__.py$|_gen.py|_test.py|_tests.py$' } @@ -310,10 +313,6 @@ metrics-html-head() { html-head --title "$title" "$base_url/base.css" "$base_url/table/table-sort.css" "$base_url/line-counts.css" } -tsv2html() { - web/table/csv2html.py --tsv "$@" -} - counts-html() { local name=$1 local title=$2 @@ -323,7 +322,7 @@ counts-html() { rm -r -f -v $tmp_dir >& 2 mkdir -v -p $tmp_dir >& 2 - echo $'category\tcategory_HREF\ttotal_lines\tnum_files' > $tmp_dir/INDEX.tsv + tsv-row category category_HREF total_lines num_files > $tmp_dir/INDEX.tsv echo $'column_name\ttype category\tstring diff --git a/soil/worker.sh b/soil/worker.sh index 072936a30f..34438269ac 100755 --- a/soil/worker.sh +++ b/soil/worker.sh @@ -10,10 +10,7 @@ set -o pipefail set -o errexit source soil/common.sh - -time-tsv() { - benchmarks/time_.py --tsv "$@" -} +source test/tsv-lib.sh dump-timezone() { @@ -279,7 +276,7 @@ run-tasks() { # show the last line echo - echo $'status\telapsed\ttask\tscript\taction\tresult_html' + tsv-row status elapsed task script action result_html tail -n 1 $tsv echo log "status=$status max_status=$max_status" diff --git a/test/tsv-lib.sh b/test/tsv-lib.sh new file mode 100644 index 0000000000..25531a2a7a --- /dev/null +++ b/test/tsv-lib.sh @@ -0,0 +1,32 @@ +# TSV utiltiies +# +# Usage: +# source test/tsv-lib.sh + +time-tsv() { + ### Run a task and output TSV + benchmarks/time_.py --tsv "$@" +} + +tsv2html() { + ### Convert TSV to an HTML table + web/table/csv2html.py --tsv "$@" +} + +tsv-row() { + ### Usage: tsv-row a b c + local i=0 + for cell in "$@"; do + if test $i -ne 0; then + echo -n $'\t' + fi + + # note: if this were QTT, then it would be quoted + echo -n "$cell" + + i=$((i + 1)) + done + + echo # newline +} + diff --git a/test/unit.sh b/test/unit.sh index dfecf071dd..b5263cf089 100755 --- a/test/unit.sh +++ b/test/unit.sh @@ -17,6 +17,7 @@ set -o errexit shopt -s strict:all 2>/dev/null || true # dogfood for OSH source test/common.sh +source test/tsv-lib.sh export PYTHONPATH='.:vendor' # repo root and vendor subdir @@ -138,36 +139,6 @@ all-in-one() { # NOTE: Show options like this: # python -m unittest discover -h -# -# TODO: Move to test/common.sh -# - -time-tsv() { - benchmarks/time_.py --tsv "$@" -} - -tsv-row() { - ### Usage: tsv-row a b c - local i=0 - for cell in "$@"; do - if test $i -ne 0; then - echo -n $'\t' - fi - - # note: if this were QTT, then it would be quoted - echo -n "$cell" - - i=$((i + 1)) - done - - echo # newline -} - -tsv2html() { - web/table/csv2html.py --tsv "$@" -} - - # # For _release/VERSION #