From b3e913f36b0c982895c0a02503ce4186a72012c0 Mon Sep 17 00:00:00 2001 From: Kosyrev Serge Date: Thu, 2 Dec 2021 18:59:57 +0300 Subject: [PATCH] bench: make jq prefiltering optional, off by default --- nix/workbench/analyse.sh | 42 +++++++++++++++++------------ nix/workbench/run.sh | 58 +++++++++++++++++++++------------------- 2 files changed, 56 insertions(+), 44 deletions(-) diff --git a/nix/workbench/analyse.sh b/nix/workbench/analyse.sh index 16a6da249cc..481c4de7af3 100644 --- a/nix/workbench/analyse.sh +++ b/nix/workbench/analyse.sh @@ -20,12 +20,13 @@ EOF } analyse() { -local skip_preparation= time= dump_logobjects= self_args=() locli_args=() since_slot= until_slot= fullness_above= +local time= dump_logobjects= self_args=() locli_args=() prefilter='true' prefilter_jq='false' while test $# -gt 0 do case "$1" in - --chain-filters ) locli_args+=($1 $2); self_args+=($1 $2); shift;; - --reanalyse | --re ) skip_preparation='true'; self_args+=($1);; + --reanalyse | --re ) prefilter='false'; self_args+=($1);; + --prefilter-jq ) prefilter_jq='true'; self_args+=($1);; --dump-logobjects ) dump_logobjects='true'; self_args+=($1);; + --chain-filters ) locli_args+=($1 $2); self_args+=($1 $2); shift;; * ) break;; esac; shift; done local op=${1:-$(usage_analyse)}; shift @@ -53,21 +54,24 @@ case "$op" in local name=${1:-current}; shift local dir=$(run get "$name") + test -n "$dir" || fail "malformed run: $name" + + echo "{ \"run\": \"$(jq .meta.tag "$dir"/meta.json --raw-output)\" }" + local adir=$dir/analysis - if test -z "$dir" - then fail "malformed run: $name"; fi + mkdir -p "$adir" ## 0. subset what we care about into the keyfile - local keyfile=$adir/substring-keys + local keyfile="$adir"/substring-keys locli analyse substring-keys > "$keyfile" ## 1. enumerate logs, filter by keyfile & consolidate local logdirs=($(ls -d "$dir"/node-*/ 2>/dev/null) $(ls -d "$dir"/analysis/node-*/ 2>/dev/null)) # "$dir"/node-*/ "$dir"/analysis/node-*/ - if test -z "$skip_preparation" -o -z "$(ls "$adir"/logs-node-*.flt.json 2>/dev/null)" + echo "{ \"prefilter\": $prefilter, \"prefilter_jq\": $prefilter_jq }" + if test "$prefilter" = 'true' -o -z "$(ls "$adir"/logs-node-*.flt.json 2>/dev/null)" then - echo "{ \"prefiltering\": true }" local jq_args=( --sort-keys --compact-output @@ -82,14 +86,15 @@ case "$op" in then msg "no logs in $d, skipping.."; fi local output="$adir"/logs-$(basename "$d").flt.json grep -hFf "$keyfile" $logfiles | - jq "${jq_args[@]}" --arg dirHostname "$(basename "$d")" \ - > "$output" & + if test "$prefilter_jq" = 'true' + then jq "${jq_args[@]}" --arg dirHostname "$(basename "$d")" + else cat + fi > "$output" & done wait fi - mkdir -p "$adir" - echo "{ \"run\": \"$(jq .meta.tag "$dir"/meta.json --raw-output)\" }" + echo "{ \"dataSetSizeMB\": $(echo $(($(cat "$adir"/*.flt.json | wc -c) / 1000 / 1000))) }" locli_args+=( --genesis "$dir"/genesis-shelley.json --run-metafile "$dir"/meta.json @@ -144,9 +149,11 @@ case "$op" in local name=${1:-current} local mach=${2:-node-1} local dir=$(run get "$name") - local adir=$dir/analysis + test -n "$dir" || fail "malformed run: $name" - msg "analysing run $(jq .meta.tag "$dir"/meta.json --raw-output)" + echo "{ \"run\": \"$(jq .meta.tag "$dir"/meta.json --raw-output)\" }" + + local adir=$dir/analysis mkdir -p "$adir" ## 0. subset what we care about into the keyfile @@ -163,10 +170,11 @@ case "$op" in ## 1. enumerate logs, filter by keyfile & consolidate local logs=($(ls "$dir"/$mach/stdout* 2>/dev/null | tac) $(ls "$dir"/$mach/node-*.json 2>/dev/null) $(ls "$dir"/analysis/$mach/node-*.json 2>/dev/null)) consolidated="$adir"/logs-$mach.json - if test -z "${logs[*]}" - then msg "no logs for $mach in run $name"; continue; fi + test -n "${logs[*]}" || + fail "no logs for $mach in run $name" - if test -z "$skip_preparation" -o -z "$(ls "$adir"/logs-$mach.json 2>/dev/null)" + echo "{ \"prefilter\": $prefilter }" + if test "$prefilter" = 'true' -o -z "$(ls "$adir"/logs-$mach.json 2>/dev/null)" then grep -hFf "$keyfile" "${logs[@]}" > "$consolidated"; fi locli_args+=( diff --git a/nix/workbench/run.sh b/nix/workbench/run.sh index 9a7050cde39..76ad65df11d 100644 --- a/nix/workbench/run.sh +++ b/nix/workbench/run.sh @@ -63,42 +63,46 @@ case "$op" in compute-path ) echo -n "$global_rundir/$1";; + fix-legacy-run-structure | fix-legacy ) + local usage="USAGE: wb run $op TAG" + local tag=${1:?$usage} + local dir=$(run compute-path "$tag") + + if test -z "$(ls -d "$dir"/node-* 2>/dev/null)" + then msg "fixing up a legacy cardano-ops run in: $dir" + if local dirs=$(ls -d "$dir"/logs/logs-*/ 2>/dev/null || true) + test -n "$dirs" + then for logdir in $dirs + do local logs_less=$(basename "$logdir" | cut -c6-) + mv "$logdir" "$dir"/$logs_less; done + elif local dirs=$(ls -d "$dir"/analysis/logs-*/ 2>/dev/null || true) + test -n "$dirs" + then for logdir in $dirs + do local logs_less=$(basename "$logdir" | cut -c6-) + mv "$logdir" "$dir"/$logs_less; done; fi + else msg "fixing up a cardano-ops run in: $dir"; fi + + jq '.meta.profile_content' "$dir"/meta.json > "$dir"/profile.json + + jq_fmutate "$dir"/env.json '. * + { type: "legacy" + , staggerPorts: false + } + ';; + check ) local usage="USAGE: wb run $op TAG" local tag=${1:?$usage} local dir=$(run compute-path "$tag") - if test "$(tr -d / <<<$tag)" != "$tag" - then fatal "run tag has slashes: $tag"; fi + test "$(tr -d / <<<$tag)" = "$tag" || + fatal "run tag has slashes: $tag" jq_check_json "$dir"/meta.json || fatal "run $tag (at $dir) missing a file: meta.json" - if test ! -f "$dir"/profile.json - then # Legacy run structure, fix up: - if test -z "$(ls -d "$dir"/logs/node-*)" - then msg "fixing up a legacy cardano-ops run in: $dir" - local topdirs=$(ls -d "$dir"/logs-*/ 2>/dev/null || true) - local anadirs=$(ls -d "$dir"/analysis/logs-*/ 2>/dev/null || true) - if test -n "$topdirs" - then for logdir in $topdirs - do local fixed=$(basename "$logdir" | cut -c6-) - mv "$logdir" "$dir"/$fixed; done - elif test -n "$anadirs" - then for logdir in $anadirs - do local fixed=$(basename "$logdir" | cut -c6-) - mv "$logdir" "$dir"/analysis/$fixed; done; fi - else msg "fixing up a cardano-ops run in: $dir" - fi - - jq '.meta.profile_content' "$dir"/meta.json > "$dir"/profile.json - - jq_fmutate "$dir"/env.json '. * - { type: "legacy" - , staggerPorts: false - } - ' - fi;; + test -f "$dir"/profile.json || + run fix-legacy-run-structure "$tag";; fix-systemstart ) local usage="USAGE: wb run $op TAG [MACH=node-1]"