Skip to content

Commit

Permalink
bench: make jq prefiltering optional, off by default
Browse files Browse the repository at this point in the history
  • Loading branch information
@deepfire
deepfire committed Jan 17, 2022
1 parent d69225b commit b3e913f
Show file tree
Hide file tree
Showing 2 changed files with 56 additions and 44 deletions.
42 changes: 25 additions & 17 deletions nix/workbench/analyse.sh
View file
Expand Up @@ -20,12 +20,13 @@ EOF
}

analyse() {
local skip_preparation= time= dump_logobjects= self_args=() locli_args=() since_slot= until_slot= fullness_above=
local time= dump_logobjects= self_args=() locli_args=() prefilter='true' prefilter_jq='false'
while test $# -gt 0
do case "$1" in
--chain-filters ) locli_args+=($1 $2); self_args+=($1 $2); shift;;
--reanalyse | --re ) skip_preparation='true'; self_args+=($1);;
--reanalyse | --re ) prefilter='false'; self_args+=($1);;
--prefilter-jq ) prefilter_jq='true'; self_args+=($1);;
--dump-logobjects ) dump_logobjects='true'; self_args+=($1);;
--chain-filters ) locli_args+=($1 $2); self_args+=($1 $2); shift;;
* ) break;; esac; shift; done

local op=${1:-$(usage_analyse)}; shift
Expand Down Expand Up @@ -53,21 +54,24 @@ case "$op" in

local name=${1:-current}; shift
local dir=$(run get "$name")
test -n "$dir" || fail "malformed run: $name"

echo "{ \"run\": \"$(jq .meta.tag "$dir"/meta.json --raw-output)\" }"

local adir=$dir/analysis
if test -z "$dir"
then fail "malformed run: $name"; fi
mkdir -p "$adir"

## 0. subset what we care about into the keyfile
local keyfile=$adir/substring-keys
local keyfile="$adir"/substring-keys
locli analyse substring-keys > "$keyfile"

## 1. enumerate logs, filter by keyfile & consolidate
local logdirs=($(ls -d "$dir"/node-*/ 2>/dev/null) $(ls -d "$dir"/analysis/node-*/ 2>/dev/null))
# "$dir"/node-*/ "$dir"/analysis/node-*/

if test -z "$skip_preparation" -o -z "$(ls "$adir"/logs-node-*.flt.json 2>/dev/null)"
echo "{ \"prefilter\": $prefilter, \"prefilter_jq\": $prefilter_jq }"
if test "$prefilter" = 'true' -o -z "$(ls "$adir"/logs-node-*.flt.json 2>/dev/null)"
then
echo "{ \"prefiltering\": true }"
local jq_args=(
--sort-keys
--compact-output
Expand All @@ -82,14 +86,15 @@ case "$op" in
then msg "no logs in $d, skipping.."; fi
local output="$adir"/logs-$(basename "$d").flt.json
grep -hFf "$keyfile" $logfiles |
jq "${jq_args[@]}" --arg dirHostname "$(basename "$d")" \
> "$output" &
if test "$prefilter_jq" = 'true'
then jq "${jq_args[@]}" --arg dirHostname "$(basename "$d")"
else cat
fi > "$output" &
done
wait
fi

mkdir -p "$adir"
echo "{ \"run\": \"$(jq .meta.tag "$dir"/meta.json --raw-output)\" }"
echo "{ \"dataSetSizeMB\": $(echo $(($(cat "$adir"/*.flt.json | wc -c) / 1000 / 1000))) }"
locli_args+=(
--genesis "$dir"/genesis-shelley.json
--run-metafile "$dir"/meta.json
Expand Down Expand Up @@ -144,9 +149,11 @@ case "$op" in
local name=${1:-current}
local mach=${2:-node-1}
local dir=$(run get "$name")
local adir=$dir/analysis
test -n "$dir" || fail "malformed run: $name"

msg "analysing run $(jq .meta.tag "$dir"/meta.json --raw-output)"
echo "{ \"run\": \"$(jq .meta.tag "$dir"/meta.json --raw-output)\" }"

local adir=$dir/analysis
mkdir -p "$adir"

## 0. subset what we care about into the keyfile
Expand All @@ -163,10 +170,11 @@ case "$op" in
## 1. enumerate logs, filter by keyfile & consolidate
local logs=($(ls "$dir"/$mach/stdout* 2>/dev/null | tac) $(ls "$dir"/$mach/node-*.json 2>/dev/null) $(ls "$dir"/analysis/$mach/node-*.json 2>/dev/null)) consolidated="$adir"/logs-$mach.json

if test -z "${logs[*]}"
then msg "no logs for $mach in run $name"; continue; fi
test -n "${logs[*]}" ||
fail "no logs for $mach in run $name"

if test -z "$skip_preparation" -o -z "$(ls "$adir"/logs-$mach.json 2>/dev/null)"
echo "{ \"prefilter\": $prefilter }"
if test "$prefilter" = 'true' -o -z "$(ls "$adir"/logs-$mach.json 2>/dev/null)"
then grep -hFf "$keyfile" "${logs[@]}" > "$consolidated"; fi

locli_args+=(
Expand Down
58 changes: 31 additions & 27 deletions nix/workbench/run.sh
View file
Expand Up @@ -63,42 +63,46 @@ case "$op" in
compute-path )
echo -n "$global_rundir/$1";;

fix-legacy-run-structure | fix-legacy )
local usage="USAGE: wb run $op TAG"
local tag=${1:?$usage}
local dir=$(run compute-path "$tag")

if test -z "$(ls -d "$dir"/node-* 2>/dev/null)"
then msg "fixing up a legacy cardano-ops run in: $dir"
if local dirs=$(ls -d "$dir"/logs/logs-*/ 2>/dev/null || true)
test -n "$dirs"
then for logdir in $dirs
do local logs_less=$(basename "$logdir" | cut -c6-)
mv "$logdir" "$dir"/$logs_less; done
elif local dirs=$(ls -d "$dir"/analysis/logs-*/ 2>/dev/null || true)
test -n "$dirs"
then for logdir in $dirs
do local logs_less=$(basename "$logdir" | cut -c6-)
mv "$logdir" "$dir"/$logs_less; done; fi
else msg "fixing up a cardano-ops run in: $dir"; fi

jq '.meta.profile_content' "$dir"/meta.json > "$dir"/profile.json

jq_fmutate "$dir"/env.json '. *
{ type: "legacy"
, staggerPorts: false
}
';;

check )
local usage="USAGE: wb run $op TAG"
local tag=${1:?$usage}
local dir=$(run compute-path "$tag")

if test "$(tr -d / <<<$tag)" != "$tag"
then fatal "run tag has slashes: $tag"; fi
test "$(tr -d / <<<$tag)" = "$tag" ||
fatal "run tag has slashes: $tag"

jq_check_json "$dir"/meta.json ||
fatal "run $tag (at $dir) missing a file: meta.json"

if test ! -f "$dir"/profile.json
then # Legacy run structure, fix up:
if test -z "$(ls -d "$dir"/logs/node-*)"
then msg "fixing up a legacy cardano-ops run in: $dir"
local topdirs=$(ls -d "$dir"/logs-*/ 2>/dev/null || true)
local anadirs=$(ls -d "$dir"/analysis/logs-*/ 2>/dev/null || true)
if test -n "$topdirs"
then for logdir in $topdirs
do local fixed=$(basename "$logdir" | cut -c6-)
mv "$logdir" "$dir"/$fixed; done
elif test -n "$anadirs"
then for logdir in $anadirs
do local fixed=$(basename "$logdir" | cut -c6-)
mv "$logdir" "$dir"/analysis/$fixed; done; fi
else msg "fixing up a cardano-ops run in: $dir"
fi

jq '.meta.profile_content' "$dir"/meta.json > "$dir"/profile.json

jq_fmutate "$dir"/env.json '. *
{ type: "legacy"
, staggerPorts: false
}
'
fi;;
test -f "$dir"/profile.json ||
run fix-legacy-run-structure "$tag";;

fix-systemstart )
local usage="USAGE: wb run $op TAG [MACH=node-1]"
Expand Down

0 comments on commit b3e913f

Please sign in to comment.