CAD-1073 bench | sanity checks: detect dbsync slot span divergence fr…

…om nodes
input-output-hk · Jul 1, 2020 · 4ba44c2 · 4ba44c2
1 parent 193b41d
commit 4ba44c2
Show file tree

Hide file tree

Showing 3 changed files with 59 additions and 0 deletions.
diff --git a/bench/lib-analyses.sh b/bench/lib-analyses.sh
@@ -27,6 +27,7 @@ analysis_unpack() {
 
         tar x -C "$dir"/analysis -af "$dir"/logs/logs-explorer.tar.xz
         tar x -C "$dir"/analysis -af "$dir"/logs/logs-nodes.tar.xz
+        tar x -C "$dir"/analysis -af "$dir"/logs/db-analysis.tar.xz '00-results-table.sql.csv'
 }
 
 analysis_list+=(analysis_log_inventory)
@@ -52,6 +53,29 @@ analysis_log_inventory()
            }' --slurpfile logs "$dir"/analysis/log-inventory.json <<<0
 }
 
+analysis_list+=(analysis_dbsync_slot_span)
+analysis_dbsync_slot_span() {
+        local dir=${1:-.}; shift
+        local machines=("$@")
+        local logs
+
+        logs=($(logs_of_nodes "$dir" "${machines[@]}"))
+
+        args=(--argjson fst_dbsync  "$(grep -E '^[0-9]+' "$dir"/analysis/00-results-table.sql.csv | head -n1 | cut -d, -f1)"
+              --argjson last_dbsync "$(grep -E '^[0-9]+' "$dir"/analysis/00-results-table.sql.csv | tail -n1 | cut -d, -f1)"
+              --argjson fst_nodes  "$(grep -Fh '"TraceStartLeadershipCheck"' "${logs[@]}" |
+                                     sort | head -n1 | jq .data.slot)"
+              --argjson last_nodes "$(grep -Fh '"TraceStartLeadershipCheck"' "${logs[@]}" |
+                                     sort | tail -n1 | jq .data.slot)"
+        )
+        json_file_prepend "$dir"/analysis.json \
+          '{ slot_spans:
+             { db_sync: { first: $fst_dbsync, last: $last_dbsync }
+             , nodes:   { first: $fst_nodes,  last: $last_nodes }
+             }
+           }' "${args[@]}" <<<0
+}
+
 analysis_list+=(analysis_timetoblock)
 analysis_timetoblock() {
         local dir=${1:-.}

diff --git a/bench/lib-analysis.sh b/bench/lib-analysis.sh
@@ -2,6 +2,14 @@
 # shellcheck disable=1091,2016
 
 
+logs_of_nodes() {
+        local dir=$1; shift
+        local machines=("$@")
+
+        for mach in ${machines[*]}
+        do ls -- "$dir"/analysis/logs-"$mach"/node-*.json; done
+}
+
 collect_jsonlog_inventory() {
         local dir=$1; shift
         local constituents=("$@")

diff --git a/bench/lib-sanity.sh b/bench/lib-sanity.sh
@@ -21,6 +21,31 @@ sanity_check_start_log_spread() {
              } + .)
           | .[]'
 }
+sanity_check_list+=(sanity_check_slot_spread_dbsync)
+sanity_check_slot_spread_dbsync() {
+        local dir=$1 t=${2:-${default_tolerances}}
+        sanity_check "$t" "$dir" '
+          $analysis.slot_spans
+          | (.db_sync.first - .nodes.first | fabs
+            | . > $allowed.slot_spread_dbsync_first)
+          ' '$analysis.slot_spans
+          | { kind:         "slot-spread-dbsync-first"
+            , dbsync_first: .db_sync.first
+            , nodes_first:  .nodes.first
+            , delta:        (.db_sync.first - .nodes.first | fabs)
+            }'
+
+        sanity_check "$t" "$dir" '
+          $analysis.slot_spans
+          | (.db_sync.last - .nodes.last | fabs
+            | . > $allowed.slot_spread_dbsync_last)
+          ' '$analysis.slot_spans
+          | { kind:         "slot-spread-dbsync-last"
+            , dbsync_last:  .db_sync.last
+            , nodes_last:   .nodes.last
+            , delta:        (.db_sync.last - .nodes.last | fabs)
+            }'
+}
 sanity_check_list+=(sanity_check_last_log_spread)
 sanity_check_last_log_spread() {
         local dir=$1 t=${2:-${default_tolerances}}
@@ -149,6 +174,8 @@ default_tolerances='
 { "tx_loss_ratio":                  0.0
 , "start_log_spread_s":             60
 , "last_log_spread_s":              60
+, "slot_spread_dbsync_first":       5
+, "slot_spread_dbsync_last":        5
 , "silence_since_last_block_s":     40
 , "cluster_startup_overhead_s":     60
 , "minimum_chain_density":          0.9