-
Notifications
You must be signed in to change notification settings - Fork 16
/
validate_producer.sh
executable file
·98 lines (84 loc) · 3.38 KB
/
validate_producer.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
#!/bin/bash
###################################################################################################
#
# This script monitors that defined producers are producing the correct number of blocks in each
# round and it also detects negative latency
#
# It is intended to be used with Slack chatops but outputs to stdout by default
#
# Please refer to https://github.com/BlockMatrixNetwork/eos-bp-failover/tree/master/producer-validator
# for more info on how to set this up
#
# Example call:
#
# ./validate_producer.sh
#
# Update APIS to include the producers you want to monitor
#
# Update NODEOS_LOG to point to your local nodeos log file
#
# Update SLACK_WEBHOOK to your own incoming webhook
#
# Made with <3 by Pete @ Block Matrix
#
# See more EOS nerd shenanigans @ https://github.com/BlockMatrixNetwork
#
###################################################################################################
# Update these for your own settings
declare -A APIS=( [pete]=blockmatrix1 [jaechung]=hkeoshkeosbp [ankh2054]=eos42freedom [mike]=eosdacserver [igorls]=eosriobrazil [xebb]=eosswedenorg )
ENDPOINT="https://proxy.eosnode.tools"
NODEOS_LOG="/mnt/stderr.txt"
# Optional slack notifications add incoming webhook url to activate it
SLACK_WEBHOOK=
SLACK_CHANNEL="#monitoring"
# Dont modify below here
SEARCH="/tmp/search.txt"
FILTER="/tmp/filter.txt"
echo "" > $FILTER
# Create temp log, assume non debug nodeos log
tail -n 5000 $NODEOS_LOG | grep "Received block" > $SEARCH
# Grab the top 21
ACTIVE=$(curl -s "$ENDPOINT/v1/chain/get_producers" -X POST -d '{"json":true, "limit":21}' | jq '.rows')
# We only care about the last 2min 6sec
LAST=$(date --date="-126 sec" "+%Y-%m-%dT%H:%M:%S")
NOW=$(date "+%Y-%m-%dT%H:%M:%S")
# Create filtered log
while read line; do
[[ $line > $LAST && $line < $NOW || $line =~ $NOW ]] && echo $line >> $FILTER
done < $SEARCH
# Handle notifications
function notify()
{
echo "$1"
[[ ! -z $SLACK_WEBHOOK ]] && curl -s -X POST --data-urlencode "payload={\"channel\": \"$SLACK_CHANNEL\", \"username\": \"EOS Bot\", \"text\": \"@$2 $1\", \"icon_emoji\": \":ghost:\"}" $SLACK_WEBHOOK > /dev/null 2>&1
}
# Check each producer
for K in "${!APIS[@]}"
do
PRODUCER=$(echo $ACTIVE | jq --arg prd "${APIS[$K]}" '.[] | select(.owner == $prd)')
PASS=1
if [ "$PRODUCER" == "" ]; then
echo "${APIS[$K]} is not in the top 21"
continue
fi
# First check is to check each producer hit their 6 second target
COUNT=$(grep -c "${APIS[$K]}" $FILTER)
if [[ "$COUNT" < "12" ]]; then
notify "${APIS[$K]} has produced less than 12 blocks: $COUNT" $K
PASS=0
fi
# Second check is to see if there has been any negative latency
NEG=$(grep "${APIS[$K]}" $FILTER | grep "latency: -")
if [ $? -eq 0 ]; then
notify "${APIS[$K]} has negative latency: $NEG" $K
PASS=0
fi
# Output if they passed all checks, the offending lines if not
if [ $PASS -eq 1 ]; then
echo "${APIS[$K]} has 12 healthy blocks and no negative latency"
else
LOG=$(grep -C 1 "${APIS[$K]}" $FILTER | sed 's/thread-0 producer_plugin.cpp:327 on_incoming_block \] //')
echo "$LOG"
[[ ! -z $SLACK_WEBHOOK ]] && curl -s -X POST --data-urlencode "payload={\"channel\": \"$SLACK_CHANNEL\", \"username\": \"EOS Bot\", \"text\": \"\`\`\`\n$LOG\n\`\`\`\", \"icon_emoji\": \":ghost:\"}" $SLACK_WEBHOOK > /dev/null 2>&1
fi
done