[d] Bash script to automatically restart Tezos programs wisely (revised 09 29 19)

More info about the purpose of this section can be found in page [b] point F-26).
A different crontab instruction should be used, replace this:

0 1 */2 * * /home/<user>/tz-restart.sh

with

0 1 */2 * * /home/<user>/restart.sh

---

The bash file restart.sh should include the following lines:

#! /bin/bash
D=$(ls /tmp/.X11-unix |tr 'X' ':')
DISPLAY=$D xterm -geometry 120x40+930+0 -e bash -c "/home/<user>/tz-restart.sh"&

---

The improved tz-restart.sh file is this:

################################################################
#! /bin/bash

# REPLACE port number, <user> and <address_baking_account> WHERE NEEDED

# BEFORE THE RESTART OPERATION
# read the existing list of endorsing/baking blocks in the file "tz-blocks" then trim old blocks
# and then if needed update the list
# by poking the network for endorsement and baking slots at new cycles
# and finally
# adjust the sleep time based on the vicinity of endorsing and baking operations

# pause
sleep_time=10

################################################################
# this is the file where the endorsement and baking block numbers are stored
blocks_file="tz-blocks"

# read the block numbers
i=1
while IFS=" " read -r block[$i] cycle[$i]
do
cyclen[$i]=${cycle[$i]##*-}
echo "block " $i ${block[$i]} ${cyclen[$i]}
i=$((i+1))
done < "$blocks_file"

# adjust the counter (no extra space at the end of the file tz-blocks)
i=$((i-1))
imax=$i
cyclemax=${cyclen[$imax]}
################################################################

current_cycle=$(TEZOS_CLIENT_UNSAFE_DISABLE_DISCLAIMER=y /home/<user>/tezos/tezos-client -port 8733 rpc get /chains/main/blocks/head/helpers/current_level | grep "level_position" | awk -F'[, ]' '{print $9}')
echo "current cycle" $current_cycle

######################################################################
# trim old blocks before the current cycle
rm tz-blocks

for j in `seq 1 $i`;
do
if [ ${cyclen[$j]} -ge $current_cycle ]
then
echo ${block[$j]} ${cycle[$j]} >> tz-blocks
fi
done

######################################################################
# check if some cycles are missing from the file list
if [ $cyclemax -lt $(($current_cycle + 5)) ]
then\

# how many new cycles need to be added to the file list
n_new_cycles=`expr $current_cycle + 5 - $cyclemax `

################################################################
# loop for 2 accounts
acct[1]="tz1RGNGDdZtZU2ku2nWSUwFzDiWookarfiX2"
acct[2]="xxxx"

# actually no, just one account
for nacct in {1..1}
do

acct=${acct[$nacct]}

######################################################################
# big loop to add endorsements and baking from new cylcles
for j in `seq 1 $n_new_cycles`;
do
new_cycle=`expr $cyclemax + $j `
echo "#"$j "new cycle" $new_cycle "acct" $acct

######################################################################
# POKE FOR NEW ENDORSEMENTS
######################################################################
# TEZOS_CLIENT_UNSAFE_DISABLE_DISCLAIMER=y
/home/<user>/tezos/tezos-client -port 8733 rpc get /chains/main/blocks/head/helpers/endorsing_rights?"cycle=$new_cycle&delegate=$acct" > dummy

k=1
while IFS= read -r line[$k]
do
# echo ${line[$k]}

if [ $k -eq 1 ]
then
# echo ${line[$k]} | grep level |awk -F'[, ]' '{print $4}'
nblock=$(echo ${line[$k]} | grep level |awk -F'[, ]' '{print $4}')
else
# echo ${line[$k]} | grep level |awk -F'[, ]' '{print $3}'
nblock=$(echo ${line[$k]} | grep level |awk -F'[, ]' '{print $3}')
fi

if [ $nblock ]
then
echo $nblock "E-"$new_cycle
echo $nblock "E-"$new_cycle >> tz-blocks
fi

k=$((k+1))
done < "dummy"
rm dummy

######################################################################
# POKE FOR NEW BAKING
######################################################################
# TEZOS_CLIENT_UNSAFE_DISABLE_DISCLAIMER=y
/home/<user>/tezos/tezos-client -port 8733 rpc get /chains/main/blocks/head/helpers/baking_rights?"cycle=$new_cycle&delegate=$acct&max_priority=4" > dummy
k=1
while IFS= read -r line[$k]
do
# echo ${line[$k]}

if [ $((k%2)) -ne 0 ]
then
# odd lines
# echo "odd" $k

if [ $k -eq 1 ]
then
# echo ${line[$k]} | grep level |awk -F'[, ]' '{print $4}'
nblock=$(echo ${line[$k]} | grep level |awk -F'[, ]' '{print $4}')
else
# echo ${line[$k]} | grep level |awk -F'[, ]' '{print $3}'
nblock=$(echo ${line[$k]} | grep level |awk -F'[, ]' '{print $3}')
fi

else
# even lines
# echo "even" $k

prio=$(echo ${line[$k]} | grep priority |awk -F'[, ]' '{print $2}')
echo $nblock "B"$prio"-"$new_cycle
echo $nblock "B"$prio"-"$new_cycle >> tz-blocks
fi

k=$((k+1))
done < "dummy"
rm dummy

######################################################################
# end of the loop for the missing cycles
done

# end of the loop for the 2 accounts
done

fi
######################################################################

# pause
sleep_time=10
################################################################
# read current block from rpc command
current_block=$(/home/max/tezos/tezos-client -port 8735 rpc get /chains/main/blocks/head/helpers/current_level? | grep level |awk -F'[, ]' '{print $3}')

# read current block from file (for testing purposes)
# current_block=$(cat tz-current-block | grep level |awk -F'[, ]' '{print $3}')

echo "current block " $current_block

################################################################
# this is the file where the endorsement and baking block numbers are stored
blocks_file="tz-blocks"

##### first sort it out ####
cp "$blocks_file" dummy
sort -n dummy > "$blocks_file"
rm dummy

# read the block numbers
i=1
while IFS=" " read -r block[$i] cycle
do
echo "block " $i ${block[$i]} $cycle
i=$((i+1))
done < "$blocks_file"

# adjust the counter (no extra space at the end of the file tz-blocks)
i=$((i-1))
imax=$i
cyclemax=${cyclen[$imax]}
#########################################################################################
# need to find in the list the first block that is equal or greater than the current block
first_p=$imax
for j in `seq 1 $imax`;
do

if [ ${block[$j]} -ge $current_block ]
then
if [ ${block[$j]} -lt ${block[$first_p]} ]
then
first_p=$j
fi
fi

done

if [ $first_p -eq $imax ]
then
echo "hoho we have a problem, all the blocks in the list are older than the current block"
echo "set this block as the last, and restart right away (fingers crossed)"
fi

echo "first future block next to the current block" $current_block "is #"$first_p ${block[$first_p]}

##########################################################################################
# if the first incoming block is at least 15 blocks away from the current block, then restart now
# alternatively check if the difference between the slowest appearance of a block
# and the fastest appearance of the next block is big enough to allow for a restart
# fastest progress -> 1 block every minute
# slowest progress -> start with 1 block every 2 minutes(20/10) then 18/10,16/10...10/10
# conservative restart time set to 15 blocks

if [ $((current_block +15)) -lt ${block[$first_p]} ]
then
sleep_time=$sleep_time
echo "got it! restart right away" $first_p ${block[$first_p]} $sleep_time"sec"

elif [ $first_p -eq $imax ]
then
sleep_time=$sleep_time
echo "not good! all blocks older than the current but restart right away" $imax ${block[$imax]} $sleep_time"sec"

elif [ $((current_block +15)) -gt ${block[$imax]} ]
then
sleep_time=`expr $sleep_time + 60 * 2 * ( ${block[$imax]} - $current_block ) `
echo "not good! current block + 15 is greater than the last block in the list, restart after it" $imax ${block[$imax]} $sleep_time"sec"\

else

### the first incoming block is too close to the current, search for another one
for scale in `seq 20 -2 10`;
do
echo "scale" $scale"/10"

for j in `seq $first_p $((imax-1))`;
do

blockj=${block[$j]}
jp=$((j+1))
blockjp=${block[$jp]}

if [ `expr $blockj + 15` -lt $blockjp ] && [ `expr $current_block + 15 + $scale * ( $blockj - $current_block ) / 10` -lt $blockjp ]
then
echo $blockjp
echo `expr $blockj + 15` `expr $current_block + 15 + $scale * ( $blockj - $current_block ) / 10` "scale" $scale
sleep_time=`expr $sleep_time + 60 * $scale * ( $blockj - $current_block ) / 10 `
echo "got it! restart later" $j $blockj `expr ( $blockj - $current_block )` $sleep_time"sec"
break 2
fi

done
done

if [ $j -eq $((imax-1)) ] && [ $scale -eq 10 ]
then
echo "ohoh we have a problem, no time windows available for restart, start now"
# exit
sleep_time=$sleep_time
fi

fi

##########################################################################
# set the sleep time before starting the stop/start operations

sleep $sleep_time

#############################################################################
# tz-stop

sleep 10
kill -2 $(pgrep -f tezos-baker-004-Pt24m4xi)
sleep 10
kill -2 $(pgrep -f tezos-endorser-004-Pt24m4xi)
sleep 20
kill -2 $(pgrep -f tezos-node)
sleep 60
sync ; echo 3 | sudo tee -a /proc/sys/vm/drop_caches

##############################################################################
# the following start commands should be adjusted according to the owner setup
# (for example below it is assumed an X11 desktop, rpc port 8733 and local tezos signer via port 9793/tcp)
# tz-start

D=$(ls /tmp/.X11-unix |tr 'X' ':')

sleep 20
DISPLAY=$D xterm -geometry 178x40+902+450 -hold -e bash -c "/home/<user>/tezos/tezos-node run"&
sleep 120

DISPLAY=$D xterm -geometry 120x40+0+0 -hold -e bash -c "/home/<user>/tezos/tezos-baker-004-Pt24m4xi -port 8733 -R tcp://localhost:9793/<address_baking_account> run with local node /media/hd/.tezos-node <alias_baking_account>"&
sleep 30
DISPLAY=$D xterm -geometry 120x40+0+470 -hold -e bash -c "/home/<user>/tezos/tezos-endorser-004-Pt24m4xi -port 8733 -R tcp://localhost:9793/<address_baking_account> run <alias_baking_account>"&
sleep 10

################################################################

(NOTE: if the remote-signer is located on a different machine, replace tcp://localhost:9793 with the IP address of the remote-signer machine, for example tcp://192.168.0.1:9793)