diff --git a/.gitignore b/.gitignore index feab1ed..188b216 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ .idea/ failover.log -app.lock \ No newline at end of file +app.lock +nodes.txt +secret.txt diff --git a/README.md b/README.md index 22cae45..38bc15e 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ## Requirements -To run the failover script you need 3 standalone servers. For the documentation we used the following names: +To run the failover script you need 3 standalone servers. For the documentation we used the following names: * master (runs the failover script with ssh connection to nodes) * relay-1 (running ark-node who is currently forging) @@ -13,20 +13,25 @@ To run the failover script you need 3 standalone servers. For the documentation * [How to secure your ark node](https://blog.ark.io/how-to-secure-your-ark-node-541254028616) * [How to set your nodes as ssh host on the master node](https://www.digitalocean.com/community/tutorials/how-to-configure-custom-connection-options-for-your-ssh-client) -**Both nodes (forging and relay) must have installed [noah](https://github.com/faustbrian/noah) correctly.** +**Both nodes (forging and relay) must have [noah](https://github.com/faustbrian/noah) installed correctly.** Note that `noah` **shouldn't** be running on these nodes, it just has to be installed! The first relay needs to be the current forging delegate node. The second relay needs to be a running ark-node that doesn't forge. The master node only need to run the failover script and have a functioning ssh connection to both nodes. + + ## Installation * `git clone https://github.com/reconnico/ark-failover.git` * `cd ~/ark-failover` * `bash failover.sh install` (if the error `No such file or directory` occurs please run `sudo updatedb`) -* insert your secret in `secret.txt` -* edit `nodes.txt` and set your nodes `forging;relay` (use ssh host names of your nodes here) + +## Configuration +* create `secret.txt` file and insert your secret. Check `secret.sample.txt` for an example. +* create `nodes.txt` file and set your nodes `forging;relay`. Use ssh host names of your nodes here. Check `nodes.sample.txt` for an example. * `bash failover.sh test` (If an error occur please repeat the last 2 steps and check your configuration) * `bash failover.sh start` +* modify vars in `variables.sh`. Example, change `network` to either `mainnet` or `devnet` (it's set to `devnet` by default), also double check if paths are set correctly. ## Commands diff --git a/modules/ark.sh b/modules/ark.sh index 1548092..11462d2 100644 --- a/modules/ark.sh +++ b/modules/ark.sh @@ -10,27 +10,22 @@ # ===================== ark_start() { - ## TODO: test when ark process is not running - ssh $1 network=${network} export_path=${export_path} 'bash -s' <<'ENDSSH' + log "Starting/Restarting node: $1" + ssh -o ConnectTimeout=10 $1 network=${network} ark_path=${ark_path} export_path=${export_path} 'bash -s' <<'ENDSSH' PATH=${export_path}:$PATH export PATH - node=`pgrep -a "node" | grep ark-node | awk '{print $1}'` - forever_process=`forever --plain list | grep ${node} | sed -nr 's/.*\[(.*)\].*/\1/p'` - - if [ "${node}" != "" ] && [ "${node}" != "0" ]; then - forever restart ${forever_process} >&- 2>&- - else - forever start app.js --genesis genesisBlock.${network}.json --config config.${network}.json >&- 2>&- - fi + forever stopall + forever start --workingDir ${ark_path} ${ark_path}app.js --genesis genesisBlock.${network}.json --config config.${network}.json ENDSSH } # ===================== +# fn currently unused # @param node $1 # ===================== ark_stop() { - ssh $1 export_path=${export_path} 'bash -s' <<'ENDSSH' + ssh -o ConnectTimeout=10 $1 export_path=${export_path} 'bash -s' <<'ENDSSH' PATH=${export_path}:$PATH export PATH forever stopall @@ -42,16 +37,17 @@ ENDSSH # ===================== block_height() { - blockheight_node=$(ssh $1 'echo $(psql -d ark_mainnet -t -c "SELECT height FROM blocks ORDER BY HEIGHT DESC LIMIT 1;" | xargs)') - blockheight_net=$(ssh $1 'heights=$(curl -s "http://localhost:4001/api/peers" | jq -r ".peers[] | .height") && echo $(echo "${heights[*]}" | sort -nr | head -n1)') + blockheight_node=$(ssh -o ConnectTimeout=10 $1 "psql -d ${db} -t -c 'SELECT height FROM blocks ORDER BY HEIGHT DESC LIMIT 1;'") + blockheight_net=$(ssh -o ConnectTimeout=10 $1 network_port=${network_port} 'heights=$(curl -s "http://localhost:${network_port}/api/peers" | jq -r ".peers[] | .height") && echo $(echo "${heights[*]}" | sort -nr | head -n1)') } # ===================== +# fn currently unused # @param node $1 # ===================== is_forging() { - result=`ssh $1 pubkey=${pubkey} "curl -s --connect-timeout 1 http://localhost:4001/api/delegates/forging/status?publicKey=$pubkey 2>/dev/null | jq \".enabled\""` + result=`ssh -o ConnectTimeout=10 $1 network_port=${network_port} pubkey=${pubkey} "curl -s --connect-timeout 1 http://localhost:{$network_port}/api/delegates/forging/status?publicKey=$pubkey 2>/dev/null | jq \".enabled\""` if [ $result = true ]; then echo $result @@ -59,4 +55,4 @@ is_forging() fi return 0 -} \ No newline at end of file +} diff --git a/modules/monitor.sh b/modules/monitor.sh index d146610..ac8cd49 100644 --- a/modules/monitor.sh +++ b/modules/monitor.sh @@ -27,45 +27,52 @@ monitor_nodes() { ## check block height of forging node block_height ${node_forging} - if [ -z ${blockheight_net} ] || [ $((${blockheight_net} - ${blockheight_node})) -gt 1 ]; then + + if [ -z ${blockheight_net} ] || [ $((${blockheight_net} - ${blockheight_node})) -gt 15 ]; then ## forging is out of sync + log "Forging node ${node_forging} is out of sync..." + SECONDS=0 lock_create - log "[switch] ${node_forging} -> ${node_relay}" + log "[switch: ${node_forging}] ${node_forging} -> ${node_relay}" ## set secret on relay set_secret ${node_relay} ${secret} ark_start ${node_relay} - log "[set secret on relay] finished!" + log "[set secret on relay: ${node_forging}] finished!" - log "[blockheight] ${blockheight_node}/${blockheight_net} (node/net)" + log "[blockheight: ${node_forging}] ${blockheight_node}/${blockheight_net} (node/net)" ## reset secret on forging node set_secret ${node_forging} log "[reset secret on forging] finished!" ## rebuild database and reset secret on forging - log "[rebuild] ${node_forging} starting..." + log "[rebuild: ${node_forging}] starting..." rebuild ${node_forging} - log "[rebuild] ${node_forging} finished!" + ## todo: if rebuild fails (server being down) don't show bellow message, show an error + log "[rebuild: ${node_forging}] finished!" ## set nodes.txt set_nodes ${node_relay} ${node_forging} - log "[set nodes] finished!" + log "[set nodes: ${node_forging}] finished!" monitor_finish fi ## check block height of relay node block_height ${node_relay} - if [ -z ${blockheight_net} ] || [ $((${blockheight_net} - ${blockheight_node})) -gt 9 ]; then + + if [ -z ${blockheight_net} ] || [ $((${blockheight_net} - ${blockheight_node})) -gt 15 ]; then ## relay is out of sync + log "Relay node ${node_relay} is out of sync..." SECONDS=0 lock_create - log "[rebuild] ${node_relay} starting..." - log "[blockheight] ${blockheight_node}/${blockheight_net} (node/net)" + log "[rebuild ${node_relay}] starting..." + log "[blockheight ${node_relay}] ${blockheight_node}/${blockheight_net} (node/net)" rebuild ${node_relay} - log "[rebuild] ${node_relay} finished!" + ## todo: if rebuild fails (server being down) don't show bellow message, show an error + log "[rebuild ${node_relay}] finished!" monitor_finish fi diff --git a/modules/rebuild.sh b/modules/rebuild.sh index b4ab42f..4dafd3d 100644 --- a/modules/rebuild.sh +++ b/modules/rebuild.sh @@ -10,11 +10,11 @@ # ===================== rebuild() { - ssh $1 export_path=${export_path} 'bash -s' <<'ENDSSH' + ssh -o ConnectTimeout=10 $1 export_path=${export_path} 'bash -s' <<'ENDSSH' PATH=${export_path}:$PATH export PATH cd ~/noah ls -la bash noah.sh rebuild ENDSSH -} \ No newline at end of file +} diff --git a/modules/secret.sh b/modules/secret.sh index b813c5c..4af85b7 100644 --- a/modules/secret.sh +++ b/modules/secret.sh @@ -26,7 +26,7 @@ set_secret() let INDEX=${INDEX}+1 done - ssh ${relay} network=${network} secret=${QUOTE_ARGS:2} 'bash -s' <<'ENDSSH' + ssh -o ConnectTimeout=10 ${relay} network=${network} secret=${QUOTE_ARGS:2} 'bash -s' <<'ENDSSH' cd `locate -b '\ark-node'` jq -r ".forging.secret = [\"${secret}\"]" config.${network}.json > config.${network}.tmp && mv config.${network}.tmp config.${network}.json ENDSSH @@ -35,4 +35,4 @@ ENDSSH get_secret() { secret=`cat ${app_dir}/secret.txt` -} \ No newline at end of file +} diff --git a/modules/test.sh b/modules/test.sh index 399bd33..2a8dd4c 100644 --- a/modules/test.sh +++ b/modules/test.sh @@ -19,7 +19,7 @@ test_run() # ===================== test_ssh() { - ssh -q $1 exit + ssh -o ConnectTimeout=10 -q $1 exit if [ $? == 255 ] then @@ -32,7 +32,7 @@ test_ssh() # ===================== test_noah() { - ssh $1 "bash noah/noah.sh test test || exit 12" + ssh -o ConnectTimeout=10 $1 "bash noah/noah.sh test test || exit 12" if [[ $? = 12 ]]; then error "Noah isn't installed correctly on $1! Please make sure that noah runs correctly on your nodes." @@ -43,4 +43,4 @@ test_secret() { ## TODO: implement secret check error "Secret is not correct! Please check your configuration." -} \ No newline at end of file +} diff --git a/nodes.txt b/nodes.sample.txt similarity index 100% rename from nodes.txt rename to nodes.sample.txt diff --git a/secret.txt b/secret.sample.txt similarity index 100% rename from secret.txt rename to secret.sample.txt diff --git a/variables.sh b/variables.sh index a58dfae..9b94f64 100644 --- a/variables.sh +++ b/variables.sh @@ -9,15 +9,24 @@ # Network # ===================== -network='mainnet' ## devnet +network='devnet' ## mainnet +if [[ $network == 'devnet' ]] + then + network_port=4002 + db="ark_devnet" + else + network_port=4001 + db="ark_mainnet" +fi # ===================== # Monitor # ===================== -monitor_sleep=120 +monitor_sleep=180 monitor_interval=7 # ===================== # Path # ===================== -export_path='/home/$USER/.nvm/versions/node/v6.9.5/bin' \ No newline at end of file +export_path='/home/$USER/.nvm/versions/node/v6.9.5/bin' +ark_path='/home/$USER/ark-node/'