Merge pull request #1 from roks0n/modifications

Fixes and improvements

.gitignore

@@ -1,3 +1,5 @@
 .idea/
 failover.log
-app.lock
+app.lock
+nodes.txt
+secret.txt

README.md

@@ -2,7 +2,7 @@
 
 ## Requirements
 
-To run the failover script you need 3 standalone servers. For the documentation we used the following names: 
+To run the failover script you need 3 standalone servers. For the documentation we used the following names:
 
 * master (runs the failover script with ssh connection to nodes)
 * relay-1 (running ark-node who is currently forging)
@@ -13,20 +13,25 @@ To run the failover script you need 3 standalone servers. For the documentation
 * [How to secure your ark node](https://blog.ark.io/how-to-secure-your-ark-node-541254028616)
 * [How to set your nodes as ssh host on the master node](https://www.digitalocean.com/community/tutorials/how-to-configure-custom-connection-options-for-your-ssh-client)
 
-**Both nodes (forging and relay) must have installed [noah](https://github.com/faustbrian/noah) correctly.**
+**Both nodes (forging and relay) must have [noah](https://github.com/faustbrian/noah) installed correctly.** Note that `noah` **shouldn't** be running on these nodes, it just has to be installed!
 
 The first relay needs to be the current forging delegate node. The second relay needs to be a running ark-node that doesn't forge.
 The master node only need to run the failover script and have a functioning ssh connection to both nodes.
 
+
+
 ## Installation
 
 * `git clone https://github.com/reconnico/ark-failover.git`
 * `cd ~/ark-failover`
 * `bash failover.sh install` (if the error `No such file or directory` occurs please run `sudo updatedb`)
-* insert your secret in `secret.txt`
-* edit `nodes.txt` and set your nodes `forging;relay` (use ssh host names of your nodes here)
+
+## Configuration
+* create `secret.txt` file and insert your secret. Check `secret.sample.txt` for an example.
+* create `nodes.txt` file and set your nodes `forging;relay`. Use ssh host names of your nodes here. Check `nodes.sample.txt` for an example.
 * `bash failover.sh test` (If an error occur please repeat the last 2 steps and check your configuration)
 * `bash failover.sh start`
+* modify vars in `variables.sh`. Example, change `network` to either `mainnet` or `devnet` (it's set to `devnet` by default), also double check if paths are set correctly.
 
 ## Commands
 

modules/ark.sh

@@ -10,27 +10,22 @@
 # =====================
 ark_start()
 {
-    ## TODO: test when ark process is not running
-    ssh $1 network=${network} export_path=${export_path} 'bash -s' <<'ENDSSH'
+    log "Starting/Restarting node: $1"
+    ssh -o ConnectTimeout=10 $1 network=${network} ark_path=${ark_path} export_path=${export_path} 'bash -s' <<'ENDSSH'
     PATH=${export_path}:$PATH
     export PATH
-    node=`pgrep -a "node" | grep ark-node | awk '{print $1}'`
-    forever_process=`forever --plain list | grep ${node} | sed -nr 's/.*\[(.*)\].*/\1/p'`
-
-    if [ "${node}" != "" ] && [ "${node}" != "0" ]; then
-        forever restart ${forever_process} >&- 2>&-
-    else
-        forever start app.js --genesis genesisBlock.${network}.json --config config.${network}.json >&- 2>&-
-    fi
+    forever stopall
+    forever start --workingDir ${ark_path} ${ark_path}app.js --genesis genesisBlock.${network}.json --config config.${network}.json
 ENDSSH
 }
 
 # =====================
+# fn currently unused
 # @param node $1
 # =====================
 ark_stop()
 {
-    ssh $1 export_path=${export_path} 'bash -s' <<'ENDSSH'
+    ssh -o ConnectTimeout=10 $1 export_path=${export_path} 'bash -s' <<'ENDSSH'
     PATH=${export_path}:$PATH
     export PATH
     forever stopall
@@ -42,21 +37,22 @@ ENDSSH
 # =====================
 block_height()
 {
-    blockheight_node=$(ssh $1 'echo $(psql -d ark_mainnet -t -c "SELECT height FROM blocks ORDER BY HEIGHT DESC LIMIT 1;" | xargs)')
-    blockheight_net=$(ssh $1 'heights=$(curl -s "http://localhost:4001/api/peers" | jq -r ".peers[] | .height") && echo $(echo "${heights[*]}" | sort -nr | head -n1)')
+    blockheight_node=$(ssh -o ConnectTimeout=10 $1 "psql -d ${db} -t -c 'SELECT height FROM blocks ORDER BY HEIGHT DESC LIMIT 1;'")
+    blockheight_net=$(ssh -o ConnectTimeout=10 $1 network_port=${network_port} 'heights=$(curl -s "http://localhost:${network_port}/api/peers" | jq -r ".peers[] | .height") && echo $(echo "${heights[*]}" | sort -nr | head -n1)')
 }
 
 # =====================
+# fn currently unused
 # @param node $1
 # =====================
 is_forging()
 {
-    result=`ssh $1 pubkey=${pubkey} "curl -s --connect-timeout 1 http://localhost:4001/api/delegates/forging/status?publicKey=$pubkey 2>/dev/null | jq \".enabled\""`
+    result=`ssh -o ConnectTimeout=10 $1 network_port=${network_port} pubkey=${pubkey} "curl -s --connect-timeout 1 http://localhost:{$network_port}/api/delegates/forging/status?publicKey=$pubkey 2>/dev/null | jq \".enabled\""`
 
     if [ $result = true ]; then
         echo $result
         return 1
     fi
 
     return 0
-}
+}

modules/monitor.sh

@@ -27,45 +27,52 @@ monitor_nodes()
 {
     ## check block height of forging node
     block_height ${node_forging}
-    if [ -z ${blockheight_net} ] || [ $((${blockheight_net} - ${blockheight_node})) -gt 1 ]; then
+
+    if [ -z ${blockheight_net} ] || [ $((${blockheight_net} - ${blockheight_node})) -gt 15 ]; then
         ## forging is out of sync
+        log "Forging node ${node_forging} is out of sync..."
+
         SECONDS=0
         lock_create
-        log "[switch] ${node_forging} -> ${node_relay}"
+        log "[switch: ${node_forging}] ${node_forging} -> ${node_relay}"
 
         ## set secret on relay
         set_secret ${node_relay} ${secret}
         ark_start ${node_relay}
-        log "[set secret on relay] finished!"
+        log "[set secret on relay: ${node_forging}] finished!"
 
-        log "[blockheight] ${blockheight_node}/${blockheight_net} (node/net)"
+        log "[blockheight: ${node_forging}] ${blockheight_node}/${blockheight_net} (node/net)"
 
         ## reset secret on forging node
         set_secret ${node_forging}
         log "[reset secret on forging] finished!"
 
         ## rebuild database and reset secret on forging
-        log "[rebuild] ${node_forging} starting..."
+        log "[rebuild: ${node_forging}] starting..."
         rebuild ${node_forging}
-        log "[rebuild] ${node_forging} finished!"
+        ## todo: if rebuild fails (server being down) don't show bellow message, show an error
+        log "[rebuild: ${node_forging}] finished!"
 
         ## set nodes.txt
         set_nodes ${node_relay} ${node_forging}
-        log "[set nodes] finished!"
+        log "[set nodes: ${node_forging}] finished!"
 
         monitor_finish
     fi
 
     ## check block height of relay node
     block_height ${node_relay}
-    if [ -z ${blockheight_net} ] || [ $((${blockheight_net} - ${blockheight_node})) -gt 9 ]; then
+
+    if [ -z ${blockheight_net} ] || [ $((${blockheight_net} - ${blockheight_node})) -gt 15 ]; then
         ## relay is out of sync
+        log "Relay node ${node_relay} is out of sync..."
         SECONDS=0
         lock_create
-        log "[rebuild] ${node_relay} starting..."
-        log "[blockheight] ${blockheight_node}/${blockheight_net} (node/net)"
+        log "[rebuild ${node_relay}] starting..."
+        log "[blockheight ${node_relay}] ${blockheight_node}/${blockheight_net} (node/net)"
         rebuild ${node_relay}
-        log "[rebuild] ${node_relay} finished!"
+        ## todo: if rebuild fails (server being down) don't show bellow message, show an error
+        log "[rebuild ${node_relay}] finished!"
 
         monitor_finish
     fi

modules/rebuild.sh

@@ -10,11 +10,11 @@
 # =====================
 rebuild()
 {
-    ssh $1 export_path=${export_path} 'bash -s' <<'ENDSSH'
+    ssh -o ConnectTimeout=10 $1 export_path=${export_path} 'bash -s' <<'ENDSSH'
         PATH=${export_path}:$PATH
         export PATH
         cd ~/noah
         ls -la
         bash noah.sh rebuild
 ENDSSH
-}
+}

modules/secret.sh

@@ -26,7 +26,7 @@ set_secret()
         let INDEX=${INDEX}+1
     done
 
-    ssh ${relay} network=${network} secret=${QUOTE_ARGS:2} 'bash -s' <<'ENDSSH'
+    ssh -o ConnectTimeout=10 ${relay} network=${network} secret=${QUOTE_ARGS:2} 'bash -s' <<'ENDSSH'
     cd `locate -b '\ark-node'`
     jq -r ".forging.secret = [\"${secret}\"]" config.${network}.json > config.${network}.tmp && mv config.${network}.tmp config.${network}.json
 ENDSSH
@@ -35,4 +35,4 @@ ENDSSH
 get_secret()
 {
     secret=`cat ${app_dir}/secret.txt`
-}
+}

modules/test.sh

@@ -19,7 +19,7 @@ test_run()
 # =====================
 test_ssh()
 {
-    ssh -q $1 exit
+    ssh -o ConnectTimeout=10 -q $1 exit
 
     if [ $? == 255 ]
         then
@@ -32,7 +32,7 @@ test_ssh()
 # =====================
 test_noah()
 {
-    ssh $1 "bash noah/noah.sh test test || exit 12"
+    ssh -o ConnectTimeout=10 $1 "bash noah/noah.sh test test || exit 12"
 
     if [[ $? = 12 ]]; then
         error "Noah isn't installed correctly on $1! Please make sure that noah runs correctly on your nodes."
@@ -43,4 +43,4 @@ test_secret()
 {
     ## TODO: implement secret check
     error "Secret is not correct! Please check your configuration."
-}
+}

variables.sh

@@ -9,15 +9,24 @@
 # Network
 # =====================
 
-network='mainnet' ## devnet
+network='devnet' ## mainnet
+if [[ $network == 'devnet' ]]
+    then
+        network_port=4002
+        db="ark_devnet"
+    else
+        network_port=4001
+        db="ark_mainnet"
+fi
 
 # =====================
 # Monitor
 # =====================
-monitor_sleep=120
+monitor_sleep=180
 monitor_interval=7
 
 # =====================
 # Path
 # =====================
-export_path='/home/$USER/.nvm/versions/node/v6.9.5/bin'
+export_path='/home/$USER/.nvm/versions/node/v6.9.5/bin'
+ark_path='/home/$USER/ark-node/'