Merge pull request #15 from Netherdrake/f/rpchealth

Enforce healthy RPC usage in watchdog

conductor/utils.py

@@ -1,10 +1,11 @@
-from steem.utils import env_unlocked
+from steem.utils import env_unlocked, parse_time
 from steem.wallet import Wallet
 from steembase.account import PrivateKey
 from steembase.storage import (
     configStorage,
     MasterPassword,
 )
+from datetime import datetime as dt
 
 
 def generate_signing_key():
@@ -27,3 +28,17 @@ def unlock_steempy_wallet():
         print('steempy wallet does not exist.'
               'Please import your active key before publishing feeds.')
         quit(1)
+
+
+def head_block_lag(steemd_instance) -> int:
+    """ Return age of head block (in seconds)."""
+    s = steemd_instance
+    head_block = s.get_block_header(s.head_block_number)
+    head_block_time = parse_time(head_block['timestamp'])
+    return (dt.utcnow() - head_block_time).seconds
+
+
+def wait_for_healthy_node(steem, blocks=100):
+    while head_block_lag(steem) > blocks:
+        print("RPC Node %s is unhealthy. Skipping..." % steem.hostname)
+        steem.next_node()

conductor/watchdog.py

@@ -6,7 +6,11 @@
 from steem import Steem
 
 from .config import witness, props
-from .utils import unlock_steempy_wallet
+from .utils import (
+    unlock_steempy_wallet,
+    head_block_lag,
+    wait_for_healthy_node,
+)
 
 steem = Steem(tcp_keepalive=False)
 null_key = 'STM1111111111111111111111111111111114T1Anm'
@@ -74,11 +78,15 @@ def watchdog(disable_after: int, keys: List[str]):
     # unlock the wallet when process starts
     unlock_steempy_wallet()
 
+    wait_for_healthy_node(steem)
     misses = total_missed()
     miss_history = []
     print("Monitoring the witness, current misses: %s" % misses)
     while True:
         try:
+            # enforce we're on a healthy rpc node
+            wait_for_healthy_node(steem)
+
             # detect new misses
             diff = total_missed() - misses
             if diff:

setup.py

@@ -8,7 +8,7 @@
 
 setup(
     name='conductor',
-    version='0.3.3',
+    version='0.3.4',
     description='Steem Witness Toolkit',
     long_description=open('README.rst').read(),
     packages=find_packages(),