watcher

#!/usr/bin/env python3
# Watch for genesis change, re-nearup when that happens
# This tool is launched by main.py as a background process

import hashlib
import logging
import os
import sys
import time

from logging import handlers

import click

from nearuplib.exceptions import NetworkError
from nearuplib.nodelib import restart_nearup, is_neard_zombie
from nearuplib.util import (
    latest_deployed_release_commit,
    latest_deployed_release_commit_has_changed,
    latest_genesis_md5sum,
    read_genesis_md5sum,
)

logging.basicConfig(
    level=logging.INFO,
    format=
    '%(asctime)s.%(msecs)03d %(levelname)s %(module)s - %(funcName)s: %(message)s',
    datefmt='%Y-%m-%d %H:%M:%S',
    handlers=[
        handlers.RotatingFileHandler(
            os.path.expanduser('~/.nearup/logs/watcher.log'),
            maxBytes=1024 * 1024,
            backupCount=10,
        ),
    ],
)


@click.group()
def main():
    pass


@main.command()
@click.argument('network',
                type=click.Choice(
                    {'guildnet', 'betanet', 'testnet', 'mainnet', 'shardnet'}))
@click.argument('home', type=str)
@click.option('--force-restart',
              is_flag=True,
              help='Force restart nearup in a loop, USED ONLY FOR TESTING')
def run(network, home, force_restart):
    current_release_commit = latest_deployed_release_commit(network)
    # Don't assume that the node has an up-to-date version of genesis.
    # Instead read the md5sum hash of the genesis that existed when we initialized home.
    current_genesis_md5sum = read_genesis_md5sum(home)

    logging.info(
        f"Starting watcher for chain: {network} with commit: {current_release_commit}"
    )

    if is_neard_zombie():
        logging.warning(
            "Detected that neard is dead when starting watcher. Restarting neard."
        )
        restart_nearup(network, home_dir=home, restart_only_new_version=False)

    while True:
        time.sleep(60)
        try:
            genesis_md5sum = latest_genesis_md5sum(network)

            if latest_deployed_release_commit_has_changed(
                    network, current_release_commit
            ) or genesis_md5sum != current_genesis_md5sum or force_restart:
                logging.info(
                    "New release has been published. Restarting nearup")

                restart_nearup(network,
                               home_dir=home,
                               restart_only_new_version=False)
                current_release_commit = latest_deployed_release_commit(network)
                current_genesis_md5sum = genesis_md5sum
            elif is_neard_zombie():
                logging.warning(
                    "Detected that neard has died. Restarting neard.")
                restart_nearup(network,
                               home_dir=home,
                               restart_only_new_version=False)

        except NetworkError as ex:
            logging.warning(f'caught networking error {ex} - will try again')

        except Exception as ex:
            logging.error(ex)
            sys.exit(1)


if __name__ == '__main__':
    main()