diff --git a/modules/__init__.py b/modules/__init__.py index d639c4fb..afb61e14 100644 --- a/modules/__init__.py +++ b/modules/__init__.py @@ -8,6 +8,7 @@ from modules.validator import ValidatorModule from modules.controller import ControllerModule from modules.liteserver import LiteserverModule +from modules.alert_bot import AlertBotModule MODES = { @@ -15,7 +16,8 @@ 'nominator-pool': NominatorPoolModule, 'single-nominator': SingleNominatorModule, 'liquid-staking': ControllerModule, - 'liteserver': LiteserverModule + 'liteserver': LiteserverModule, + 'alert-bot': AlertBotModule } @@ -55,6 +57,8 @@ class Setting: 'defaultCustomOverlaysUrl': Setting(None, 'https://ton-blockchain.github.io/fallback_custom_overlays.json', 'Default custom overlays config url'), 'debug': Setting(None, False, 'Debug mtc console mode. Prints Traceback on errors'), 'subscribe_tg_channel': Setting('validator', False, 'Disables warning about subscribing to the `TON STATUS` channel'), + 'BotToken': Setting('alert-bot', None, 'Alerting Telegram bot token'), + 'ChatId': Setting('alert-bot', None, 'Alerting Telegram chat id') } diff --git a/modules/alert_bot.py b/modules/alert_bot.py new file mode 100644 index 00000000..5ba4d99a --- /dev/null +++ b/modules/alert_bot.py @@ -0,0 +1,274 @@ +import dataclasses +import time +import requests + +from modules.module import MtcModule +from mypylib.mypylib import get_timestamp, print_table, color_print +from mytoncore import get_hostname +from mytonctrl.utils import timestamp2utcdatetime + + +@dataclasses.dataclass +class Alert: + severity: str + text: str + timeout: int + + +HOUR = 3600 +VALIDATION_PERIOD = 65536 +FREEZE_PERIOD = 32768 + + +ALERTS = { + "low_wallet_balance": Alert( + "low", + "Validator wallet {wallet} balance is low: {balance} TON.", + 18*HOUR + ), + "db_usage_80": Alert( + "high", + """TON DB usage > 80%. Clean the TON database: + https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming + or (and) set node\'s archive ttl to lower value.""", + 24*HOUR + ), + "db_usage_95": Alert( + "critical", + """TON DB usage > 95%. Disk is almost full, clean the TON database immediately: + https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming + or (and) set node\'s archive ttl to lower value.""", + 6*HOUR + ), + "low_efficiency": Alert( + "high", + """Validator efficiency is low: {efficiency}%.""", + VALIDATION_PERIOD // 3 + ), + "out_of_sync": Alert( + "critical", + "Node is out of sync on {sync} sec.", + 0 + ), + "service_down": Alert( + "critical", + "validator.service is down.", + 0 + ), + "adnl_connection_failed": Alert( + "high", + "ADNL connection to node failed", + 3*HOUR + ), + "zero_block_created": Alert( + "critical", + "Validator has not created any blocks in the last {hours} hours.", + VALIDATION_PERIOD // 3 + ), + "validator_slashed": Alert( + "high", + "Validator has been slashed in previous round for {amount} TON", + FREEZE_PERIOD + ), +} + + +class AlertBotModule(MtcModule): + + description = 'Telegram bot alerts' + default_value = False + + def __init__(self, ton, local, *args, **kwargs): + super().__init__(ton, local, *args, **kwargs) + self.validator_module = None + self.inited = False + self.hostname = None + self.token = self.ton.local.db.get("BotToken") + self.chat_id = self.ton.local.db.get("ChatId") + + def send_message(self, text: str): + if self.token is None: + raise Exception("send_message error: token is not initialized") + if self.chat_id is None: + raise Exception("send_message error: chat_id is not initialized") + request_url = f"https://api.telegram.org/bot{self.token}/sendMessage" + data = {'chat_id': self.chat_id, 'text': text, 'parse_mode': 'HTML'} + response = requests.post(request_url, data=data, timeout=3) + if response.status_code != 200: + raise Exception(f"send_message error: {response.text}") + response = response.json() + if not response['ok']: + raise Exception(f"send_message error: {response}") + + def send_alert(self, alert_name: str, *args, **kwargs): + if not self.alert_is_enabled(alert_name): + return + last_sent = self.get_alert_sent(alert_name) + time_ = timestamp2utcdatetime(int(time.time())) + alert = ALERTS.get(alert_name) + if alert is None: + raise Exception(f"Alert {alert_name} not found") + text = f''' +❗️ MyTonCtrl Alert {alert_name} ❗️ + +Hostname: {self.hostname} +Time: {time_} ({int(time.time())}) +Severity: {alert.severity} + +Alert text: +
{alert.text.format(*args, **kwargs)}
+''' + if time.time() - last_sent > alert.timeout: + self.send_message(text) + self.set_alert_sent(alert_name) + + def set_global_vars(self): + # set global vars for correct alerts timeouts for current network + config15 = self.ton.GetConfig15() + global VALIDATION_PERIOD, FREEZE_PERIOD + VALIDATION_PERIOD = config15["validatorsElectedFor"] + FREEZE_PERIOD = config15["stakeHeldFor"] + + def init(self): + if not self.ton.get_mode_value('alert-bot'): + return + if self.token is None or self.chat_id is None: + raise Exception("BotToken or ChatId is not set") + from modules.validator import ValidatorModule + self.validator_module = ValidatorModule(self.ton, self.local) + self.hostname = get_hostname() + self.set_global_vars() + self.inited = True + + def get_alert_from_db(self, alert_name: str): + if 'alerts' not in self.ton.local.db: + self.ton.local.db['alerts'] = {} + if alert_name not in self.ton.local.db['alerts']: + self.ton.local.db['alerts'][alert_name] = {'sent': 0, 'enabled': True} + return self.ton.local.db['alerts'][alert_name] + + def set_alert_sent(self, alert_name: str): + alert = self.get_alert_from_db(alert_name) + alert['sent'] = int(time.time()) + + def get_alert_sent(self, alert_name: str): + alert = self.get_alert_from_db(alert_name) + return alert.get('sent', 0) + + def alert_is_enabled(self, alert_name: str): + alert = self.get_alert_from_db(alert_name) + return alert.get('enabled', True) # default is True + + def set_alert_enabled(self, alert_name: str, enabled: bool): + alert = self.get_alert_from_db(alert_name) + alert['enabled'] = enabled + self.ton.local.save() + + def enable_alert(self, args): + if len(args) != 1: + raise Exception("Usage: enable_alert ") + alert_name = args[0] + self.set_alert_enabled(alert_name, True) + color_print("enable_alert - {green}OK{endc}") + + def disable_alert(self, args): + if len(args) != 1: + raise Exception("Usage: disable_alert ") + alert_name = args[0] + self.set_alert_enabled(alert_name, False) + color_print("disable_alert - {green}OK{endc}") + + def print_alerts(self, args): + table = [['Name', 'Enabled', 'Last sent']] + for alert_name in ALERTS: + alert = self.get_alert_from_db(alert_name) + table.append([alert_name, alert['enabled'], alert['sent']]) + print_table(table) + + def test_alert(self, args): + self.send_message('Test alert') + + def check_db_usage(self): + usage = self.ton.GetDbUsage() + if usage > 95: + self.send_alert("db_usage_95") + elif usage > 80: + self.send_alert("db_usage_80") + + def check_validator_wallet_balance(self): + if not self.ton.using_validator(): + return + validator_wallet = self.ton.GetValidatorWallet() + validator_account = self.ton.GetAccount(validator_wallet.addrB64) + if validator_account.balance < 10: + self.send_alert("low_wallet_balance", wallet=validator_wallet.addrB64, balance=validator_account.balance) + + def check_efficiency(self): + if not self.ton.using_validator(): + return + validator = self.validator_module.find_myself(self.ton.GetValidatorsList(fast=True)) + if validator is None or validator.is_masterchain is False or validator.efficiency is None: + return + config34 = self.ton.GetConfig34() + if (time.time() - config34.startWorkTime) / (config34.endWorkTime - config34.startWorkTime) < 0.8: + return # less than 80% of round passed + if validator.efficiency < 90: + self.send_alert("low_efficiency", efficiency=validator.efficiency) + + def check_validator_working(self): + validator_status = self.ton.GetValidatorStatus() + if not validator_status.is_working: + self.send_alert("service_down") + + def check_sync(self): + validator_status = self.ton.GetValidatorStatus() + if validator_status.is_working and validator_status.out_of_sync >= 20: + self.send_alert("out_of_sync", sync=validator_status.out_of_sync) + + def check_zero_blocks_created(self): + if not self.ton.using_validator(): + return + ts = get_timestamp() + period = VALIDATION_PERIOD // 3 # 6h for mainnet, 40m for testnet + start, end = ts - period, ts - 60 + config34 = self.ton.GetConfig34() + if start < config34.startWorkTime: # round started recently + return + validators = self.ton.GetValidatorsList(start=start, end=end) + validator = self.validator_module.find_myself(validators) + if validator is None or validator.blocks_created > 0: + return + self.send_alert("zero_block_created", hours=round(period // 3600, 1)) + + def check_slashed(self): + if not self.ton.using_validator(): + return + c = self.validator_module.get_my_complaint() + if c is not None: + self.send_alert("validator_slashed", amount=int(c['suggestedFine'])) + + def check_adnl_connection_failed(self): + from modules.utilities import UtilitiesModule + utils_module = UtilitiesModule(self.ton, self.local) + ok, error = utils_module.check_adnl_connection() + if not ok: + self.send_alert("adnl_connection_failed") + + def check_status(self): + if not self.inited: + self.init() + + self.local.try_function(self.check_db_usage) + self.local.try_function(self.check_validator_wallet_balance) + self.local.try_function(self.check_efficiency) # todo: alert if validator is going to be slashed + self.local.try_function(self.check_validator_working) + self.local.try_function(self.check_zero_blocks_created) + self.local.try_function(self.check_sync) + self.local.try_function(self.check_slashed) + self.local.try_function(self.check_adnl_connection_failed) + + def add_console_commands(self, console): + console.AddItem("enable_alert", self.enable_alert, self.local.translate("enable_alert_cmd")) + console.AddItem("disable_alert", self.disable_alert, self.local.translate("disable_alert_cmd")) + console.AddItem("list_alerts", self.print_alerts, self.local.translate("list_alerts_cmd")) + console.AddItem("test_alert", self.test_alert, self.local.translate("test_alert_cmd")) diff --git a/modules/utilities.py b/modules/utilities.py index f659b65a..da108152 100644 --- a/modules/utilities.py +++ b/modules/utilities.py @@ -1,9 +1,10 @@ -import base64 import json -import os +import random import subprocess import time +import requests + from mypylib.mypylib import color_print, print_table, color_text, timeago, bcolors from modules.module import MtcModule @@ -335,6 +336,35 @@ def print_validator_list(self, args): print_table(table) # end define + def check_adnl_connection(self): + telemetry = self.ton.local.db.get("sendTelemetry", False) + check_adnl = self.ton.local.db.get("checkAdnl", telemetry) + if not check_adnl: + return True, '' + self.local.add_log('Checking ADNL connection to local node', 'info') + hosts = ['45.129.96.53', '5.154.181.153', '2.56.126.137', '91.194.11.68', '45.12.134.214', '138.124.184.27', + '103.106.3.171'] + hosts = random.sample(hosts, k=3) + data = self.ton.get_local_adnl_data() + error = '' + ok = True + for host in hosts: + url = f'http://{host}/adnl_check' + try: + response = requests.post(url, json=data, timeout=5).json() + except Exception as e: + ok = False + error = f'{{red}}Failed to check ADNL connection to local node: {type(e)}: {e}{{endc}}' + continue + result = response.get("ok") + if result: + ok = True + break + if not result: + ok = False + error = f'{{red}}Failed to check ADNL connection to local node: {response.get("message")}{{endc}}' + return ok, error + def get_pool_data(self, args): try: pool_name = args[0] diff --git a/modules/validator.py b/modules/validator.py index 3ce5cd6f..39317924 100644 --- a/modules/validator.py +++ b/modules/validator.py @@ -91,6 +91,17 @@ def check_efficiency(self, args): print("Couldn't find this validator in the current round") # end define + def get_my_complaint(self): + config32 = self.ton.GetConfig32() + save_complaints = self.ton.GetSaveComplaints() + complaints = save_complaints.get(str(config32['startWorkTime'])) + if not complaints: + return + for c in complaints.values(): + if c["adnl"] == self.ton.GetAdnlAddr() and c["isPassed"]: + return c + # end define + def add_console_commands(self, console): console.AddItem("vo", self.vote_offer, self.local.translate("vo_cmd")) console.AddItem("ve", self.vote_election_entry, self.local.translate("ve_cmd")) diff --git a/mytoncore/functions.py b/mytoncore/functions.py index 686fb657..148c07d1 100755 --- a/mytoncore/functions.py +++ b/mytoncore/functions.py @@ -569,6 +569,10 @@ def General(local): from modules.custom_overlays import CustomOverlayModule local.start_cycle(CustomOverlayModule(ton, local).custom_overlays, sec=60, args=()) + if ton.get_mode_value('alert-bot'): + from modules.alert_bot import AlertBotModule + local.start_cycle(AlertBotModule(ton, local).check_status, sec=1000, args=()) + thr_sleep() # end define diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 127db692..76cf58cd 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -2440,21 +2440,23 @@ def GetValidatorsLoad(self, start, end, saveCompFiles=False) -> dict: return data #end define - def GetValidatorsList(self, past=False, fast=False): + def GetValidatorsList(self, past=False, fast=False, start=None, end=None): # Get buffer - bname = "validatorsList" + str(past) + bname = "validatorsList" + str(past) + str(start) + str(end) buff = self.GetFunctionBuffer(bname, timeout=60) if buff: return buff #end if - timestamp = get_timestamp() - end = timestamp - 60 config = self.GetConfig34() - if fast: - start = end - 1000 - else: - start = config.get("startWorkTime") + if end is None: + timestamp = get_timestamp() + end = timestamp - 60 + if start is None: + if fast: + start = end - 1000 + else: + start = config.get("startWorkTime") if past: config = self.GetConfig32() start = config.get("startWorkTime") @@ -3089,6 +3091,9 @@ def using_validator(self): def using_liteserver(self): return self.get_mode_value('liteserver') + def using_alert_bot(self): + return self.get_mode_value('alert-bot') + def Tlb2Json(self, text): # Заменить скобки start = 0 diff --git a/mytoncore/utils.py b/mytoncore/utils.py index 0a8bdc91..a31e299c 100644 --- a/mytoncore/utils.py +++ b/mytoncore/utils.py @@ -1,6 +1,7 @@ import base64 import json import re +import subprocess def str2b64(s): @@ -97,3 +98,6 @@ def parse_db_stats(path: str): result[s[0]] = {k: float(v) for k, v in items} return result # end define + +def get_hostname(): + return subprocess.run(["hostname", "-f"], stdout=subprocess.PIPE).stdout.decode().strip() diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 1020c5c9..44a868e5 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -135,6 +135,11 @@ def inject_globals(func): module = ControllerModule(ton, local) module.add_console_commands(console) + if ton.using_alert_bot(): + from modules.alert_bot import AlertBotModule + module = AlertBotModule(ton, local) + module.add_console_commands(console) + console.AddItem("cleanup", inject_globals(cleanup_validator_db), local.translate("cleanup_cmd")) console.AddItem("benchmark", inject_globals(run_benchmark), local.translate("benchmark_cmd")) # console.AddItem("activate_ton_storage_provider", inject_globals(activate_ton_storage_provider), local.translate("activate_ton_storage_provider_cmd")) @@ -480,42 +485,21 @@ def check_tg_channel(local, ton): #end difine def check_slashed(local, ton): - config32 = ton.GetConfig32() - save_complaints = ton.GetSaveComplaints() - complaints = save_complaints.get(str(config32['startWorkTime'])) - if not complaints: + validator_status = ton.GetValidatorStatus() + if not ton.using_validator() or not validator_status.is_working or validator_status.out_of_sync >= 20: return - for c in complaints.values(): - if c["adnl"] == ton.GetAdnlAddr() and c["isPassed"]: - print_warning(local, "slashed_warning") + from modules import ValidatorModule + validator_module = ValidatorModule(ton, local) + c = validator_module.get_my_complaint() + if c: + warning = local.translate("slashed_warning").format(int(c['suggestedFine'])) + print_warning(local, warning) #end define def check_adnl(local, ton): - telemetry = ton.local.db.get("sendTelemetry", False) - check_adnl = ton.local.db.get("checkAdnl", telemetry) - local.add_log('Checking ADNL connection to local node', 'info') - if not check_adnl: - return - hosts = ['45.129.96.53', '5.154.181.153', '2.56.126.137', '91.194.11.68', '45.12.134.214', '138.124.184.27', '103.106.3.171'] - hosts = random.sample(hosts, k=3) - data = ton.get_local_adnl_data() - error = '' - ok = True - for host in hosts: - url = f'http://{host}/adnl_check' - try: - response = requests.post(url, json=data, timeout=5).json() - except Exception as e: - ok = False - error = f'{{red}}Failed to check ADNL connection to local node: {type(e)}: {e}{{endc}}' - continue - result = response.get("ok") - if result: - ok = True - break - if not result: - ok = False - error = f'{{red}}Failed to check ADNL connection to local node: {response.get("message")}{{endc}}' + from modules.utilities import UtilitiesModule + utils_module = UtilitiesModule(ton, local) + ok, error = utils_module.check_adnl_connection() if not ok: print_warning(local, error) #end define diff --git a/mytonctrl/resources/translate.json b/mytonctrl/resources/translate.json index 632158b4..929eea8a 100644 --- a/mytonctrl/resources/translate.json +++ b/mytonctrl/resources/translate.json @@ -445,9 +445,9 @@ "zh_TW": "{red}錯誤 - 驗證器的 UDP 端口無法從外部訪問.{endc}" }, "slashed_warning": { - "en": "{red}You were fined by 101 TON for low efficiency in the previous round.{endc}", - "ru": "{red}Вы были оштрафованы на 101 TON за низкую эффективность в предыдущем раунде.{endc}", - "zh_TW": "{red}您因上一輪效率低而被罰款 101 TON。{endc}" + "en": "{red}You were fined by {0} TON for low efficiency in the previous round.{endc}", + "ru": "{red}Вы были оштрафованы на {0} TON за низкую эффективность в предыдущем раунде.{endc}", + "zh_TW": "{red}您因上一輪效率低而被罰款 {0} TON。{endc}" }, "add_custom_overlay_cmd": { "en": "Add custom overlay", @@ -464,6 +464,26 @@ "ru": "Удалить пользовательский оверлей", "zh_TW": "刪除自定義覆蓋" }, + "enable_alert_cmd": { + "en": "Enable specific Telegram Bot alert", + "ru": "Включить определенное оповещение через Telegram Bot", + "zh_TW": "啟用特定的 Telegram Bot 警報" + }, + "disable_alert_cmd": { + "en": "Disable specific Telegram Bot alert", + "ru": "Отключить определенное оповещение через Telegram Bot", + "zh_TW": "禁用特定的 Telegram Bot 警報" + }, + "list_alerts_cmd": { + "en": "List all available Telegram Bot alerts", + "ru": "Список всех доступных оповещений через Telegram Bot", + "zh_TW": "列出所有可用的 Telegram Bot 警報" + }, + "test_alert_cmd": { + "en": "Send test alert via Telegram Bot", + "ru": "Отправить тестовое оповещение через Telegram Bot", + "zh_TW": "通過 Telegram Bot 發送測試警報" + }, "cleanup_cmd": { "en": "Clean node old logs and temp files", "ru": "Очистить старые логи и временные файлы ноды",