diff --git a/modules/__init__.py b/modules/__init__.py
index d639c4fb..afb61e14 100644
--- a/modules/__init__.py
+++ b/modules/__init__.py
@@ -8,6 +8,7 @@
from modules.validator import ValidatorModule
from modules.controller import ControllerModule
from modules.liteserver import LiteserverModule
+from modules.alert_bot import AlertBotModule
MODES = {
@@ -15,7 +16,8 @@
'nominator-pool': NominatorPoolModule,
'single-nominator': SingleNominatorModule,
'liquid-staking': ControllerModule,
- 'liteserver': LiteserverModule
+ 'liteserver': LiteserverModule,
+ 'alert-bot': AlertBotModule
}
@@ -55,6 +57,8 @@ class Setting:
'defaultCustomOverlaysUrl': Setting(None, 'https://ton-blockchain.github.io/fallback_custom_overlays.json', 'Default custom overlays config url'),
'debug': Setting(None, False, 'Debug mtc console mode. Prints Traceback on errors'),
'subscribe_tg_channel': Setting('validator', False, 'Disables warning about subscribing to the `TON STATUS` channel'),
+ 'BotToken': Setting('alert-bot', None, 'Alerting Telegram bot token'),
+ 'ChatId': Setting('alert-bot', None, 'Alerting Telegram chat id')
}
diff --git a/modules/alert_bot.py b/modules/alert_bot.py
new file mode 100644
index 00000000..5ba4d99a
--- /dev/null
+++ b/modules/alert_bot.py
@@ -0,0 +1,274 @@
+import dataclasses
+import time
+import requests
+
+from modules.module import MtcModule
+from mypylib.mypylib import get_timestamp, print_table, color_print
+from mytoncore import get_hostname
+from mytonctrl.utils import timestamp2utcdatetime
+
+
+@dataclasses.dataclass
+class Alert:
+ severity: str
+ text: str
+ timeout: int
+
+
+HOUR = 3600
+VALIDATION_PERIOD = 65536
+FREEZE_PERIOD = 32768
+
+
+ALERTS = {
+ "low_wallet_balance": Alert(
+ "low",
+ "Validator wallet {wallet} balance is low: {balance} TON.",
+ 18*HOUR
+ ),
+ "db_usage_80": Alert(
+ "high",
+ """TON DB usage > 80%. Clean the TON database:
+ https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming
+ or (and) set node\'s archive ttl to lower value.""",
+ 24*HOUR
+ ),
+ "db_usage_95": Alert(
+ "critical",
+ """TON DB usage > 95%. Disk is almost full, clean the TON database immediately:
+ https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming
+ or (and) set node\'s archive ttl to lower value.""",
+ 6*HOUR
+ ),
+ "low_efficiency": Alert(
+ "high",
+ """Validator efficiency is low: {efficiency}%.""",
+ VALIDATION_PERIOD // 3
+ ),
+ "out_of_sync": Alert(
+ "critical",
+ "Node is out of sync on {sync} sec.",
+ 0
+ ),
+ "service_down": Alert(
+ "critical",
+ "validator.service is down.",
+ 0
+ ),
+ "adnl_connection_failed": Alert(
+ "high",
+ "ADNL connection to node failed",
+ 3*HOUR
+ ),
+ "zero_block_created": Alert(
+ "critical",
+ "Validator has not created any blocks in the last {hours} hours.",
+ VALIDATION_PERIOD // 3
+ ),
+ "validator_slashed": Alert(
+ "high",
+ "Validator has been slashed in previous round for {amount} TON",
+ FREEZE_PERIOD
+ ),
+}
+
+
+class AlertBotModule(MtcModule):
+
+ description = 'Telegram bot alerts'
+ default_value = False
+
+ def __init__(self, ton, local, *args, **kwargs):
+ super().__init__(ton, local, *args, **kwargs)
+ self.validator_module = None
+ self.inited = False
+ self.hostname = None
+ self.token = self.ton.local.db.get("BotToken")
+ self.chat_id = self.ton.local.db.get("ChatId")
+
+ def send_message(self, text: str):
+ if self.token is None:
+ raise Exception("send_message error: token is not initialized")
+ if self.chat_id is None:
+ raise Exception("send_message error: chat_id is not initialized")
+ request_url = f"https://api.telegram.org/bot{self.token}/sendMessage"
+ data = {'chat_id': self.chat_id, 'text': text, 'parse_mode': 'HTML'}
+ response = requests.post(request_url, data=data, timeout=3)
+ if response.status_code != 200:
+ raise Exception(f"send_message error: {response.text}")
+ response = response.json()
+ if not response['ok']:
+ raise Exception(f"send_message error: {response}")
+
+ def send_alert(self, alert_name: str, *args, **kwargs):
+ if not self.alert_is_enabled(alert_name):
+ return
+ last_sent = self.get_alert_sent(alert_name)
+ time_ = timestamp2utcdatetime(int(time.time()))
+ alert = ALERTS.get(alert_name)
+ if alert is None:
+ raise Exception(f"Alert {alert_name} not found")
+ text = f'''
+❗️ MyTonCtrl Alert {alert_name} ❗️
+
+Hostname: {self.hostname}
+Time: {time_} ({int(time.time())})
+Severity: {alert.severity}
+
+Alert text:
+
{alert.text.format(*args, **kwargs)}
+'''
+ if time.time() - last_sent > alert.timeout:
+ self.send_message(text)
+ self.set_alert_sent(alert_name)
+
+ def set_global_vars(self):
+ # set global vars for correct alerts timeouts for current network
+ config15 = self.ton.GetConfig15()
+ global VALIDATION_PERIOD, FREEZE_PERIOD
+ VALIDATION_PERIOD = config15["validatorsElectedFor"]
+ FREEZE_PERIOD = config15["stakeHeldFor"]
+
+ def init(self):
+ if not self.ton.get_mode_value('alert-bot'):
+ return
+ if self.token is None or self.chat_id is None:
+ raise Exception("BotToken or ChatId is not set")
+ from modules.validator import ValidatorModule
+ self.validator_module = ValidatorModule(self.ton, self.local)
+ self.hostname = get_hostname()
+ self.set_global_vars()
+ self.inited = True
+
+ def get_alert_from_db(self, alert_name: str):
+ if 'alerts' not in self.ton.local.db:
+ self.ton.local.db['alerts'] = {}
+ if alert_name not in self.ton.local.db['alerts']:
+ self.ton.local.db['alerts'][alert_name] = {'sent': 0, 'enabled': True}
+ return self.ton.local.db['alerts'][alert_name]
+
+ def set_alert_sent(self, alert_name: str):
+ alert = self.get_alert_from_db(alert_name)
+ alert['sent'] = int(time.time())
+
+ def get_alert_sent(self, alert_name: str):
+ alert = self.get_alert_from_db(alert_name)
+ return alert.get('sent', 0)
+
+ def alert_is_enabled(self, alert_name: str):
+ alert = self.get_alert_from_db(alert_name)
+ return alert.get('enabled', True) # default is True
+
+ def set_alert_enabled(self, alert_name: str, enabled: bool):
+ alert = self.get_alert_from_db(alert_name)
+ alert['enabled'] = enabled
+ self.ton.local.save()
+
+ def enable_alert(self, args):
+ if len(args) != 1:
+ raise Exception("Usage: enable_alert ")
+ alert_name = args[0]
+ self.set_alert_enabled(alert_name, True)
+ color_print("enable_alert - {green}OK{endc}")
+
+ def disable_alert(self, args):
+ if len(args) != 1:
+ raise Exception("Usage: disable_alert ")
+ alert_name = args[0]
+ self.set_alert_enabled(alert_name, False)
+ color_print("disable_alert - {green}OK{endc}")
+
+ def print_alerts(self, args):
+ table = [['Name', 'Enabled', 'Last sent']]
+ for alert_name in ALERTS:
+ alert = self.get_alert_from_db(alert_name)
+ table.append([alert_name, alert['enabled'], alert['sent']])
+ print_table(table)
+
+ def test_alert(self, args):
+ self.send_message('Test alert')
+
+ def check_db_usage(self):
+ usage = self.ton.GetDbUsage()
+ if usage > 95:
+ self.send_alert("db_usage_95")
+ elif usage > 80:
+ self.send_alert("db_usage_80")
+
+ def check_validator_wallet_balance(self):
+ if not self.ton.using_validator():
+ return
+ validator_wallet = self.ton.GetValidatorWallet()
+ validator_account = self.ton.GetAccount(validator_wallet.addrB64)
+ if validator_account.balance < 10:
+ self.send_alert("low_wallet_balance", wallet=validator_wallet.addrB64, balance=validator_account.balance)
+
+ def check_efficiency(self):
+ if not self.ton.using_validator():
+ return
+ validator = self.validator_module.find_myself(self.ton.GetValidatorsList(fast=True))
+ if validator is None or validator.is_masterchain is False or validator.efficiency is None:
+ return
+ config34 = self.ton.GetConfig34()
+ if (time.time() - config34.startWorkTime) / (config34.endWorkTime - config34.startWorkTime) < 0.8:
+ return # less than 80% of round passed
+ if validator.efficiency < 90:
+ self.send_alert("low_efficiency", efficiency=validator.efficiency)
+
+ def check_validator_working(self):
+ validator_status = self.ton.GetValidatorStatus()
+ if not validator_status.is_working:
+ self.send_alert("service_down")
+
+ def check_sync(self):
+ validator_status = self.ton.GetValidatorStatus()
+ if validator_status.is_working and validator_status.out_of_sync >= 20:
+ self.send_alert("out_of_sync", sync=validator_status.out_of_sync)
+
+ def check_zero_blocks_created(self):
+ if not self.ton.using_validator():
+ return
+ ts = get_timestamp()
+ period = VALIDATION_PERIOD // 3 # 6h for mainnet, 40m for testnet
+ start, end = ts - period, ts - 60
+ config34 = self.ton.GetConfig34()
+ if start < config34.startWorkTime: # round started recently
+ return
+ validators = self.ton.GetValidatorsList(start=start, end=end)
+ validator = self.validator_module.find_myself(validators)
+ if validator is None or validator.blocks_created > 0:
+ return
+ self.send_alert("zero_block_created", hours=round(period // 3600, 1))
+
+ def check_slashed(self):
+ if not self.ton.using_validator():
+ return
+ c = self.validator_module.get_my_complaint()
+ if c is not None:
+ self.send_alert("validator_slashed", amount=int(c['suggestedFine']))
+
+ def check_adnl_connection_failed(self):
+ from modules.utilities import UtilitiesModule
+ utils_module = UtilitiesModule(self.ton, self.local)
+ ok, error = utils_module.check_adnl_connection()
+ if not ok:
+ self.send_alert("adnl_connection_failed")
+
+ def check_status(self):
+ if not self.inited:
+ self.init()
+
+ self.local.try_function(self.check_db_usage)
+ self.local.try_function(self.check_validator_wallet_balance)
+ self.local.try_function(self.check_efficiency) # todo: alert if validator is going to be slashed
+ self.local.try_function(self.check_validator_working)
+ self.local.try_function(self.check_zero_blocks_created)
+ self.local.try_function(self.check_sync)
+ self.local.try_function(self.check_slashed)
+ self.local.try_function(self.check_adnl_connection_failed)
+
+ def add_console_commands(self, console):
+ console.AddItem("enable_alert", self.enable_alert, self.local.translate("enable_alert_cmd"))
+ console.AddItem("disable_alert", self.disable_alert, self.local.translate("disable_alert_cmd"))
+ console.AddItem("list_alerts", self.print_alerts, self.local.translate("list_alerts_cmd"))
+ console.AddItem("test_alert", self.test_alert, self.local.translate("test_alert_cmd"))
diff --git a/modules/utilities.py b/modules/utilities.py
index f659b65a..da108152 100644
--- a/modules/utilities.py
+++ b/modules/utilities.py
@@ -1,9 +1,10 @@
-import base64
import json
-import os
+import random
import subprocess
import time
+import requests
+
from mypylib.mypylib import color_print, print_table, color_text, timeago, bcolors
from modules.module import MtcModule
@@ -335,6 +336,35 @@ def print_validator_list(self, args):
print_table(table)
# end define
+ def check_adnl_connection(self):
+ telemetry = self.ton.local.db.get("sendTelemetry", False)
+ check_adnl = self.ton.local.db.get("checkAdnl", telemetry)
+ if not check_adnl:
+ return True, ''
+ self.local.add_log('Checking ADNL connection to local node', 'info')
+ hosts = ['45.129.96.53', '5.154.181.153', '2.56.126.137', '91.194.11.68', '45.12.134.214', '138.124.184.27',
+ '103.106.3.171']
+ hosts = random.sample(hosts, k=3)
+ data = self.ton.get_local_adnl_data()
+ error = ''
+ ok = True
+ for host in hosts:
+ url = f'http://{host}/adnl_check'
+ try:
+ response = requests.post(url, json=data, timeout=5).json()
+ except Exception as e:
+ ok = False
+ error = f'{{red}}Failed to check ADNL connection to local node: {type(e)}: {e}{{endc}}'
+ continue
+ result = response.get("ok")
+ if result:
+ ok = True
+ break
+ if not result:
+ ok = False
+ error = f'{{red}}Failed to check ADNL connection to local node: {response.get("message")}{{endc}}'
+ return ok, error
+
def get_pool_data(self, args):
try:
pool_name = args[0]
diff --git a/modules/validator.py b/modules/validator.py
index 3ce5cd6f..39317924 100644
--- a/modules/validator.py
+++ b/modules/validator.py
@@ -91,6 +91,17 @@ def check_efficiency(self, args):
print("Couldn't find this validator in the current round")
# end define
+ def get_my_complaint(self):
+ config32 = self.ton.GetConfig32()
+ save_complaints = self.ton.GetSaveComplaints()
+ complaints = save_complaints.get(str(config32['startWorkTime']))
+ if not complaints:
+ return
+ for c in complaints.values():
+ if c["adnl"] == self.ton.GetAdnlAddr() and c["isPassed"]:
+ return c
+ # end define
+
def add_console_commands(self, console):
console.AddItem("vo", self.vote_offer, self.local.translate("vo_cmd"))
console.AddItem("ve", self.vote_election_entry, self.local.translate("ve_cmd"))
diff --git a/mytoncore/functions.py b/mytoncore/functions.py
index 686fb657..148c07d1 100755
--- a/mytoncore/functions.py
+++ b/mytoncore/functions.py
@@ -569,6 +569,10 @@ def General(local):
from modules.custom_overlays import CustomOverlayModule
local.start_cycle(CustomOverlayModule(ton, local).custom_overlays, sec=60, args=())
+ if ton.get_mode_value('alert-bot'):
+ from modules.alert_bot import AlertBotModule
+ local.start_cycle(AlertBotModule(ton, local).check_status, sec=1000, args=())
+
thr_sleep()
# end define
diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py
index 127db692..76cf58cd 100644
--- a/mytoncore/mytoncore.py
+++ b/mytoncore/mytoncore.py
@@ -2440,21 +2440,23 @@ def GetValidatorsLoad(self, start, end, saveCompFiles=False) -> dict:
return data
#end define
- def GetValidatorsList(self, past=False, fast=False):
+ def GetValidatorsList(self, past=False, fast=False, start=None, end=None):
# Get buffer
- bname = "validatorsList" + str(past)
+ bname = "validatorsList" + str(past) + str(start) + str(end)
buff = self.GetFunctionBuffer(bname, timeout=60)
if buff:
return buff
#end if
- timestamp = get_timestamp()
- end = timestamp - 60
config = self.GetConfig34()
- if fast:
- start = end - 1000
- else:
- start = config.get("startWorkTime")
+ if end is None:
+ timestamp = get_timestamp()
+ end = timestamp - 60
+ if start is None:
+ if fast:
+ start = end - 1000
+ else:
+ start = config.get("startWorkTime")
if past:
config = self.GetConfig32()
start = config.get("startWorkTime")
@@ -3089,6 +3091,9 @@ def using_validator(self):
def using_liteserver(self):
return self.get_mode_value('liteserver')
+ def using_alert_bot(self):
+ return self.get_mode_value('alert-bot')
+
def Tlb2Json(self, text):
# Заменить скобки
start = 0
diff --git a/mytoncore/utils.py b/mytoncore/utils.py
index 0a8bdc91..a31e299c 100644
--- a/mytoncore/utils.py
+++ b/mytoncore/utils.py
@@ -1,6 +1,7 @@
import base64
import json
import re
+import subprocess
def str2b64(s):
@@ -97,3 +98,6 @@ def parse_db_stats(path: str):
result[s[0]] = {k: float(v) for k, v in items}
return result
# end define
+
+def get_hostname():
+ return subprocess.run(["hostname", "-f"], stdout=subprocess.PIPE).stdout.decode().strip()
diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py
index 1020c5c9..44a868e5 100755
--- a/mytonctrl/mytonctrl.py
+++ b/mytonctrl/mytonctrl.py
@@ -135,6 +135,11 @@ def inject_globals(func):
module = ControllerModule(ton, local)
module.add_console_commands(console)
+ if ton.using_alert_bot():
+ from modules.alert_bot import AlertBotModule
+ module = AlertBotModule(ton, local)
+ module.add_console_commands(console)
+
console.AddItem("cleanup", inject_globals(cleanup_validator_db), local.translate("cleanup_cmd"))
console.AddItem("benchmark", inject_globals(run_benchmark), local.translate("benchmark_cmd"))
# console.AddItem("activate_ton_storage_provider", inject_globals(activate_ton_storage_provider), local.translate("activate_ton_storage_provider_cmd"))
@@ -480,42 +485,21 @@ def check_tg_channel(local, ton):
#end difine
def check_slashed(local, ton):
- config32 = ton.GetConfig32()
- save_complaints = ton.GetSaveComplaints()
- complaints = save_complaints.get(str(config32['startWorkTime']))
- if not complaints:
+ validator_status = ton.GetValidatorStatus()
+ if not ton.using_validator() or not validator_status.is_working or validator_status.out_of_sync >= 20:
return
- for c in complaints.values():
- if c["adnl"] == ton.GetAdnlAddr() and c["isPassed"]:
- print_warning(local, "slashed_warning")
+ from modules import ValidatorModule
+ validator_module = ValidatorModule(ton, local)
+ c = validator_module.get_my_complaint()
+ if c:
+ warning = local.translate("slashed_warning").format(int(c['suggestedFine']))
+ print_warning(local, warning)
#end define
def check_adnl(local, ton):
- telemetry = ton.local.db.get("sendTelemetry", False)
- check_adnl = ton.local.db.get("checkAdnl", telemetry)
- local.add_log('Checking ADNL connection to local node', 'info')
- if not check_adnl:
- return
- hosts = ['45.129.96.53', '5.154.181.153', '2.56.126.137', '91.194.11.68', '45.12.134.214', '138.124.184.27', '103.106.3.171']
- hosts = random.sample(hosts, k=3)
- data = ton.get_local_adnl_data()
- error = ''
- ok = True
- for host in hosts:
- url = f'http://{host}/adnl_check'
- try:
- response = requests.post(url, json=data, timeout=5).json()
- except Exception as e:
- ok = False
- error = f'{{red}}Failed to check ADNL connection to local node: {type(e)}: {e}{{endc}}'
- continue
- result = response.get("ok")
- if result:
- ok = True
- break
- if not result:
- ok = False
- error = f'{{red}}Failed to check ADNL connection to local node: {response.get("message")}{{endc}}'
+ from modules.utilities import UtilitiesModule
+ utils_module = UtilitiesModule(ton, local)
+ ok, error = utils_module.check_adnl_connection()
if not ok:
print_warning(local, error)
#end define
diff --git a/mytonctrl/resources/translate.json b/mytonctrl/resources/translate.json
index 632158b4..929eea8a 100644
--- a/mytonctrl/resources/translate.json
+++ b/mytonctrl/resources/translate.json
@@ -445,9 +445,9 @@
"zh_TW": "{red}錯誤 - 驗證器的 UDP 端口無法從外部訪問.{endc}"
},
"slashed_warning": {
- "en": "{red}You were fined by 101 TON for low efficiency in the previous round.{endc}",
- "ru": "{red}Вы были оштрафованы на 101 TON за низкую эффективность в предыдущем раунде.{endc}",
- "zh_TW": "{red}您因上一輪效率低而被罰款 101 TON。{endc}"
+ "en": "{red}You were fined by {0} TON for low efficiency in the previous round.{endc}",
+ "ru": "{red}Вы были оштрафованы на {0} TON за низкую эффективность в предыдущем раунде.{endc}",
+ "zh_TW": "{red}您因上一輪效率低而被罰款 {0} TON。{endc}"
},
"add_custom_overlay_cmd": {
"en": "Add custom overlay",
@@ -464,6 +464,26 @@
"ru": "Удалить пользовательский оверлей",
"zh_TW": "刪除自定義覆蓋"
},
+ "enable_alert_cmd": {
+ "en": "Enable specific Telegram Bot alert",
+ "ru": "Включить определенное оповещение через Telegram Bot",
+ "zh_TW": "啟用特定的 Telegram Bot 警報"
+ },
+ "disable_alert_cmd": {
+ "en": "Disable specific Telegram Bot alert",
+ "ru": "Отключить определенное оповещение через Telegram Bot",
+ "zh_TW": "禁用特定的 Telegram Bot 警報"
+ },
+ "list_alerts_cmd": {
+ "en": "List all available Telegram Bot alerts",
+ "ru": "Список всех доступных оповещений через Telegram Bot",
+ "zh_TW": "列出所有可用的 Telegram Bot 警報"
+ },
+ "test_alert_cmd": {
+ "en": "Send test alert via Telegram Bot",
+ "ru": "Отправить тестовое оповещение через Telegram Bot",
+ "zh_TW": "通過 Telegram Bot 發送測試警報"
+ },
"cleanup_cmd": {
"en": "Clean node old logs and temp files",
"ru": "Очистить старые логи и временные файлы ноды",