From da653d8a7cd56108e61d6ebf87adca3c42949d61 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 3 Oct 2024 21:37:44 +0400 Subject: [PATCH 01/20] add AlertBot --- modules/__init__.py | 6 +- modules/alert_bot.py | 151 +++++++++++++++++++++++++++++++++++++++++ mytoncore/functions.py | 4 ++ mytoncore/mytoncore.py | 3 + mytoncore/utils.py | 4 ++ 5 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 modules/alert_bot.py diff --git a/modules/__init__.py b/modules/__init__.py index d639c4fb..afb61e14 100644 --- a/modules/__init__.py +++ b/modules/__init__.py @@ -8,6 +8,7 @@ from modules.validator import ValidatorModule from modules.controller import ControllerModule from modules.liteserver import LiteserverModule +from modules.alert_bot import AlertBotModule MODES = { @@ -15,7 +16,8 @@ 'nominator-pool': NominatorPoolModule, 'single-nominator': SingleNominatorModule, 'liquid-staking': ControllerModule, - 'liteserver': LiteserverModule + 'liteserver': LiteserverModule, + 'alert-bot': AlertBotModule } @@ -55,6 +57,8 @@ class Setting: 'defaultCustomOverlaysUrl': Setting(None, 'https://ton-blockchain.github.io/fallback_custom_overlays.json', 'Default custom overlays config url'), 'debug': Setting(None, False, 'Debug mtc console mode. Prints Traceback on errors'), 'subscribe_tg_channel': Setting('validator', False, 'Disables warning about subscribing to the `TON STATUS` channel'), + 'BotToken': Setting('alert-bot', None, 'Alerting Telegram bot token'), + 'ChatId': Setting('alert-bot', None, 'Alerting Telegram chat id') } diff --git a/modules/alert_bot.py b/modules/alert_bot.py new file mode 100644 index 00000000..6118ffb1 --- /dev/null +++ b/modules/alert_bot.py @@ -0,0 +1,151 @@ +import dataclasses +import time + +from modules.module import MtcModule +from mytoncore import get_hostname +from mytonctrl.utils import timestamp2utcdatetime + + +class Alert(dataclasses.dataclass): + severity: str + text: str + timeout: int + + +HOUR = 3600 + + +ALERTS = { + "low_wallet_balance": Alert( + "medium", + "Validator wallet {wallet} balance is low: {balance} TON.", + 18*HOUR + ), + "db_usage_80": Alert( + "high", + """TON DB usage > 80%. Clean the TON database: + https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming + or (and) set node\'s archive ttl to lower value.""", + 24*HOUR + ), + "db_usage_95": Alert( + "critical", + """TON DB usage > 95%. Disk is almost full, clean the TON database immediately: + https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming + or (and) set node\'s archive ttl to lower value.""", + 6*HOUR + ), + "low_efficiency": Alert( + "high", + """Validator efficiency is low: {efficiency}%.""", + 12*HOUR + ), + "zero_block_created": Alert( + "critical", + "No blocks created for the last 6 hours.", + 6*HOUR + ), + "out_of_sync": Alert( + "critical", + "Node is out of sync on {sync} sec.", + 0 + ) +} + + +class AlertBotModule(MtcModule): + + description = 'Telegram bot alerts' + default_value = False + + def __init__(self, ton, local, *args, **kwargs): + super().__init__(ton, local, *args, **kwargs) + self.inited = False + self.hostname = None + self.bot = None + self.token = self.ton.local.db.get("BotToken") + self.chat_id = self.ton.local.db.get("ChatId") + + def send_message(self, text: str): + if self.bot is not None: + self.bot.send_message(self.chat_id, text) + else: + raise Exception("send_message error: bot is not initialized") + + def send_alert(self, alert_name: str, *args, **kwargs): + last_sent = self.get_alert_sent(alert_name) + time_ = timestamp2utcdatetime(int(time.time())) + alert = ALERTS.get(alert_name) + if alert is None: + raise Exception(f"Alert {alert_name} not found") + text = f''' +MyTonCtrl Alert {alert_name} + +Hostname: {self.hostname} +Time: {time_} ({int(time.time())}) +Severity: {alert.severity} +Next alert of this type in: {alert.timeout} sec + +Alert text: +
{alert.text.format(*args, **kwargs)}
+''' + if time.time() - last_sent > alert.timeout: + self.send_message(text) + self.set_alert_sent(alert_name) + + def init(self): + if not self.ton.get_mode_value('alert-bot'): + return + if self.token is None or self.chat_id is None: + raise Exception("BotToken or ChatId is not set") + import telebot + self.bot = telebot.TeleBot(self.token, parse_mode="HTML") + self.hostname = get_hostname() + self.inited = True + + def set_alert_sent(self, alert_name: str): + self.ton.local.db['alerts'][alert_name] = int(time.time()) + + def get_alert_sent(self, alert_name: str): + return self.ton.local.db['alerts'].get(alert_name, 0) + + def check_db_usage(self): + usage = self.ton.GetDbUsage() + if usage > 95: + self.send_alert("db_usage_95") + elif usage > 80: + self.send_alert("db_usage_80") + + def check_validator_wallet_balance(self): + validator_wallet = self.ton.GetValidatorWallet() + validator_account = self.ton.GetAccount(validator_wallet.addrB64) + if validator_account.balance < 50: + self.send_alert("low_wallet_balance", wallet=validator_wallet.addrB64, balance=validator_account.balance) + + def check_efficiency(self): + from modules.validator import ValidatorModule + validator = ValidatorModule(self.ton, self.local).find_myself(self.ton.GetValidatorsList()) + if validator is None or validator.is_masterchain is False or validator.efficiency is None: + return + config34 = self.ton.GetConfig34() + if (time.time() - config34.startWorkTime) / (config34.endWorkTime - config34.startWorkTime) < 0.8: + return # less than 80% of round passed + if validator.efficiency < 90: + self.send_alert("low_efficiency", efficiency=validator.efficiency) + + def check_sync(self): + validator_status = self.ton.GetValidatorStatus() + if not validator_status.is_working or validator_status.out_of_sync >= 20: + self.send_alert("out_of_sync", sync=validator_status.out_of_sync) + + def check_status(self): + if not self.inited: + self.init() + + self.local.try_function(self.check_db_usage) + self.local.try_function(self.check_validator_wallet_balance) + self.local.try_function(self.check_efficiency) # todo: alert if validator is going to be slashed + self.local.try_function(self.check_sync) + + def add_console_commands(self, console): + ... diff --git a/mytoncore/functions.py b/mytoncore/functions.py index 686fb657..148c07d1 100755 --- a/mytoncore/functions.py +++ b/mytoncore/functions.py @@ -569,6 +569,10 @@ def General(local): from modules.custom_overlays import CustomOverlayModule local.start_cycle(CustomOverlayModule(ton, local).custom_overlays, sec=60, args=()) + if ton.get_mode_value('alert-bot'): + from modules.alert_bot import AlertBotModule + local.start_cycle(AlertBotModule(ton, local).check_status, sec=1000, args=()) + thr_sleep() # end define diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 87ab360f..e90fe6bd 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -3054,6 +3054,9 @@ def check_enable_mode(self, name): if name == 'liquid-staking': from mytoninstaller.settings import enable_ton_http_api enable_ton_http_api(self.local) + if name == 'alert-bot': + args = ["pip", "install", "pytelegrambotapi==4.23.0"] + subprocess.run(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=10) def enable_mode(self, name): if name not in MODES: diff --git a/mytoncore/utils.py b/mytoncore/utils.py index 0a8bdc91..a31e299c 100644 --- a/mytoncore/utils.py +++ b/mytoncore/utils.py @@ -1,6 +1,7 @@ import base64 import json import re +import subprocess def str2b64(s): @@ -97,3 +98,6 @@ def parse_db_stats(path: str): result[s[0]] = {k: float(v) for k, v in items} return result # end define + +def get_hostname(): + return subprocess.run(["hostname", "-f"], stdout=subprocess.PIPE).stdout.decode().strip() From 0b7d6f3b9b5b2156459327f5a7f71b22cc855599 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 3 Oct 2024 21:42:57 +0400 Subject: [PATCH 02/20] fix dataclass --- modules/alert_bot.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 6118ffb1..198a79b9 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -6,7 +6,8 @@ from mytonctrl.utils import timestamp2utcdatetime -class Alert(dataclasses.dataclass): +@dataclasses.dataclass +class Alert: severity: str text: str timeout: int From 1a1d069afb640bdba04af588c8d7f1a3ac4545cc Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 3 Oct 2024 21:47:29 +0400 Subject: [PATCH 03/20] fix alerts in db --- modules/alert_bot.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 198a79b9..f7d43e06 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -105,9 +105,13 @@ def init(self): self.inited = True def set_alert_sent(self, alert_name: str): + if 'alerts' not in self.ton.local.db: + self.ton.local.db['alerts'] = {} self.ton.local.db['alerts'][alert_name] = int(time.time()) def get_alert_sent(self, alert_name: str): + if 'alerts' not in self.ton.local.db: + return 0 return self.ton.local.db['alerts'].get(alert_name, 0) def check_db_usage(self): From 3f0c559a27f522012cb06b77d1887435a004fcbc Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 3 Oct 2024 21:53:26 +0400 Subject: [PATCH 04/20] add service_down alert --- modules/alert_bot.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index f7d43e06..3c180a62 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -50,7 +50,12 @@ class Alert: "critical", "Node is out of sync on {sync} sec.", 0 - ) + ), + "service_down": Alert( + "critical", + "validator.service is down.", + 0 + ), } @@ -129,7 +134,7 @@ def check_validator_wallet_balance(self): def check_efficiency(self): from modules.validator import ValidatorModule - validator = ValidatorModule(self.ton, self.local).find_myself(self.ton.GetValidatorsList()) + validator = ValidatorModule(self.ton, self.local).find_myself(self.ton.GetValidatorsList(fast=True)) if validator is None or validator.is_masterchain is False or validator.efficiency is None: return config34 = self.ton.GetConfig34() @@ -138,9 +143,14 @@ def check_efficiency(self): if validator.efficiency < 90: self.send_alert("low_efficiency", efficiency=validator.efficiency) + def check_validator_working(self): + validator_status = self.ton.GetValidatorStatus() + if not validator_status.is_working: + self.send_alert("service_down") + def check_sync(self): validator_status = self.ton.GetValidatorStatus() - if not validator_status.is_working or validator_status.out_of_sync >= 20: + if validator_status.is_working and validator_status.out_of_sync >= 20: self.send_alert("out_of_sync", sync=validator_status.out_of_sync) def check_status(self): @@ -150,6 +160,8 @@ def check_status(self): self.local.try_function(self.check_db_usage) self.local.try_function(self.check_validator_wallet_balance) self.local.try_function(self.check_efficiency) # todo: alert if validator is going to be slashed + self.local.try_function(self.check_validator_working) + self.local.try_function(self.check_validator_working) self.local.try_function(self.check_sync) def add_console_commands(self, console): From 4d6354575711461151746827399039c5aeac47eb Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 3 Oct 2024 21:55:43 +0400 Subject: [PATCH 05/20] fix alerts timeout --- modules/alert_bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 3c180a62..c73f774f 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -90,7 +90,7 @@ def send_alert(self, alert_name: str, *args, **kwargs): Hostname: {self.hostname} Time: {time_} ({int(time.time())}) Severity: {alert.severity} -Next alert of this type in: {alert.timeout} sec +Next alert of this type not earlier than: {max(alert.timeout, 1000)} sec Alert text:
{alert.text.format(*args, **kwargs)}
From 029acf05c1b7a7f988428f3077c68c922bc14f85 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 3 Oct 2024 21:57:33 +0400 Subject: [PATCH 06/20] fix double service_down alert --- modules/alert_bot.py | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index c73f774f..b0b45384 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -161,7 +161,6 @@ def check_status(self): self.local.try_function(self.check_validator_wallet_balance) self.local.try_function(self.check_efficiency) # todo: alert if validator is going to be slashed self.local.try_function(self.check_validator_working) - self.local.try_function(self.check_validator_working) self.local.try_function(self.check_sync) def add_console_commands(self, console): From 1ee5e9b427dd13d8efe91ba3b7895e3230739ac6 Mon Sep 17 00:00:00 2001 From: yungwine Date: Mon, 7 Oct 2024 11:20:06 +0400 Subject: [PATCH 07/20] update alerts text --- modules/alert_bot.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index b0b45384..88062309 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -18,7 +18,7 @@ class Alert: ALERTS = { "low_wallet_balance": Alert( - "medium", + "low", "Validator wallet {wallet} balance is low: {balance} TON.", 18*HOUR ), @@ -43,7 +43,7 @@ class Alert: ), "zero_block_created": Alert( "critical", - "No blocks created for the last 6 hours.", + "Validator has not created any blocks in the last 6 hours.", 6*HOUR ), "out_of_sync": Alert( @@ -56,6 +56,11 @@ class Alert: "validator.service is down.", 0 ), + "adnl_connection_failed": Alert( + "high", + "ADNL connection to node failed", + 3*HOUR + ), } @@ -85,7 +90,7 @@ def send_alert(self, alert_name: str, *args, **kwargs): if alert is None: raise Exception(f"Alert {alert_name} not found") text = f''' -MyTonCtrl Alert {alert_name} +❗️ MyTonCtrl Alert {alert_name} ❗️ Hostname: {self.hostname} Time: {time_} ({int(time.time())}) From 02ca414e7f7fff9f3317612fcc348ef7ae346a80 Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 8 Oct 2024 12:48:02 +0400 Subject: [PATCH 08/20] add zero_blocks_created alert --- modules/alert_bot.py | 30 +++++++++++++++++++++++------- mytoncore/mytoncore.py | 16 +++++++++------- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 88062309..2e031357 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -41,11 +41,6 @@ class Alert: """Validator efficiency is low: {efficiency}%.""", 12*HOUR ), - "zero_block_created": Alert( - "critical", - "Validator has not created any blocks in the last 6 hours.", - 6*HOUR - ), "out_of_sync": Alert( "critical", "Node is out of sync on {sync} sec.", @@ -61,6 +56,11 @@ class Alert: "ADNL connection to node failed", 3*HOUR ), + "zero_block_created": Alert( + "critical", + "Validator has not created any blocks in the last 6 hours.", + 6 * HOUR + ), } @@ -71,6 +71,7 @@ class AlertBotModule(MtcModule): def __init__(self, ton, local, *args, **kwargs): super().__init__(ton, local, *args, **kwargs) + self.validator_module = None self.inited = False self.hostname = None self.bot = None @@ -109,6 +110,8 @@ def init(self): return if self.token is None or self.chat_id is None: raise Exception("BotToken or ChatId is not set") + from modules.validator import ValidatorModule + self.validator_module = ValidatorModule(self.ton, self.local) import telebot self.bot = telebot.TeleBot(self.token, parse_mode="HTML") self.hostname = get_hostname() @@ -132,14 +135,17 @@ def check_db_usage(self): self.send_alert("db_usage_80") def check_validator_wallet_balance(self): + if not self.ton.using_validator(): + return validator_wallet = self.ton.GetValidatorWallet() validator_account = self.ton.GetAccount(validator_wallet.addrB64) if validator_account.balance < 50: self.send_alert("low_wallet_balance", wallet=validator_wallet.addrB64, balance=validator_account.balance) def check_efficiency(self): - from modules.validator import ValidatorModule - validator = ValidatorModule(self.ton, self.local).find_myself(self.ton.GetValidatorsList(fast=True)) + if not self.ton.using_validator(): + return + validator = self.validator_module.find_myself(self.ton.GetValidatorsList(fast=True)) if validator is None or validator.is_masterchain is False or validator.efficiency is None: return config34 = self.ton.GetConfig34() @@ -158,6 +164,15 @@ def check_sync(self): if validator_status.is_working and validator_status.out_of_sync >= 20: self.send_alert("out_of_sync", sync=validator_status.out_of_sync) + def check_zero_blocks_created(self): + if not self.ton.using_validator(): + return + validators = self.ton.GetValidatorsList(start=-6*HOUR, end=-60) + validator = self.validator_module.find_myself(validators) + if validator is None or validator.blocks_created > 0: + return + self.send_alert("zero_block_created") + def check_status(self): if not self.inited: self.init() @@ -166,6 +181,7 @@ def check_status(self): self.local.try_function(self.check_validator_wallet_balance) self.local.try_function(self.check_efficiency) # todo: alert if validator is going to be slashed self.local.try_function(self.check_validator_working) + self.local.try_function(self.check_zero_blocks_created) self.local.try_function(self.check_sync) def add_console_commands(self, console): diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index e90fe6bd..8e691687 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -2444,7 +2444,7 @@ def GetValidatorsLoad(self, start, end, saveCompFiles=False) -> dict: return data #end define - def GetValidatorsList(self, past=False, fast=False): + def GetValidatorsList(self, past=False, fast=False, start=None, end=None): # Get buffer bname = "validatorsList" + str(past) buff = self.GetFunctionBuffer(bname, timeout=60) @@ -2452,13 +2452,15 @@ def GetValidatorsList(self, past=False, fast=False): return buff #end if - timestamp = get_timestamp() - end = timestamp - 60 config = self.GetConfig34() - if fast: - start = end - 1000 - else: - start = config.get("startWorkTime") + if start is None: + if fast: + start = end - 1000 + else: + start = config.get("startWorkTime") + if end is None: + timestamp = get_timestamp() + end = timestamp - 60 if past: config = self.GetConfig32() start = config.get("startWorkTime") From e0bb941200059c897143978aef71ca92bc487c9d Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 8 Oct 2024 19:56:21 +0400 Subject: [PATCH 09/20] add alert if validator was slashed --- modules/alert_bot.py | 13 +++++++++++++ modules/validator.py | 11 +++++++++++ mytonctrl/mytonctrl.py | 15 ++++++++------- mytonctrl/resources/translate.json | 6 +++--- 4 files changed, 35 insertions(+), 10 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 2e031357..a513d545 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -61,6 +61,11 @@ class Alert: "Validator has not created any blocks in the last 6 hours.", 6 * HOUR ), + "validator_slashed": Alert( + "high", + "Validator has been slashed in previous round for {amount} TON", + 0 + ), } @@ -173,6 +178,13 @@ def check_zero_blocks_created(self): return self.send_alert("zero_block_created") + def check_slashed(self): + if not self.ton.using_validator(): + return + c = self.validator_module.get_my_complaint() + if c is not None: + self.send_alert("validator_slashed", amount=int(c['suggestedFine'])) + def check_status(self): if not self.inited: self.init() @@ -183,6 +195,7 @@ def check_status(self): self.local.try_function(self.check_validator_working) self.local.try_function(self.check_zero_blocks_created) self.local.try_function(self.check_sync) + self.local.try_function(self.check_slashed) def add_console_commands(self, console): ... diff --git a/modules/validator.py b/modules/validator.py index 3ce5cd6f..39317924 100644 --- a/modules/validator.py +++ b/modules/validator.py @@ -91,6 +91,17 @@ def check_efficiency(self, args): print("Couldn't find this validator in the current round") # end define + def get_my_complaint(self): + config32 = self.ton.GetConfig32() + save_complaints = self.ton.GetSaveComplaints() + complaints = save_complaints.get(str(config32['startWorkTime'])) + if not complaints: + return + for c in complaints.values(): + if c["adnl"] == self.ton.GetAdnlAddr() and c["isPassed"]: + return c + # end define + def add_console_commands(self, console): console.AddItem("vo", self.vote_offer, self.local.translate("vo_cmd")) console.AddItem("ve", self.vote_election_entry, self.local.translate("ve_cmd")) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 1020c5c9..07753126 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -480,14 +480,15 @@ def check_tg_channel(local, ton): #end difine def check_slashed(local, ton): - config32 = ton.GetConfig32() - save_complaints = ton.GetSaveComplaints() - complaints = save_complaints.get(str(config32['startWorkTime'])) - if not complaints: + validator_status = ton.GetValidatorStatus() + if not ton.using_validator() or not validator_status.is_working or validator_status.out_of_sync >= 20: return - for c in complaints.values(): - if c["adnl"] == ton.GetAdnlAddr() and c["isPassed"]: - print_warning(local, "slashed_warning") + from modules import ValidatorModule + validator_module = ValidatorModule(ton, local) + c = validator_module.get_my_complaint() + if c: + warning = local.translate("slashed_warning").format(int(c['suggestedFine'])) + print_warning(local, warning) #end define def check_adnl(local, ton): diff --git a/mytonctrl/resources/translate.json b/mytonctrl/resources/translate.json index 632158b4..60e1501a 100644 --- a/mytonctrl/resources/translate.json +++ b/mytonctrl/resources/translate.json @@ -445,9 +445,9 @@ "zh_TW": "{red}錯誤 - 驗證器的 UDP 端口無法從外部訪問.{endc}" }, "slashed_warning": { - "en": "{red}You were fined by 101 TON for low efficiency in the previous round.{endc}", - "ru": "{red}Вы были оштрафованы на 101 TON за низкую эффективность в предыдущем раунде.{endc}", - "zh_TW": "{red}您因上一輪效率低而被罰款 101 TON。{endc}" + "en": "{red}You were fined by {0} for low efficiency in the previous round.{endc}", + "ru": "{red}Вы были оштрафованы на {0} за низкую эффективность в предыдущем раунде.{endc}", + "zh_TW": "{red}您因上一輪效率低而被罰款 {0}。{endc}" }, "add_custom_overlay_cmd": { "en": "Add custom overlay", From d3c456ca33da2802e82ff36ac941a94da114e52d Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 9 Oct 2024 13:35:48 +0400 Subject: [PATCH 10/20] rm telebot dependency for alert-bot mode --- modules/alert_bot.py | 20 ++++++++++++-------- mytoncore/mytoncore.py | 3 --- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index a513d545..f3faca8f 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -1,5 +1,6 @@ import dataclasses import time +import requests from modules.module import MtcModule from mytoncore import get_hostname @@ -64,7 +65,7 @@ class Alert: "validator_slashed": Alert( "high", "Validator has been slashed in previous round for {amount} TON", - 0 + 10*HOUR ), } @@ -79,15 +80,20 @@ def __init__(self, ton, local, *args, **kwargs): self.validator_module = None self.inited = False self.hostname = None - self.bot = None self.token = self.ton.local.db.get("BotToken") self.chat_id = self.ton.local.db.get("ChatId") def send_message(self, text: str): - if self.bot is not None: - self.bot.send_message(self.chat_id, text) - else: - raise Exception("send_message error: bot is not initialized") + if self.token is None: + raise Exception("send_message error: token is not initialized") + request_url = f"https://api.telegram.org/bot{self.token}/sendMessage" + data = {'chat_id': self.chat_id, 'text': text, 'parse_mode': 'HTML'} + response = requests.post(request_url, data=data, timeout=3) + if response.status_code != 200: + raise Exception(f"send_message error: {response.text}") + response = response.json() + if not response['ok']: + raise Exception(f"send_message error: {response}") def send_alert(self, alert_name: str, *args, **kwargs): last_sent = self.get_alert_sent(alert_name) @@ -117,8 +123,6 @@ def init(self): raise Exception("BotToken or ChatId is not set") from modules.validator import ValidatorModule self.validator_module = ValidatorModule(self.ton, self.local) - import telebot - self.bot = telebot.TeleBot(self.token, parse_mode="HTML") self.hostname = get_hostname() self.inited = True diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index e6cfcd16..44975561 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -3049,9 +3049,6 @@ def check_enable_mode(self, name): if name == 'liquid-staking': from mytoninstaller.settings import enable_ton_http_api enable_ton_http_api(self.local) - if name == 'alert-bot': - args = ["pip", "install", "pytelegrambotapi==4.23.0"] - subprocess.run(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=10) def enable_mode(self, name): if name not in MODES: From 4df15f50630a8eaab6623b5ece3e658e8c7f0e27 Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 9 Oct 2024 13:46:02 +0400 Subject: [PATCH 11/20] fix typo --- mytonctrl/resources/translate.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mytonctrl/resources/translate.json b/mytonctrl/resources/translate.json index 60e1501a..47be11b4 100644 --- a/mytonctrl/resources/translate.json +++ b/mytonctrl/resources/translate.json @@ -445,9 +445,9 @@ "zh_TW": "{red}錯誤 - 驗證器的 UDP 端口無法從外部訪問.{endc}" }, "slashed_warning": { - "en": "{red}You were fined by {0} for low efficiency in the previous round.{endc}", - "ru": "{red}Вы были оштрафованы на {0} за низкую эффективность в предыдущем раунде.{endc}", - "zh_TW": "{red}您因上一輪效率低而被罰款 {0}。{endc}" + "en": "{red}You were fined by {0} TON for low efficiency in the previous round.{endc}", + "ru": "{red}Вы были оштрафованы на {0} TON за низкую эффективность в предыдущем раунде.{endc}", + "zh_TW": "{red}您因上一輪效率低而被罰款 {0} TON。{endc}" }, "add_custom_overlay_cmd": { "en": "Add custom overlay", From b2e817ffacd31027115ec5cf49f375acbdc76c79 Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 9 Oct 2024 20:55:13 +0400 Subject: [PATCH 12/20] fix GetValidatorsList --- mytoncore/mytoncore.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 44975561..e7155502 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -2447,14 +2447,14 @@ def GetValidatorsList(self, past=False, fast=False, start=None, end=None): #end if config = self.GetConfig34() + if end is None: + timestamp = get_timestamp() + end = timestamp - 60 if start is None: if fast: start = end - 1000 else: start = config.get("startWorkTime") - if end is None: - timestamp = get_timestamp() - end = timestamp - 60 if past: config = self.GetConfig32() start = config.get("startWorkTime") From e12f9a08c7aad3df2358a433e80e3854194bb516 Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 9 Oct 2024 21:14:47 +0400 Subject: [PATCH 13/20] fix buffering GetValidatorsList --- mytoncore/mytoncore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index e7155502..88c3e873 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -2440,7 +2440,7 @@ def GetValidatorsLoad(self, start, end, saveCompFiles=False) -> dict: def GetValidatorsList(self, past=False, fast=False, start=None, end=None): # Get buffer - bname = "validatorsList" + str(past) + bname = "validatorsList" + str(past) + str(start) + str(end) buff = self.GetFunctionBuffer(bname, timeout=60) if buff: return buff From 101dd9c18120245a19376e66081399450f4b59ba Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 9 Oct 2024 22:23:17 +0400 Subject: [PATCH 14/20] add constants to alerting --- modules/alert_bot.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index f3faca8f..b01e706b 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -3,6 +3,7 @@ import requests from modules.module import MtcModule +from mypylib.mypylib import get_timestamp from mytoncore import get_hostname from mytonctrl.utils import timestamp2utcdatetime @@ -15,6 +16,8 @@ class Alert: HOUR = 3600 +VALIDATION_PERIOD = 65536 +FREEZE_PERIOD = 32768 ALERTS = { @@ -40,7 +43,7 @@ class Alert: "low_efficiency": Alert( "high", """Validator efficiency is low: {efficiency}%.""", - 12*HOUR + VALIDATION_PERIOD // 3 ), "out_of_sync": Alert( "critical", @@ -59,13 +62,13 @@ class Alert: ), "zero_block_created": Alert( "critical", - "Validator has not created any blocks in the last 6 hours.", - 6 * HOUR + "Validator has not created any blocks in the last {hours} hours.", + VALIDATION_PERIOD // 3 ), "validator_slashed": Alert( "high", "Validator has been slashed in previous round for {amount} TON", - 10*HOUR + FREEZE_PERIOD ), } @@ -116,6 +119,13 @@ def send_alert(self, alert_name: str, *args, **kwargs): self.send_message(text) self.set_alert_sent(alert_name) + def set_global_vars(self): + # set global vars for correct alerts timeouts for current network + config15 = self.ton.GetConfig15() + global VALIDATION_PERIOD, FREEZE_PERIOD + VALIDATION_PERIOD = config15["validatorsElectedFor"] + FREEZE_PERIOD = config15["stakeHeldFor"] + def init(self): if not self.ton.get_mode_value('alert-bot'): return @@ -124,6 +134,7 @@ def init(self): from modules.validator import ValidatorModule self.validator_module = ValidatorModule(self.ton, self.local) self.hostname = get_hostname() + self.set_global_vars() self.inited = True def set_alert_sent(self, alert_name: str): @@ -148,7 +159,7 @@ def check_validator_wallet_balance(self): return validator_wallet = self.ton.GetValidatorWallet() validator_account = self.ton.GetAccount(validator_wallet.addrB64) - if validator_account.balance < 50: + if validator_account.balance < 10: self.send_alert("low_wallet_balance", wallet=validator_wallet.addrB64, balance=validator_account.balance) def check_efficiency(self): @@ -176,11 +187,14 @@ def check_sync(self): def check_zero_blocks_created(self): if not self.ton.using_validator(): return - validators = self.ton.GetValidatorsList(start=-6*HOUR, end=-60) + ts = get_timestamp() + period = VALIDATION_PERIOD // 3 # 6h for mainnet, 40m for testnet + start, end = ts - period, ts - 60 + validators = self.ton.GetValidatorsList(start=start, end=end) validator = self.validator_module.find_myself(validators) if validator is None or validator.blocks_created > 0: return - self.send_alert("zero_block_created") + self.send_alert("zero_block_created", hours=period // 3600) def check_slashed(self): if not self.ton.using_validator(): From 8375ed072f2dc7bce9e104e1e4922d2af1297028 Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 9 Oct 2024 22:29:36 +0400 Subject: [PATCH 15/20] fix check_zero_blocks_created --- modules/alert_bot.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index b01e706b..21ed63f1 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -190,6 +190,9 @@ def check_zero_blocks_created(self): ts = get_timestamp() period = VALIDATION_PERIOD // 3 # 6h for mainnet, 40m for testnet start, end = ts - period, ts - 60 + config34 = self.ton.GetConfig34() + if start < config34.startWorkTime: # round started recently + return validators = self.ton.GetValidatorsList(start=start, end=end) validator = self.validator_module.find_myself(validators) if validator is None or validator.blocks_created > 0: From 09e1e81260cf57305f64377b66f8225dbd157c65 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 10 Oct 2024 16:20:53 +0400 Subject: [PATCH 16/20] rm 'Next alert...' from alerting message --- modules/alert_bot.py | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 21ed63f1..8a748276 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -110,7 +110,6 @@ def send_alert(self, alert_name: str, *args, **kwargs): Hostname: {self.hostname} Time: {time_} ({int(time.time())}) Severity: {alert.severity} -Next alert of this type not earlier than: {max(alert.timeout, 1000)} sec Alert text:
{alert.text.format(*args, **kwargs)}
From a27fcec182257a39bcd93b0a0336bcdbc11e6a2d Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 10 Oct 2024 16:26:57 +0400 Subject: [PATCH 17/20] add check_adnl_connection_failed --- modules/alert_bot.py | 8 ++++++++ modules/utilities.py | 34 ++++++++++++++++++++++++++++++++-- mytonctrl/mytonctrl.py | 28 +++------------------------- 3 files changed, 43 insertions(+), 27 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 8a748276..339f0fcd 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -205,6 +205,13 @@ def check_slashed(self): if c is not None: self.send_alert("validator_slashed", amount=int(c['suggestedFine'])) + def check_adnl_connection_failed(self): + from modules.utilities import UtilitiesModule + utils_module = UtilitiesModule(self.ton, self.local) + ok, error = utils_module.check_adnl_connection() + if not ok: + self.send_alert("adnl_connection_failed") + def check_status(self): if not self.inited: self.init() @@ -216,6 +223,7 @@ def check_status(self): self.local.try_function(self.check_zero_blocks_created) self.local.try_function(self.check_sync) self.local.try_function(self.check_slashed) + self.local.try_function(self.check_adnl_connection_failed) def add_console_commands(self, console): ... diff --git a/modules/utilities.py b/modules/utilities.py index 958a564a..a19ea581 100644 --- a/modules/utilities.py +++ b/modules/utilities.py @@ -1,9 +1,10 @@ -import base64 import json -import os +import random import subprocess import time +import requests + from mypylib.mypylib import color_print, print_table, color_text, timeago, bcolors from modules.module import MtcModule @@ -335,6 +336,35 @@ def print_validator_list(self, args): print_table(table) # end define + def check_adnl_connection(self): + telemetry = self.ton.local.db.get("sendTelemetry", False) + check_adnl = self.ton.local.db.get("checkAdnl", telemetry) + if not check_adnl: + return True, '' + self.local.add_log('Checking ADNL connection to local node', 'info') + hosts = ['45.129.96.53', '5.154.181.153', '2.56.126.137', '91.194.11.68', '45.12.134.214', '138.124.184.27', + '103.106.3.171'] + hosts = random.sample(hosts, k=3) + data = self.ton.get_local_adnl_data() + error = '' + ok = True + for host in hosts: + url = f'http://{host}/adnl_check' + try: + response = requests.post(url, json=data, timeout=5).json() + except Exception as e: + ok = False + error = f'{{red}}Failed to check ADNL connection to local node: {type(e)}: {e}{{endc}}' + continue + result = response.get("ok") + if result: + ok = True + break + if not result: + ok = False + error = f'{{red}}Failed to check ADNL connection to local node: {response.get("message")}{{endc}}' + return ok, error + def add_console_commands(self, console): console.AddItem("vas", self.view_account_status, self.local.translate("vas_cmd")) console.AddItem("vah", self.view_account_history, self.local.translate("vah_cmd")) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 07753126..535de65a 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -492,31 +492,9 @@ def check_slashed(local, ton): #end define def check_adnl(local, ton): - telemetry = ton.local.db.get("sendTelemetry", False) - check_adnl = ton.local.db.get("checkAdnl", telemetry) - local.add_log('Checking ADNL connection to local node', 'info') - if not check_adnl: - return - hosts = ['45.129.96.53', '5.154.181.153', '2.56.126.137', '91.194.11.68', '45.12.134.214', '138.124.184.27', '103.106.3.171'] - hosts = random.sample(hosts, k=3) - data = ton.get_local_adnl_data() - error = '' - ok = True - for host in hosts: - url = f'http://{host}/adnl_check' - try: - response = requests.post(url, json=data, timeout=5).json() - except Exception as e: - ok = False - error = f'{{red}}Failed to check ADNL connection to local node: {type(e)}: {e}{{endc}}' - continue - result = response.get("ok") - if result: - ok = True - break - if not result: - ok = False - error = f'{{red}}Failed to check ADNL connection to local node: {response.get("message")}{{endc}}' + from modules.utilities import UtilitiesModule + utils_module = UtilitiesModule(ton, local) + ok, error = utils_module.check_adnl_connection() if not ok: print_warning(local, error) #end define From ffd8c4175908a11822eb54b8f0f7ca546ac08231 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 10 Oct 2024 16:28:31 +0400 Subject: [PATCH 18/20] round hours in zero_block_created alert --- modules/alert_bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 339f0fcd..4deda90c 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -196,7 +196,7 @@ def check_zero_blocks_created(self): validator = self.validator_module.find_myself(validators) if validator is None or validator.blocks_created > 0: return - self.send_alert("zero_block_created", hours=period // 3600) + self.send_alert("zero_block_created", hours=round(period // 3600, 1)) def check_slashed(self): if not self.ton.using_validator(): From b47ab237d3cbfd9b4152e8d35e7c960055b1de2d Mon Sep 17 00:00:00 2001 From: yungwine Date: Mon, 14 Oct 2024 19:59:38 +0400 Subject: [PATCH 19/20] add en(dis)bling alerts --- modules/alert_bot.py | 53 ++++++++++++++++++++++++++---- mytoncore/mytoncore.py | 3 ++ mytonctrl/mytonctrl.py | 5 +++ mytonctrl/resources/translate.json | 15 +++++++++ 4 files changed, 69 insertions(+), 7 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 4deda90c..8756a9c9 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -3,7 +3,7 @@ import requests from modules.module import MtcModule -from mypylib.mypylib import get_timestamp +from mypylib.mypylib import get_timestamp, print_table, color_print from mytoncore import get_hostname from mytonctrl.utils import timestamp2utcdatetime @@ -99,6 +99,8 @@ def send_message(self, text: str): raise Exception(f"send_message error: {response}") def send_alert(self, alert_name: str, *args, **kwargs): + if not self.alert_is_enabled(alert_name): + return last_sent = self.get_alert_sent(alert_name) time_ = timestamp2utcdatetime(int(time.time())) alert = ALERTS.get(alert_name) @@ -136,15 +138,50 @@ def init(self): self.set_global_vars() self.inited = True - def set_alert_sent(self, alert_name: str): + def get_alert_from_db(self, alert_name: str): if 'alerts' not in self.ton.local.db: self.ton.local.db['alerts'] = {} - self.ton.local.db['alerts'][alert_name] = int(time.time()) + if alert_name not in self.ton.local.db['alerts']: + self.ton.local.db['alerts'][alert_name] = {'sent': 0, 'enabled': True} + return self.ton.local.db['alerts'][alert_name] + + def set_alert_sent(self, alert_name: str): + alert = self.get_alert_from_db(alert_name) + alert['sent'] = int(time.time()) def get_alert_sent(self, alert_name: str): - if 'alerts' not in self.ton.local.db: - return 0 - return self.ton.local.db['alerts'].get(alert_name, 0) + alert = self.get_alert_from_db(alert_name) + return alert.get('sent', 0) + + def alert_is_enabled(self, alert_name: str): + alert = self.get_alert_from_db(alert_name) + return alert.get('enabled', True) # default is True + + def set_alert_enabled(self, alert_name: str, enabled: bool): + alert = self.get_alert_from_db(alert_name) + alert['enabled'] = enabled + self.ton.local.save() + + def enable_alert(self, args): + if len(args) != 1: + raise Exception("Usage: enable_alert ") + alert_name = args[0] + self.set_alert_enabled(alert_name, True) + color_print("enable_alert - {green}OK{endc}") + + def disable_alert(self, args): + if len(args) != 1: + raise Exception("Usage: disable_alert ") + alert_name = args[0] + self.set_alert_enabled(alert_name, False) + color_print("disable_alert - {green}OK{endc}") + + def print_alerts(self, args): + table = [['Name', 'Enabled', 'Last sent']] + for alert_name in ALERTS: + alert = self.get_alert_from_db(alert_name) + table.append([alert_name, alert['enabled'], alert['sent']]) + print_table(table) def check_db_usage(self): usage = self.ton.GetDbUsage() @@ -226,4 +263,6 @@ def check_status(self): self.local.try_function(self.check_adnl_connection_failed) def add_console_commands(self, console): - ... + console.AddItem("enable_alert", self.enable_alert, self.local.translate("enable_alert_cmd")) + console.AddItem("disable_alert", self.disable_alert, self.local.translate("disable_alert_cmd")) + console.AddItem("list_alerts", self.print_alerts, self.local.translate("list_alerts_cmd")) diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 88c3e873..2b7e2119 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -3089,6 +3089,9 @@ def using_validator(self): def using_liteserver(self): return self.get_mode_value('liteserver') + def using_alert_bot(self): + return self.get_mode_value('alert-bot') + def Tlb2Json(self, text): # Заменить скобки start = 0 diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 535de65a..44a868e5 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -135,6 +135,11 @@ def inject_globals(func): module = ControllerModule(ton, local) module.add_console_commands(console) + if ton.using_alert_bot(): + from modules.alert_bot import AlertBotModule + module = AlertBotModule(ton, local) + module.add_console_commands(console) + console.AddItem("cleanup", inject_globals(cleanup_validator_db), local.translate("cleanup_cmd")) console.AddItem("benchmark", inject_globals(run_benchmark), local.translate("benchmark_cmd")) # console.AddItem("activate_ton_storage_provider", inject_globals(activate_ton_storage_provider), local.translate("activate_ton_storage_provider_cmd")) diff --git a/mytonctrl/resources/translate.json b/mytonctrl/resources/translate.json index 47be11b4..2a5ae89e 100644 --- a/mytonctrl/resources/translate.json +++ b/mytonctrl/resources/translate.json @@ -464,6 +464,21 @@ "ru": "Удалить пользовательский оверлей", "zh_TW": "刪除自定義覆蓋" }, + "enable_alert_cmd": { + "en": "Enable specific Telegram Bot alert", + "ru": "Включить определенное оповещение через Telegram Bot", + "zh_TW": "啟用特定的 Telegram Bot 警報" + }, + "disable_alert_cmd": { + "en": "Disable specific Telegram Bot alert", + "ru": "Отключить определенное оповещение через Telegram Bot", + "zh_TW": "禁用特定的 Telegram Bot 警報" + }, + "list_alerts_cmd": { + "en": "List all available Telegram Bot alerts", + "ru": "Список всех доступных оповещений через Telegram Bot", + "zh_TW": "列出所有可用的 Telegram Bot 警報" + }, "cleanup_cmd": { "en": "Clean node old logs and temp files", "ru": "Очистить старые логи и временные файлы ноды", From bf84a530c0b94e6097d0b574cc886d9767ca8765 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 17 Oct 2024 21:40:44 +0400 Subject: [PATCH 20/20] add test_alert cmd --- modules/alert_bot.py | 6 ++++++ mytonctrl/resources/translate.json | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 8756a9c9..5ba4d99a 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -89,6 +89,8 @@ def __init__(self, ton, local, *args, **kwargs): def send_message(self, text: str): if self.token is None: raise Exception("send_message error: token is not initialized") + if self.chat_id is None: + raise Exception("send_message error: chat_id is not initialized") request_url = f"https://api.telegram.org/bot{self.token}/sendMessage" data = {'chat_id': self.chat_id, 'text': text, 'parse_mode': 'HTML'} response = requests.post(request_url, data=data, timeout=3) @@ -183,6 +185,9 @@ def print_alerts(self, args): table.append([alert_name, alert['enabled'], alert['sent']]) print_table(table) + def test_alert(self, args): + self.send_message('Test alert') + def check_db_usage(self): usage = self.ton.GetDbUsage() if usage > 95: @@ -266,3 +271,4 @@ def add_console_commands(self, console): console.AddItem("enable_alert", self.enable_alert, self.local.translate("enable_alert_cmd")) console.AddItem("disable_alert", self.disable_alert, self.local.translate("disable_alert_cmd")) console.AddItem("list_alerts", self.print_alerts, self.local.translate("list_alerts_cmd")) + console.AddItem("test_alert", self.test_alert, self.local.translate("test_alert_cmd")) diff --git a/mytonctrl/resources/translate.json b/mytonctrl/resources/translate.json index 2a5ae89e..929eea8a 100644 --- a/mytonctrl/resources/translate.json +++ b/mytonctrl/resources/translate.json @@ -479,6 +479,11 @@ "ru": "Список всех доступных оповещений через Telegram Bot", "zh_TW": "列出所有可用的 Telegram Bot 警報" }, + "test_alert_cmd": { + "en": "Send test alert via Telegram Bot", + "ru": "Отправить тестовое оповещение через Telegram Bot", + "zh_TW": "通過 Telegram Bot 發送測試警報" + }, "cleanup_cmd": { "en": "Clean node old logs and temp files", "ru": "Очистить старые логи и временные файлы ноды",