From 2ff35884a5b43c92453bccd8421dc2141b08b5b6 Mon Sep 17 00:00:00 2001 From: Maksim Kurbatov <94808996+yungwine@users.noreply.github.com> Date: Wed, 8 Jan 2025 16:36:19 +0400 Subject: [PATCH 1/8] add stake info alerting --- modules/alert_bot.py | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 2ebf4034..5b99a7a5 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -18,6 +18,7 @@ class Alert: HOUR = 3600 VALIDATION_PERIOD = 65536 FREEZE_PERIOD = 32768 +ELECTIONS_START_BEFORE = 8192 ALERTS = { @@ -70,6 +71,16 @@ class Alert: "Validator has been slashed in previous round for {amount} TON", FREEZE_PERIOD ), + "stake_not_accepted": Alert( + "high", + "Validator's stake has not been accepted", + ELECTIONS_START_BEFORE + ), + "stake_accepted": Alert( + "info", + "Validator's stake {stake} TON has been accepted", + ELECTIONS_START_BEFORE + ) } @@ -128,9 +139,10 @@ def send_alert(self, alert_name: str, *args, **kwargs): def set_global_vars(self): # set global vars for correct alerts timeouts for current network config15 = self.ton.GetConfig15() - global VALIDATION_PERIOD, FREEZE_PERIOD + global VALIDATION_PERIOD, FREEZE_PERIOD, ELECTIONS_START_BEFORE VALIDATION_PERIOD = config15["validatorsElectedFor"] FREEZE_PERIOD = config15["stakeHeldFor"] + ELECTIONS_START_BEFORE = config15["electionsStartBefore"] def init(self): if not self.ton.get_mode_value('alert-bot'): @@ -271,6 +283,32 @@ def check_adnl_connection_failed(self): if not ok: self.send_alert("adnl_connection_failed") + def get_myself_from_election(self): + config = self.ton.GetConfig36() + if not config["validators"]: + return + validator = self.validator_module.find_myself(config["validators"]) + if validator is None: + return False + save_elections = self.ton.GetSaveElections() + elections = save_elections.get(str(config["startWorkTime"])) + if elections is None: + return validator + adnl = self.ton.GetAdnlAddr() + validator['stake'] = elections[adnl].get('stake') + return validator + + def check_stake(self): + if not self.ton.using_validator(): + return + res = self.get_myself_from_election() + if res is None: + return + if res is False: + self.send_alert("stake_not_accepted") + return + self.send_alert("stake_accepted", stake=res.get('stake')) + def check_status(self): if not self.ton.using_alert_bot(): return @@ -285,6 +323,7 @@ def check_status(self): self.local.try_function(self.check_sync) self.local.try_function(self.check_slashed) self.local.try_function(self.check_adnl_connection_failed) + self.local.try_function(self.check_stake) def add_console_commands(self, console): console.AddItem("enable_alert", self.enable_alert, self.local.translate("enable_alert_cmd")) From 68c459f4a17e2af3dd69f8d8f3e12dd9da0baa5f Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 9 Jan 2025 12:06:44 +0400 Subject: [PATCH 2/8] send info alerts without sound --- modules/alert_bot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 5b99a7a5..676f0fac 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -99,13 +99,13 @@ def __init__(self, ton, local, *args, **kwargs): self.chat_id = None self.last_db_check = 0 - def send_message(self, text: str): + def send_message(self, text: str, silent: bool = False): if self.token is None: raise Exception("send_message error: token is not initialized") if self.chat_id is None: raise Exception("send_message error: chat_id is not initialized") request_url = f"https://api.telegram.org/bot{self.token}/sendMessage" - data = {'chat_id': self.chat_id, 'text': text, 'parse_mode': 'HTML'} + data = {'chat_id': self.chat_id, 'text': text, 'parse_mode': 'HTML', 'disable_notification': silent} response = requests.post(request_url, data=data, timeout=3) if response.status_code != 200: raise Exception(f"send_message error: {response.text}") @@ -133,7 +133,7 @@ def send_alert(self, alert_name: str, *args, **kwargs):
{alert.text.format(*args, **kwargs)}
''' if time.time() - last_sent > alert.timeout: - self.send_message(text) + self.send_message(text, alert.severity == "info") # send info alerts without sound self.set_alert_sent(alert_name) def set_global_vars(self): From 1e3590ee3270c2d156f4c9674988b116e2165e3a Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 21 Jan 2025 11:16:14 +0400 Subject: [PATCH 3/8] add alerts on stake return --- modules/alert_bot.py | 54 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 676f0fac..0ab807eb 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -80,7 +80,17 @@ class Alert: "info", "Validator's stake {stake} TON has been accepted", ELECTIONS_START_BEFORE - ) + ), + "stake_returned": Alert( + "info", + "Validator's stake {stake} TON has been returned on address {address}. The reward amount is {reward} TON.", + 0 + ), + "stake_not_returned": Alert( + "high", + "Validator's stake has not been returned on address {address}.", + 0 + ), } @@ -283,25 +293,28 @@ def check_adnl_connection_failed(self): if not ok: self.send_alert("adnl_connection_failed") - def get_myself_from_election(self): - config = self.ton.GetConfig36() + def get_myself_from_election(self, config: dict): if not config["validators"]: return - validator = self.validator_module.find_myself(config["validators"]) - if validator is None: - return False + adnl = self.ton.GetAdnlAddr() save_elections = self.ton.GetSaveElections() elections = save_elections.get(str(config["startWorkTime"])) if elections is None: - return validator - adnl = self.ton.GetAdnlAddr() + return + if adnl not in elections: # didn't participate in elections + return + validator = self.validator_module.find_myself(config["validators"]) + if validator is None: + return False validator['stake'] = elections[adnl].get('stake') + validator['walletAddr'] = elections[adnl].get('walletAddr') return validator - def check_stake(self): + def check_stake_sent(self): if not self.ton.using_validator(): return - res = self.get_myself_from_election() + config = self.ton.GetConfig36() + res = self.get_myself_from_election(config) if res is None: return if res is False: @@ -309,6 +322,24 @@ def check_stake(self): return self.send_alert("stake_accepted", stake=res.get('stake')) + def check_stake_returned(self): + if not self.ton.using_validator(): + return + config = self.ton.GetConfig32() + if not (config['endWorkTime'] + FREEZE_PERIOD + 1200 <= time.time() < config['endWorkTime'] + FREEZE_PERIOD + 1260): # check between 20th and 21st minutes after stakes have been unfrozen + return + res = self.get_myself_from_election(config) + if not res: + return + trs = self.ton.GetAccountHistory(self.ton.GetAccount(res["walletAddr"]), limit=10) + + for tr in trs: + if tr.time >= config['endWorkTime'] + FREEZE_PERIOD and tr.srcAddr == '3333333333333333333333333333333333333333333333333333333333333333' and tr.body.startswith('F96F7324'): # Elector Recover Stake Response + self.send_alert("stake_returned", stake=res.get('stake'), address=res["walletAddr"], reward=tr.value - res.get('stake', 0)) + return + self.send_alert("stake_not_returned", address=res["walletAddr"]) + + def check_status(self): if not self.ton.using_alert_bot(): return @@ -323,7 +354,8 @@ def check_status(self): self.local.try_function(self.check_sync) self.local.try_function(self.check_slashed) self.local.try_function(self.check_adnl_connection_failed) - self.local.try_function(self.check_stake) + self.local.try_function(self.check_stake_sent) + self.local.try_function(self.check_stake_returned) def add_console_commands(self, console): console.AddItem("enable_alert", self.enable_alert, self.local.translate("enable_alert_cmd")) From b7aca1ccb064d3ad1324d378381874fcef33ca04 Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 22 Jan 2025 16:11:03 +0400 Subject: [PATCH 4/8] round values in stake alerts --- modules/alert_bot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 0ab807eb..c3a4cfae 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -320,7 +320,7 @@ def check_stake_sent(self): if res is False: self.send_alert("stake_not_accepted") return - self.send_alert("stake_accepted", stake=res.get('stake')) + self.send_alert("stake_accepted", stake=round(res.get('stake'), 2)) def check_stake_returned(self): if not self.ton.using_validator(): @@ -335,7 +335,7 @@ def check_stake_returned(self): for tr in trs: if tr.time >= config['endWorkTime'] + FREEZE_PERIOD and tr.srcAddr == '3333333333333333333333333333333333333333333333333333333333333333' and tr.body.startswith('F96F7324'): # Elector Recover Stake Response - self.send_alert("stake_returned", stake=res.get('stake'), address=res["walletAddr"], reward=tr.value - res.get('stake', 0)) + self.send_alert("stake_returned", stake=round(tr.value, 2), address=res["walletAddr"], reward=round(tr.value - res.get('stake', 0), 2)) return self.send_alert("stake_not_returned", address=res["walletAddr"]) From 2c1a238a6bf47de7f3f35b3fcf9c183d80e8c35b Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 23 Jan 2025 13:13:40 +0400 Subject: [PATCH 5/8] add voting alert --- modules/alert_bot.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index c3a4cfae..c7348a23 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -84,12 +84,17 @@ class Alert: "stake_returned": Alert( "info", "Validator's stake {stake} TON has been returned on address {address}. The reward amount is {reward} TON.", - 0 + 60 ), "stake_not_returned": Alert( "high", "Validator's stake has not been returned on address {address}.", - 0 + 60 + ), + "voting": Alert( + "high", + "Found proposals with hashes `{hashes}` that have significant amount of votes, but current validator didn't vote for them. Please check @tonstatus for more details.", + VALIDATION_PERIOD ), } @@ -339,6 +344,19 @@ def check_stake_returned(self): return self.send_alert("stake_not_returned", address=res["walletAddr"]) + def check_voting(self): + if not self.ton.using_validator(): + return + validator_index = self.ton.GetValidatorIndex() + if validator_index == -1: + return + need_to_vote = [] + offers = self.ton.GetOffers() + for offer in offers: + if not offer['isPassed'] and offer['approvedPercent'] >= 50 and validator_index not in offer['votedValidators']: + need_to_vote.append(offer['hash']) + if need_to_vote: + self.send_alert("voting", hashes=' '.join(need_to_vote)) def check_status(self): if not self.ton.using_alert_bot(): @@ -356,6 +374,7 @@ def check_status(self): self.local.try_function(self.check_adnl_connection_failed) self.local.try_function(self.check_stake_sent) self.local.try_function(self.check_stake_returned) + self.local.try_function(self.check_voting) def add_console_commands(self, console): console.AddItem("enable_alert", self.enable_alert, self.local.translate("enable_alert_cmd")) From 20637b63c7a565c16bc00fe8a0f82061b974a488 Mon Sep 17 00:00:00 2001 From: yungwine Date: Fri, 24 Jan 2025 11:46:53 +0400 Subject: [PATCH 6/8] fix alerts timeouts for testnet --- modules/alert_bot.py | 160 ++++++++++++++++++++++--------------------- 1 file changed, 83 insertions(+), 77 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index c7348a23..c1ee2867 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -21,82 +21,87 @@ class Alert: ELECTIONS_START_BEFORE = 8192 -ALERTS = { - "low_wallet_balance": Alert( - "low", - "Validator wallet {wallet} balance is low: {balance} TON.", - 18*HOUR - ), - "db_usage_80": Alert( - "high", - """TON DB usage > 80%. Clean the TON database: - https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming - or (and) set node\'s archive ttl to lower value.""", - 24*HOUR - ), - "db_usage_95": Alert( - "critical", - """TON DB usage > 95%. Disk is almost full, clean the TON database immediately: - https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming - or (and) set node\'s archive ttl to lower value.""", - 6*HOUR - ), - "low_efficiency": Alert( - "high", - """Validator efficiency is low: {efficiency}%.""", - VALIDATION_PERIOD // 3 - ), - "out_of_sync": Alert( - "critical", - "Node is out of sync on {sync} sec.", - 300 - ), - "service_down": Alert( - "critical", - "validator.service is down.", - 300 - ), - "adnl_connection_failed": Alert( - "high", - "ADNL connection to node failed", - 3*HOUR - ), - "zero_block_created": Alert( - "critical", - "Validator has not created any blocks in the last {hours} hours.", - VALIDATION_PERIOD // 3 - ), - "validator_slashed": Alert( - "high", - "Validator has been slashed in previous round for {amount} TON", - FREEZE_PERIOD - ), - "stake_not_accepted": Alert( - "high", - "Validator's stake has not been accepted", - ELECTIONS_START_BEFORE - ), - "stake_accepted": Alert( - "info", - "Validator's stake {stake} TON has been accepted", - ELECTIONS_START_BEFORE - ), - "stake_returned": Alert( - "info", - "Validator's stake {stake} TON has been returned on address {address}. The reward amount is {reward} TON.", - 60 - ), - "stake_not_returned": Alert( - "high", - "Validator's stake has not been returned on address {address}.", - 60 - ), - "voting": Alert( - "high", - "Found proposals with hashes `{hashes}` that have significant amount of votes, but current validator didn't vote for them. Please check @tonstatus for more details.", - VALIDATION_PERIOD - ), -} +ALERTS = {} + + +def init_alerts(): + global ALERTS + ALERTS = { + "low_wallet_balance": Alert( + "low", + "Validator wallet {wallet} balance is low: {balance} TON.", + 18 * HOUR + ), + "db_usage_80": Alert( + "high", + """TON DB usage > 80%. Clean the TON database: + https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming + or (and) set node\'s archive ttl to lower value.""", + 24 * HOUR + ), + "db_usage_95": Alert( + "critical", + """TON DB usage > 95%. Disk is almost full, clean the TON database immediately: + https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming + or (and) set node\'s archive ttl to lower value.""", + 6 * HOUR + ), + "low_efficiency": Alert( + "high", + """Validator efficiency is low: {efficiency}%.""", + VALIDATION_PERIOD // 3 + ), + "out_of_sync": Alert( + "critical", + "Node is out of sync on {sync} sec.", + 300 + ), + "service_down": Alert( + "critical", + "validator.service is down.", + 300 + ), + "adnl_connection_failed": Alert( + "high", + "ADNL connection to node failed", + 3 * HOUR + ), + "zero_block_created": Alert( + "critical", + "Validator has not created any blocks in the last {hours} hours.", + VALIDATION_PERIOD // 3 + ), + "validator_slashed": Alert( + "high", + "Validator has been slashed in previous round for {amount} TON", + FREEZE_PERIOD + ), + "stake_not_accepted": Alert( + "high", + "Validator's stake has not been accepted", + ELECTIONS_START_BEFORE + ), + "stake_accepted": Alert( + "info", + "Validator's stake {stake} TON has been accepted", + ELECTIONS_START_BEFORE + ), + "stake_returned": Alert( + "info", + "Validator's stake {stake} TON has been returned on address {address}. The reward amount is {reward} TON.", + 60 + ), + "stake_not_returned": Alert( + "high", + "Validator's stake has not been returned on address {address}.", + 60 + ), + "voting": Alert( + "high", + "Found proposals with hashes `{hashes}` that have significant amount of votes, but current validator didn't vote for them. Please check @tonstatus for more details.", + VALIDATION_PERIOD + ), + } class AlertBotModule(MtcModule): @@ -171,6 +176,7 @@ def init(self): self.hostname = get_hostname() self.ip = self.ton.get_validator_engine_ip() self.set_global_vars() + init_alerts() self.inited = True def get_alert_from_db(self, alert_name: str): @@ -354,7 +360,7 @@ def check_voting(self): offers = self.ton.GetOffers() for offer in offers: if not offer['isPassed'] and offer['approvedPercent'] >= 50 and validator_index not in offer['votedValidators']: - need_to_vote.append(offer['hash']) + need_to_vote.append(offer['hash']) if need_to_vote: self.send_alert("voting", hashes=' '.join(need_to_vote)) From af306067ce8ac6087d08d759ddbdc0f64e1ad55f Mon Sep 17 00:00:00 2001 From: yungwine Date: Sat, 25 Jan 2025 13:02:27 +0400 Subject: [PATCH 7/8] update stake alert --- modules/alert_bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index c1ee2867..1e4f5132 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -337,7 +337,7 @@ def check_stake_returned(self): if not self.ton.using_validator(): return config = self.ton.GetConfig32() - if not (config['endWorkTime'] + FREEZE_PERIOD + 1200 <= time.time() < config['endWorkTime'] + FREEZE_PERIOD + 1260): # check between 20th and 21st minutes after stakes have been unfrozen + if not (config['endWorkTime'] + FREEZE_PERIOD + 1800 <= time.time() < config['endWorkTime'] + FREEZE_PERIOD + 1801): # check between 25th and 26th minutes after stakes have been unfrozen return res = self.get_myself_from_election(config) if not res: From a201d6d66e375c718a3a6c6a4c3f0a4fdd7fed33 Mon Sep 17 00:00:00 2001 From: yungwine Date: Sat, 25 Jan 2025 21:24:17 +0400 Subject: [PATCH 8/8] update stake alert --- modules/alert_bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 1e4f5132..521bf0d9 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -337,7 +337,7 @@ def check_stake_returned(self): if not self.ton.using_validator(): return config = self.ton.GetConfig32() - if not (config['endWorkTime'] + FREEZE_PERIOD + 1800 <= time.time() < config['endWorkTime'] + FREEZE_PERIOD + 1801): # check between 25th and 26th minutes after stakes have been unfrozen + if not (config['endWorkTime'] + FREEZE_PERIOD + 1800 <= time.time() < config['endWorkTime'] + FREEZE_PERIOD + 1860): # check between 25th and 26th minutes after stakes have been unfrozen return res = self.get_myself_from_election(config) if not res: