Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
79 commits
Select commit Hold shift + click to select a range
f73c2af
feat: Enhance dump download with aria2 for improved speed and reliabi…
enchanted-elephant1 Aug 23, 2024
00fbb95
tmp-file-first.
enchanted-elephant1 Aug 23, 2024
56c5236
cleanup
enchanted-elephant1 Aug 23, 2024
02a36df
add backups support for mtc
yungwine Sep 12, 2024
1412440
fix restore_backup
yungwine Sep 12, 2024
0987a21
fix backups
yungwine Sep 12, 2024
43e5b3e
fix backups
yungwine Sep 12, 2024
2c28d4a
rm verbosity from tar
yungwine Sep 12, 2024
bd9567d
fix backups
yungwine Sep 12, 2024
c5264a8
feat: Update install.sh to allow passing in USER arg
leonlarin Sep 13, 2024
0818be3
Merge pull request #338 from enchanted-elephant1/feature/speed-up-arc…
igroman787 Sep 14, 2024
afabd52
chown keyring in restore_backup
yungwine Sep 17, 2024
858eba2
replace ip in node config in restore_backup
yungwine Sep 17, 2024
caa91e0
refactor checking local adnl con
yungwine Sep 18, 2024
f7a5fe7
fix adnl checking
yungwine Sep 18, 2024
0f4e5ca
show only masterchain efficiency for masterchain validator
yungwine Sep 18, 2024
52c143a
fix check_ef
yungwine Sep 19, 2024
5991651
Merge pull request #364 from yungwine/backup
igroman787 Sep 23, 2024
8340f16
Merge pull request #366 from yungwine/adnl
igroman787 Sep 23, 2024
13a6933
improve backups
yungwine Sep 24, 2024
96d3d23
fix enabling THA
yungwine Sep 24, 2024
c5e0026
update THA installer script
yungwine Sep 24, 2024
bcbfa15
Merge pull request #368 from yungwine/backup
igroman787 Sep 24, 2024
8146d4e
fix buffering funciton for GetValidatorsLoad
yungwine Sep 25, 2024
dd9c30e
Merge branch 'ton-blockchain:dev' into dev
yungwine Sep 26, 2024
3b25c96
fix DownloadDump
yungwine Sep 26, 2024
75fdd96
improve WaitTransaction seqno fetching
yungwine Oct 1, 2024
c89c21a
do not repeat WithdrawFromPool on errors
yungwine Oct 1, 2024
a7d37b5
allow fines 1% of stake for not working validators
yungwine Oct 2, 2024
6ef922b
update fine warnings
yungwine Oct 2, 2024
6252a27
Merge pull request #374 from yungwine/master-complaints
igroman787 Oct 3, 2024
48c2acb
Merge pull request #373 from yungwine/wait-transaction
igroman787 Oct 3, 2024
3a37741
Merge pull request #370 from yungwine/dev
igroman787 Oct 3, 2024
d226dfd
Merge pull request #17 from yungwine/ton-http-api
yungwine Oct 3, 2024
4d899a3
Merge pull request #18 from yungwine/efficiency
yungwine Oct 3, 2024
c0db7f0
Merge pull request #19 from yungwine/dump
yungwine Oct 3, 2024
da653d8
add AlertBot
yungwine Oct 3, 2024
0b7d6f3
fix dataclass
yungwine Oct 3, 2024
1a1d069
fix alerts in db
yungwine Oct 3, 2024
3f0c559
add service_down alert
yungwine Oct 3, 2024
4d63545
fix alerts timeout
yungwine Oct 3, 2024
029acf0
fix double service_down alert
yungwine Oct 3, 2024
47a1904
Merge pull request #365 from yungwine/efficiency
igroman787 Oct 3, 2024
7b8ac88
Merge pull request #369 from yungwine/ton-http-api
igroman787 Oct 3, 2024
e005612
Merge pull request #371 from yungwine/dump
igroman787 Oct 3, 2024
1ee5e9b
update alerts text
yungwine Oct 7, 2024
02ca414
add zero_blocks_created alert
yungwine Oct 8, 2024
67d9620
Revert "update complaints validation and warnings"
yungwine Oct 8, 2024
306f7b3
fix fine warning for testnet
yungwine Oct 8, 2024
4a45167
fix typo
yungwine Oct 8, 2024
07c7a6a
Merge pull request #376 from yungwine/revert-374-master-complaints
igroman787 Oct 8, 2024
ccf17ab
Merge branch 'ton-blockchain:dev' into dev
yungwine Oct 8, 2024
165c3f2
Merge pull request #20 from yungwine/dev
yungwine Oct 8, 2024
e0bb941
add alert if validator was slashed
yungwine Oct 8, 2024
d3c456c
rm telebot dependency for alert-bot mode
yungwine Oct 9, 2024
4df15f5
fix typo
yungwine Oct 9, 2024
b2e817f
fix GetValidatorsList
yungwine Oct 9, 2024
e12f9a0
fix buffering GetValidatorsList
yungwine Oct 9, 2024
101dd9c
add constants to alerting
yungwine Oct 9, 2024
8375ed0
fix check_zero_blocks_created
yungwine Oct 9, 2024
de67b55
return git_pool_data cmd
yungwine Oct 10, 2024
09e1e81
rm 'Next alert...' from alerting message
yungwine Oct 10, 2024
a27fcec
add check_adnl_connection_failed
yungwine Oct 10, 2024
ffd8c41
round hours in zero_block_created alert
yungwine Oct 10, 2024
fdde23b
fine non-master vals that created zero blocks
yungwine Oct 11, 2024
b47ab23
add en(dis)bling alerts
yungwine Oct 14, 2024
d8d165d
Merge pull request #362 from leonlarin/feat/install-script-user-arg
igroman787 Oct 15, 2024
6fafd47
Merge pull request #378 from yungwine/shard-complaints
igroman787 Oct 15, 2024
f8aa2f3
Merge pull request #377 from yungwine/dev
igroman787 Oct 15, 2024
d73259b
Merge branch 'dev' into bot
yungwine Oct 16, 2024
bf84a53
add test_alert cmd
yungwine Oct 17, 2024
19eca04
Merge pull request #379 from yungwine/bot
igroman787 Oct 21, 2024
a4c9fb8
bugfix
igroman787 Oct 21, 2024
c51e00a
fix low_efficiency alert
yungwine Oct 22, 2024
59f2f37
print efficiency for non-master validators if its 0
yungwine Oct 22, 2024
f94a7dc
fix slashed_warning
yungwine Oct 24, 2024
79bf05b
update alerting thread time
yungwine Oct 24, 2024
0a0acb1
Merge pull request #380 from yungwine/bot
igroman787 Oct 24, 2024
2777801
bugfix
igroman787 Oct 26, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion modules/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,16 @@
from modules.validator import ValidatorModule
from modules.controller import ControllerModule
from modules.liteserver import LiteserverModule
from modules.alert_bot import AlertBotModule


MODES = {
'validator': ValidatorModule,
'nominator-pool': NominatorPoolModule,
'single-nominator': SingleNominatorModule,
'liquid-staking': ControllerModule,
'liteserver': LiteserverModule
'liteserver': LiteserverModule,
'alert-bot': AlertBotModule
}


Expand Down Expand Up @@ -55,6 +57,8 @@ class Setting:
'defaultCustomOverlaysUrl': Setting(None, 'https://ton-blockchain.github.io/fallback_custom_overlays.json', 'Default custom overlays config url'),
'debug': Setting(None, False, 'Debug mtc console mode. Prints Traceback on errors'),
'subscribe_tg_channel': Setting('validator', False, 'Disables warning about subscribing to the `TON STATUS` channel'),
'BotToken': Setting('alert-bot', None, 'Alerting Telegram bot token'),
'ChatId': Setting('alert-bot', None, 'Alerting Telegram chat id')
}


Expand Down
285 changes: 285 additions & 0 deletions modules/alert_bot.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,285 @@
import dataclasses
import time
import requests

from modules.module import MtcModule
from mypylib.mypylib import get_timestamp, print_table, color_print
from mytoncore import get_hostname
from mytonctrl.utils import timestamp2utcdatetime


@dataclasses.dataclass
class Alert:
severity: str
text: str
timeout: int


HOUR = 3600
VALIDATION_PERIOD = 65536
FREEZE_PERIOD = 32768


ALERTS = {
"low_wallet_balance": Alert(
"low",
"Validator wallet {wallet} balance is low: {balance} TON.",
18*HOUR
),
"db_usage_80": Alert(
"high",
"""TON DB usage > 80%. Clean the TON database:
https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming
or (and) set node\'s archive ttl to lower value.""",
24*HOUR
),
"db_usage_95": Alert(
"critical",
"""TON DB usage > 95%. Disk is almost full, clean the TON database immediately:
https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming
or (and) set node\'s archive ttl to lower value.""",
6*HOUR
),
"low_efficiency": Alert(
"high",
"""Validator efficiency is low: {efficiency}%.""",
VALIDATION_PERIOD // 3
),
"out_of_sync": Alert(
"critical",
"Node is out of sync on {sync} sec.",
0
),
"service_down": Alert(
"critical",
"validator.service is down.",
0
),
"adnl_connection_failed": Alert(
"high",
"ADNL connection to node failed",
3*HOUR
),
"zero_block_created": Alert(
"critical",
"Validator has not created any blocks in the last {hours} hours.",
VALIDATION_PERIOD // 3
),
"validator_slashed": Alert(
"high",
"Validator has been slashed in previous round for {amount} TON",
FREEZE_PERIOD
),
}


class AlertBotModule(MtcModule):

description = 'Telegram bot alerts'
default_value = False

def __init__(self, ton, local, *args, **kwargs):
super().__init__(ton, local, *args, **kwargs)
self.validator_module = None
self.inited = False
self.hostname = None
self.token = None
self.chat_id = None
self.last_db_check = 0

def send_message(self, text: str):
if self.token is None:
raise Exception("send_message error: token is not initialized")
if self.chat_id is None:
raise Exception("send_message error: chat_id is not initialized")
request_url = f"https://api.telegram.org/bot{self.token}/sendMessage"
data = {'chat_id': self.chat_id, 'text': text, 'parse_mode': 'HTML'}
response = requests.post(request_url, data=data, timeout=3)
if response.status_code != 200:
raise Exception(f"send_message error: {response.text}")
response = response.json()
if not response['ok']:
raise Exception(f"send_message error: {response}")

def send_alert(self, alert_name: str, *args, **kwargs):
if not self.alert_is_enabled(alert_name):
return
last_sent = self.get_alert_sent(alert_name)
time_ = timestamp2utcdatetime(int(time.time()))
alert = ALERTS.get(alert_name)
if alert is None:
raise Exception(f"Alert {alert_name} not found")
text = f'''
❗️ <b>MyTonCtrl Alert {alert_name}</b> ❗️

Hostname: <code>{self.hostname}</code>
Time: <code>{time_}</code> (<code>{int(time.time())}</code>)
Severity: <code>{alert.severity}</code>

Alert text:
<blockquote> {alert.text.format(*args, **kwargs)} </blockquote>
'''
if time.time() - last_sent > alert.timeout:
self.send_message(text)
self.set_alert_sent(alert_name)

def set_global_vars(self):
# set global vars for correct alerts timeouts for current network
config15 = self.ton.GetConfig15()
global VALIDATION_PERIOD, FREEZE_PERIOD
VALIDATION_PERIOD = config15["validatorsElectedFor"]
FREEZE_PERIOD = config15["stakeHeldFor"]

def init(self):
if not self.ton.get_mode_value('alert-bot'):
return
self.token = self.ton.local.db.get("BotToken")
self.chat_id = self.ton.local.db.get("ChatId")
if self.token is None or self.chat_id is None:
raise Exception("BotToken or ChatId is not set")
from modules.validator import ValidatorModule
self.validator_module = ValidatorModule(self.ton, self.local)
self.hostname = get_hostname()
self.set_global_vars()
self.inited = True

def get_alert_from_db(self, alert_name: str):
if 'alerts' not in self.ton.local.db:
self.ton.local.db['alerts'] = {}
if alert_name not in self.ton.local.db['alerts']:
self.ton.local.db['alerts'][alert_name] = {'sent': 0, 'enabled': True}
return self.ton.local.db['alerts'][alert_name]

def set_alert_sent(self, alert_name: str):
alert = self.get_alert_from_db(alert_name)
alert['sent'] = int(time.time())

def get_alert_sent(self, alert_name: str):
alert = self.get_alert_from_db(alert_name)
return alert.get('sent', 0)

def alert_is_enabled(self, alert_name: str):
alert = self.get_alert_from_db(alert_name)
return alert.get('enabled', True) # default is True

def set_alert_enabled(self, alert_name: str, enabled: bool):
alert = self.get_alert_from_db(alert_name)
alert['enabled'] = enabled
self.ton.local.save()

def enable_alert(self, args):
if len(args) != 1:
raise Exception("Usage: enable_alert <alert_name>")
alert_name = args[0]
self.set_alert_enabled(alert_name, True)
color_print("enable_alert - {green}OK{endc}")

def disable_alert(self, args):
if len(args) != 1:
raise Exception("Usage: disable_alert <alert_name>")
alert_name = args[0]
self.set_alert_enabled(alert_name, False)
color_print("disable_alert - {green}OK{endc}")

def print_alerts(self, args):
table = [['Name', 'Enabled', 'Last sent']]
for alert_name in ALERTS:
alert = self.get_alert_from_db(alert_name)
table.append([alert_name, alert['enabled'], alert['sent']])
print_table(table)

def test_alert(self, args):
self.send_message('Test alert')

def check_db_usage(self):
if time.time() - self.last_db_check < 600:
return
self.last_db_check = time.time()
usage = self.ton.GetDbUsage()
if usage > 95:
self.send_alert("db_usage_95")
elif usage > 80:
self.send_alert("db_usage_80")

def check_validator_wallet_balance(self):
if not self.ton.using_validator():
return
validator_wallet = self.ton.GetValidatorWallet()
validator_account = self.ton.GetAccount(validator_wallet.addrB64)
if validator_account.balance < 10:
self.send_alert("low_wallet_balance", wallet=validator_wallet.addrB64, balance=validator_account.balance)

def check_efficiency(self):
if not self.ton.using_validator():
return
validator = self.validator_module.find_myself(self.ton.GetValidatorsList())
if validator is None or validator.efficiency is None:
return
config34 = self.ton.GetConfig34()
if (time.time() - config34.startWorkTime) / (config34.endWorkTime - config34.startWorkTime) < 0.8:
return # less than 80% of round passed
if validator.is_masterchain is False:
if validator.efficiency != 0:
return
if validator.efficiency < 90:
self.send_alert("low_efficiency", efficiency=validator.efficiency)

def check_validator_working(self):
validator_status = self.ton.GetValidatorStatus()
if not validator_status.is_working:
self.send_alert("service_down")

def check_sync(self):
validator_status = self.ton.GetValidatorStatus()
if validator_status.is_working and validator_status.out_of_sync >= 20:
self.send_alert("out_of_sync", sync=validator_status.out_of_sync)

def check_zero_blocks_created(self):
if not self.ton.using_validator():
return
ts = get_timestamp()
period = VALIDATION_PERIOD // 3 # 6h for mainnet, 40m for testnet
start, end = ts - period, ts - 60
config34 = self.ton.GetConfig34()
if start < config34.startWorkTime: # round started recently
return
validators = self.ton.GetValidatorsList(start=start, end=end)
validator = self.validator_module.find_myself(validators)
if validator is None or validator.blocks_created > 0:
return
self.send_alert("zero_block_created", hours=round(period // 3600, 1))

def check_slashed(self):
if not self.ton.using_validator():
return
c = self.validator_module.get_my_complaint()
if c is not None:
self.send_alert("validator_slashed", amount=int(c['suggestedFine']))

def check_adnl_connection_failed(self):
from modules.utilities import UtilitiesModule
utils_module = UtilitiesModule(self.ton, self.local)
ok, error = utils_module.check_adnl_connection()
if not ok:
self.send_alert("adnl_connection_failed")

def check_status(self):
if not self.ton.using_alert_bot():
return
if not self.inited:
self.init()

self.local.try_function(self.check_db_usage)
self.local.try_function(self.check_validator_wallet_balance)
self.local.try_function(self.check_efficiency) # todo: alert if validator is going to be slashed
self.local.try_function(self.check_validator_working)
self.local.try_function(self.check_zero_blocks_created)
self.local.try_function(self.check_sync)
self.local.try_function(self.check_slashed)
self.local.try_function(self.check_adnl_connection_failed)

def add_console_commands(self, console):
console.AddItem("enable_alert", self.enable_alert, self.local.translate("enable_alert_cmd"))
console.AddItem("disable_alert", self.disable_alert, self.local.translate("disable_alert_cmd"))
console.AddItem("list_alerts", self.print_alerts, self.local.translate("list_alerts_cmd"))
console.AddItem("test_alert", self.test_alert, self.local.translate("test_alert_cmd"))
50 changes: 48 additions & 2 deletions modules/utilities.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
import base64
import json
import os
import random
import subprocess
import time

import requests

from mypylib.mypylib import color_print, print_table, color_text, timeago, bcolors
from modules.module import MtcModule

Expand Down Expand Up @@ -335,6 +336,50 @@ def print_validator_list(self, args):
print_table(table)
# end define

def check_adnl_connection(self):
telemetry = self.ton.local.db.get("sendTelemetry", False)
check_adnl = self.ton.local.db.get("checkAdnl", telemetry)
if not check_adnl:
return True, ''
self.local.add_log('Checking ADNL connection to local node', 'info')
hosts = ['45.129.96.53', '5.154.181.153', '2.56.126.137', '91.194.11.68', '45.12.134.214', '138.124.184.27',
'103.106.3.171']
hosts = random.sample(hosts, k=3)
data = self.ton.get_local_adnl_data()
error = ''
ok = True
for host in hosts:
url = f'http://{host}/adnl_check'
try:
response = requests.post(url, json=data, timeout=5).json()
except Exception as e:
ok = False
error = f'{{red}}Failed to check ADNL connection to local node: {type(e)}: {e}{{endc}}'
continue
result = response.get("ok")
if result:
ok = True
break
if not result:
ok = False
error = f'{{red}}Failed to check ADNL connection to local node: {response.get("message")}{{endc}}'
return ok, error

def get_pool_data(self, args):
try:
pool_name = args[0]
except:
color_print("{red}Bad args. Usage:{endc} get_pool_data <pool-name | pool-addr>")
return
if self.ton.IsAddr(pool_name):
pool_addr = pool_name
else:
pool = self.ton.GetLocalPool(pool_name)
pool_addr = pool.addrB64
pool_data = self.ton.GetPoolData(pool_addr)
print(json.dumps(pool_data, indent=4))
# end define

def add_console_commands(self, console):
console.AddItem("vas", self.view_account_status, self.local.translate("vas_cmd"))
console.AddItem("vah", self.view_account_history, self.local.translate("vah_cmd"))
Expand All @@ -350,3 +395,4 @@ def add_console_commands(self, console):
console.AddItem("vl", self.print_validator_list, self.local.translate("vl_cmd"))
console.AddItem("cl", self.print_complaints_list, self.local.translate("cl_cmd"))

console.AddItem("get_pool_data", self.get_pool_data, self.local.translate("get_pool_data_cmd"))
Loading