merge mytonctrl2_dev into mytonctrl2_testnet

modules/liteserver.py

@@ -5,23 +5,8 @@
 
 class LiteserverModule(MtcModule):
 
-    description = 'For liteserver usage only - can\'t be used with validator.'
+    description = 'For liteserver usage only without validator.'
     default_value = False
 
-    def enable(self):
-        from mytoninstaller.mytoninstaller import set_node_argument
-        set_node_argument(self.local, ["--celldb-no-preload-all"])
-        data = psutil.virtual_memory()
-        ram = data.total / 2**30
-        if ram < 100:
-            set_node_argument(self.local, ["--celldb-cache-size", "1073741824"])
-
-    def disable(self):
-        from mytoninstaller.mytoninstaller import set_node_argument
-        from mytoninstaller.node_args import get_node_args
-        set_node_argument(self.local, ["--celldb-no-preload-all", "-d"])
-        if get_node_args()['--celldb-cache-size']:
-            set_node_argument(self.local, ["--celldb-cache-size", "-d"])
-
     def add_console_commands(self, console):
         ...

modules/nominator_pool.py

@@ -78,42 +78,6 @@ def activate_pool(self, args):
         self.do_activate_pool(pool)
         color_print("ActivatePool - {green}OK{endc}")
 
-    def do_deposit_to_pool(self, pool_addr, amount):
-        wallet = self.ton.GetValidatorWallet()
-        bocPath = self.ton.local.buffer.my_temp_dir + wallet.name + "validator-deposit-query.boc"
-        fiftScript = self.ton.contractsDir + "nominator-pool/func/validator-deposit.fif"
-        args = [fiftScript, bocPath]
-        result = self.ton.fift.Run(args)
-        resultFilePath = self.ton.SignBocWithWallet(wallet, bocPath, pool_addr, amount)
-        self.ton.SendFile(resultFilePath, wallet)
-
-    def deposit_to_pool(self, args):
-        try:
-            poll_addr = args[0]
-            amount = float(args[1])
-        except:
-            color_print("{red}Bad args. Usage:{endc} deposit_to_pool <pool-addr> <amount>")
-            return
-        self.do_deposit_to_pool(poll_addr, amount)
-        color_print("DepositToPool - {green}OK{endc}")
-
-    def do_withdraw_from_pool(self, pool_addr, amount):
-        pool_data = self.ton.GetPoolData(pool_addr)
-        if pool_data["state"] == 0:
-            self.ton.WithdrawFromPoolProcess(pool_addr, amount)
-        else:
-            self.ton.PendWithdrawFromPool(pool_addr, amount)
-
-    def withdraw_from_pool(self, args):
-        try:
-            pool_addr = args[0]
-            amount = float(args[1])
-        except:
-            color_print("{red}Bad args. Usage:{endc} withdraw_from_pool <pool-addr> <amount>")
-            return
-        self.do_withdraw_from_pool(pool_addr, amount)
-        color_print("WithdrawFromPool - {green}OK{endc}")
-
     def update_validator_set(self, args):
         try:
             pool_addr = args[0]
@@ -127,8 +91,4 @@ def update_validator_set(self, args):
     def add_console_commands(self, console):
         console.AddItem("new_pool", self.new_pool, self.local.translate("new_pool_cmd"))
         console.AddItem("activate_pool", self.activate_pool, self.local.translate("activate_pool_cmd"))
-        console.AddItem("deposit_to_pool", self.deposit_to_pool, self.local.translate("deposit_to_pool_cmd"))
-        console.AddItem("withdraw_from_pool", self.withdraw_from_pool, self.local.translate("withdraw_from_pool_cmd"))
         console.AddItem("update_validator_set", self.update_validator_set, self.local.translate("update_validator_set_cmd"))
-
-

modules/pool.py

@@ -58,7 +58,45 @@ def check_download_pool_contract_scripts(self):
         if not os.path.isdir(contract_path):
             self.ton.DownloadContract("https://github.com/ton-blockchain/nominator-pool")
 
+    def do_deposit_to_pool(self, pool_addr, amount):
+        wallet = self.ton.GetValidatorWallet()
+        bocPath = self.ton.local.buffer.my_temp_dir + wallet.name + "validator-deposit-query.boc"
+        fiftScript = self.ton.contractsDir + "nominator-pool/func/validator-deposit.fif"
+        args = [fiftScript, bocPath]
+        result = self.ton.fift.Run(args)
+        resultFilePath = self.ton.SignBocWithWallet(wallet, bocPath, pool_addr, amount)
+        self.ton.SendFile(resultFilePath, wallet)
+
+    def deposit_to_pool(self, args):
+        try:
+            poll_addr = args[0]
+            amount = float(args[1])
+        except:
+            color_print("{red}Bad args. Usage:{endc} deposit_to_pool <pool-addr> <amount>")
+            return
+        self.do_deposit_to_pool(poll_addr, amount)
+        color_print("DepositToPool - {green}OK{endc}")
+
+    def do_withdraw_from_pool(self, pool_addr, amount):
+        pool_data = self.ton.GetPoolData(pool_addr)
+        if pool_data["state"] == 0:
+            self.ton.WithdrawFromPoolProcess(pool_addr, amount)
+        else:
+            self.ton.PendWithdrawFromPool(pool_addr, amount)
+
+    def withdraw_from_pool(self, args):
+        try:
+            pool_addr = args[0]
+            amount = float(args[1])
+        except:
+            color_print("{red}Bad args. Usage:{endc} withdraw_from_pool <pool-addr> <amount>")
+            return
+        self.do_withdraw_from_pool(pool_addr, amount)
+        color_print("WithdrawFromPool - {green}OK{endc}")
+
     def add_console_commands(self, console):
         console.AddItem("pools_list", self.print_pools_list, self.local.translate("pools_list_cmd"))
         console.AddItem("delete_pool", self.delete_pool, self.local.translate("delete_pool_cmd"))
         console.AddItem("import_pool", self.import_pool, self.local.translate("import_pool_cmd"))
+        console.AddItem("deposit_to_pool", self.deposit_to_pool, self.local.translate("deposit_to_pool_cmd"))
+        console.AddItem("withdraw_from_pool", self.withdraw_from_pool, self.local.translate("withdraw_from_pool_cmd"))

mytoncore/functions.py

@@ -10,6 +10,7 @@
 import subprocess
 
 from mytoncore.mytoncore import MyTonCore
+from mytonctrl.utils import fix_git_config
 from mytoninstaller.config import GetConfig
 from mypylib.mypylib import (
     b2mb,
@@ -420,7 +421,9 @@ def Telemetry(local, ton):
 
     # Get git hashes
     gitHashes = dict()
-    gitHashes["mytonctrl"] = get_git_hash("/usr/src/mytonctrl")
+    mtc_path = "/usr/src/mytonctrl"
+    local.try_function(fix_git_config, args=[mtc_path])
+    gitHashes["mytonctrl"] = get_git_hash(mtc_path)
     gitHashes["validator"] = GetBinGitHash(
         "/usr/bin/ton/validator-engine/validator-engine")
     data["gitHashes"] = gitHashes

mytoncore/mytoncore.py

@@ -2280,6 +2280,9 @@ def GetComplaints(self, electionId=None, past=False):
 			complaints[chash] = item
 		#end for
 
+		# sort complaints by their creation time and hash
+		complaints = dict(sorted(complaints.items(), key=lambda item: (item[1]["createdTime"], item[0])))
+
 		# Set buffer
 		self.SetFunctionBuffer(bname, complaints)
 
@@ -3266,7 +3269,6 @@ def check_enable_mode(self, name):
 			if self.using_validator():
 				raise Exception(f'Cannot enable liteserver mode while validator mode is enabled. '
 								f'Use `disable_mode validator` first.')
-			MODES['liteserver'](self, self.local).enable()
 		if name == 'validator':
 			if self.using_liteserver():
 				raise Exception(f'Cannot enable validator mode while liteserver mode is enabled. '
@@ -3284,8 +3286,6 @@ def disable_mode(self, name):
 		current_modes = self.get_modes()
 		if name not in current_modes:
 			raise Exception(f'Unknown module name: {name}. Available modes: {", ".join(MODES)}')
-		if name == 'liteserver':
-			MODES['liteserver'](self, self.local).disable()
 		current_modes[name] = False
 		self.local.save()
 

mytonctrl/mytonctrl.py

@@ -39,8 +39,9 @@
 	GetSwapInfo,
 	GetBinGitHash,
 )
+from mytoncore.telemetry import is_host_virtual
 from mytonctrl.migrate import run_migrations
-from mytonctrl.utils import GetItemFromList, timestamp2utcdatetime
+from mytonctrl.utils import GetItemFromList, timestamp2utcdatetime, fix_git_config
 
 import sys, getopt, os
 
@@ -237,7 +238,7 @@ def check_installer_user(local):
 #end define
 
 
-def PreUp(local, ton):
+def PreUp(local: MyPyClass, ton: MyTonCore):
 	CheckMytonctrlUpdate(local)
 	check_installer_user(local)
 	check_vport(local, ton)
@@ -249,8 +250,10 @@ def PreUp(local, ton):
 
 def Installer(args):
 	# args = ["python3", "/usr/src/mytonctrl/mytoninstaller.py"]
-	args = ["python3", "-m", "mytoninstaller"]
-	subprocess.run(args)
+	cmd = ["python3", "-m", "mytoninstaller"]
+	if args:
+		cmd += ["-c", *args]
+	subprocess.run(cmd)
 #end define
 
 
@@ -290,21 +293,6 @@ def check_vport(local, ton):
 #end define
 
 
-def fix_git_config(git_path: str):
-	args = ["git", "status"]
-	try:
-		process = subprocess.run(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=git_path, timeout=3)
-		err = process.stderr.decode("utf-8")
-	except Exception as e:
-		err = str(e)
-	if err:
-		if 'git config --global --add safe.directory' in err:
-			args = ["git", "config", "--global", "--add", "safe.directory", git_path]
-			subprocess.run(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=3)
-		else:
-			raise Exception(f'Failed to check git status: {err}')
-#end define
-
 def check_git(input_args, default_repo, text):
 	src_dir = "/usr/src"
 	git_path = f"{src_dir}/{default_repo}"
@@ -467,8 +455,34 @@ def check_disk_usage(local, ton):
 		print_warning(local, "disk_usage_warning")
 #end define
 
+def check_sync(local, ton):
+	validator_status = ton.GetValidatorStatus()
+	if not validator_status.is_working or validator_status.out_of_sync >= 20:
+		print_warning(local, "sync_warning")
+#end define
+
+def check_validator_balance(local, ton):
+	if ton.using_validator():
+		validator_wallet = ton.GetValidatorWallet()
+		validator_account = local.try_function(ton.GetAccount, args=[validator_wallet.addrB64])
+		if validator_account is None:
+			local.add_log(f"Failed to check validator wallet balance", "warning")
+			return
+		if validator_account.balance < 100:
+			print_warning(local, "validator_balance_warning")
+#end define
+
+def check_vps(local, ton):
+	if ton.using_validator():
+		data = local.try_function(is_host_virtual)
+		if data and data["virtual"]:
+			color_print(f"Virtualization detected: {data['product_name']}")
+
 def warnings(local, ton):
 	check_disk_usage(local, ton)
+	check_sync(local, ton)
+	check_validator_balance(local, ton)
+	check_vps(local, ton)
 #end define
 
 def CheckTonUpdate(local):
@@ -708,6 +722,8 @@ def PrintLocalStatus(local, adnlAddr, validatorIndex, validatorEfficiency, valid
 	validatorBinGitPath = "/usr/bin/ton/validator-engine/validator-engine"
 	mtcGitHash = get_git_hash(mtcGitPath, short=True)
 	validatorGitHash = GetBinGitHash(validatorBinGitPath, short=True)
+	fix_git_config(mtcGitPath)
+	fix_git_config(validatorGitPath)
 	mtcGitBranch = get_git_branch(mtcGitPath)
 	validatorGitBranch = get_git_branch(validatorGitPath)
 	mtcGitHash_text = bcolors.yellow_text(mtcGitHash)

mytonctrl/resources/translate.json

@@ -419,6 +419,26 @@
 		"ru": "{green}Доступно обновление TON. {red}Пожалуйста, обновите его с помощью команды `upgrade`.{endc}",
 		"zh_TW": "{green}TON 有可用更新. {red}請使用 `upgrade` 命令進行更新.{endc}"
 	},
+	"disk_usage_warning": {
+		"en": "{red} Disk is almost full, clean the TON database immediately: https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming {endc}",
+		"ru": "{red} Диск почти заполнен, немедленно очистите базу данных TON: https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming {endc}",
+		"zh_TW": "{red} 磁盤幾乎滿了，立即清理 TON 數據庫: https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming {endc}"
+	},
+	"sync_warning": {
+		"en": "{red} Node is out of sync. The displayed status is incomplete. {endc}",
+		"ru": "{red} Нода не синхронизирована с сетью. Отображаемый статус не полный. {endc}",
+		"zh_TW": "{red} 节点不与网络同步。顯示的狀態不完整。 {endc}"
+	},
+	"validator_balance_warning": {
+		"en": "{red} Validator wallet balance is low. {endc}",
+		"ru": "{red} Баланс кошелька валидатора низкий. {endc}",
+		"zh_TW": "{red} 驗證者錢包餘額不足。 {endc}"
+	},
+	"vps_warning": {
+		"en": "{red} Validator is running on a VPS. Use a dedicated server for better performance. {endc}",
+		"ru": "{red} Валидатор работает на VPS. Используйте выделенный сервер (дедик) для лучшей производительности. {endc}",
+		"zh_TW": "{red} 驗證者在 VPS 上運行。使用專用服務器以獲得更好的性能。 {endc}"
+	},
 	"vport_error": {
 		"en": "{red}Error - UDP port of the validator is not accessible from the outside.{endc}",
 		"ru": "{red}Ошибка - UDP порт валидатора недоступен извне.{endc}",

mytonctrl/utils.py

@@ -1,3 +1,4 @@
+import subprocess
 import time
 
 
@@ -12,3 +13,19 @@ def GetItemFromList(data, index):
         return data[index]
     except:
         pass
+
+
+def fix_git_config(git_path: str):
+    args = ["git", "status"]
+    try:
+        process = subprocess.run(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=git_path, timeout=3)
+        err = process.stderr.decode("utf-8")
+    except Exception as e:
+        err = str(e)
+    if err:
+        if 'git config --global --add safe.directory' in err:
+            args = ["git", "config", "--global", "--add", "safe.directory", git_path]
+            subprocess.run(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=3)
+        else:
+            raise Exception(f'Failed to check git status: {err}')
+#end define

mytoninstaller/mytoninstaller.py

@@ -238,12 +238,29 @@ def Event(local, name):
 #end define
 
 
-def General(local):
+def Command(local, args, console):
+	cmd = args[0]
+	args = args[1:]
+	for item in console.menu_items:
+		if cmd == item.cmd:
+			console._try(item.func, args)
+			print()
+			local.exit()
+	print(console.unknown_cmd)
+	local.exit()
+#end define
+
+
+def General(local, console):
 	if "-u" in sys.argv:
 		ux = sys.argv.index("-u")
 		user = sys.argv[ux+1]
 		local.buffer.user = user
 		Refresh(local)
+	if "-c" in sys.argv:
+		cx = sys.argv.index("-c")
+		args = sys.argv[cx+1:]
+		Command(local, args, console)
 	if "-e" in sys.argv:
 		ex = sys.argv.index("-e")
 		name = sys.argv[ex+1]
@@ -282,7 +299,7 @@ def mytoninstaller():
 
 	Init(local, console)
 	if len(sys.argv) > 1:
-		General(local)
+		General(local, console)
 	else:
 		console.Run()
 	local.exit()

mytoninstaller/settings.py

@@ -34,6 +34,7 @@ def FirstNodeSettings(local):
 	validatorAppPath = local.buffer.validator_app_path
 	globalConfigPath = local.buffer.global_config_path
 	vconfig_path = local.buffer.vconfig_path
+	archive_ttl = 2592000 if local.buffer.mode == 'liteserver' else 86400
 
 	# Проверить конфигурацию
 	if os.path.isfile(vconfig_path):
@@ -57,8 +58,7 @@ def FirstNodeSettings(local):
 
 	# Прописать автозагрузку
 	cpus = psutil.cpu_count() - 1
-	cmd = "{validatorAppPath} --threads {cpus} --daemonize --global-config {globalConfigPath} --db {ton_db_dir} --logname {tonLogPath} --state-ttl 604800 --verbosity 1"
-	cmd = cmd.format(validatorAppPath=validatorAppPath, globalConfigPath=globalConfigPath, ton_db_dir=ton_db_dir, tonLogPath=tonLogPath, cpus=cpus)
+	cmd = f"{validatorAppPath} --threads {cpus} --daemonize --global-config {globalConfigPath} --db {ton_db_dir} --logname {tonLogPath} --state-ttl 604800 --archive-ttl {archive_ttl} --verbosity 1"
 	add2systemd(name="validator", user=vuser, start=cmd) # post="/usr/bin/python3 /usr/src/mytonctrl/mytoncore.py -e \"validator down\""
 
 	# Получить внешний ip адрес
@@ -891,6 +891,10 @@ def CreateSymlinks(local):
 
 
 def EnableMode(local):
+	args = ["python3", "-m", "mytoncore", "-e"]
 	if local.buffer.mode == 'liteserver':
-		args = ["python3", "-m", "mytoncore", "-e", "enable_liteserver_mode"]
-		subprocess.run(args)
+		args.append("enable_liteserver_mode")
+	else:
+		return
+	args = ["su", "-l", local.buffer.user, "-c", ' '.join(args)]
+	subprocess.run(args)