From aaa758ddf0a66a6b79643cd9f6c9a1b94a07958b Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 2 Apr 2024 21:12:51 +0800 Subject: [PATCH 01/40] add custom overlays --- custom_overlays.py | 124 +++++++++++++++++++++++++++++++++++++++++++++ mytoncore.py | 2 + mytonctrl.py | 5 ++ 3 files changed, 131 insertions(+) create mode 100644 custom_overlays.py diff --git a/custom_overlays.py b/custom_overlays.py new file mode 100644 index 00000000..c6fbcdb3 --- /dev/null +++ b/custom_overlays.py @@ -0,0 +1,124 @@ +import json + +from mypylib.mypylib import color_print +from mytoncore import local as mytoncore_local, hex2base64, MyTonCore +from mytonctrl import ton + + +def parse_config(name: str, config: dict, vset: list = None): + """ + Converts config to validator-console friendly format + :param name: custom overlay name + :param config: config + :param vset: list of validators adnl addresses, can be None if `@validators` not in config + :return: + """ + result = { + "name": name, + "nodes": [] + } + for k, v in config.items(): + result["nodes"].append({ + "adnl_id": hex2base64(k), + "msg_sender": v["msg_sender"], + "msg_sender_priority": v["msg_sender_priority"], + }) + if k == '@validators' and v: + if vset is None: + raise Exception("Validators set is not defined but @validators is in config") + for v_adnl in vset: + result["nodes"].append({ + "adnl_id": hex2base64(v_adnl), + "msg_sender": False, + }) + return result + + +def add_custom_overlay(args): + if len(args) != 2: + color_print("{red}Bad args. Usage:{endc} add_custom_overlay ") + return + path = args[1] + with open(path, 'r') as f: + config = json.load(f) + mytoncore_local.db['custom_overlays'][args[0]] = config + + +def list_custom_overlays(args): + for k, v in mytoncore_local.db['custom_overlays'].items(): + color_print(f"Custom overlay {{bold}}{k}{{endc}}:") + print(json.dumps(v, indent=4)) + + +def delete_custom_overlay(args): + if len(args) != 1: + color_print("{red}Bad args. Usage:{endc} delete_custom_overlay ") + return + del mytoncore_local.db['custom_overlays'][args[0]] + + +def delete_custom_overlay_from_vc(name: str): + result = ton.validatorConsole.Run(f"delcustomoverlay {name}") + return 'success' in result + + +def add_custom_overlay_to_vc(config: dict): + mytoncore_local.add_log(f"Adding custom overlay {config['name']}", "debug") + path = ton.tempDir + f'/custom_overlay_{config["name"]}.json' + with open(path, 'w') as f: + json.dump(config, f) + result = ton.validatorConsole.Run(f"addcustomoverlay {path}") + return 'success' in result + + +def deploy_custom_overlays(): + result = ton.validatorConsole.Run("showcustomoverlays") + if 'unknown command' in result: + return # node old version + names = [] + for line in result.split('\n'): + if line.startswith('Overlay'): + names.append(line.split(' ')[1].replace('"', '')) + + config34 = ton.GetConfig34() + current_el_id = config34['startWorkTime'] + current_vset = [i["adnlAddr"] for i in config34['validators']] + + config36 = ton.GetConfig36() + next_el_id = config36['startWorkTime'] if config36['validators'] else 0 + next_vset = [i["adnlAddr"] for i in config36['validators']] + + for name in names: + # check that overlay still exists in mtc db + pure_name = name + suffix = name.split('_')[-1] + if suffix.startswith('elid') and suffix.split('elid')[-1].isdigit(): # probably election id + pure_name = '_'.join(name.split('_')[:-1]) + el_id = int(suffix.split('elid')[-1].isdigit()) + if el_id not in (current_el_id, next_el_id): + mytoncore_local.add_log(f"Overlay {name} is not in current or next election, deleting", "debug") + delete_custom_overlay_from_vc(name) # delete overlay if election id is not in current or next election + continue + + if pure_name not in mytoncore_local.db['custom_overlays']: + mytoncore_local.add_log(f"Overlay {name} is not in mtc db, deleting", "debug") + delete_custom_overlay_from_vc(name) # delete overlay if it's not in mtc db + + for name, config in mytoncore_local.db['custom_overlays'].items(): + if name in names: + continue + if '@validators' in config: + new_name = name + '_elid' + str(current_el_id) + if new_name not in names: + node_config = parse_config(new_name, config, current_vset) + add_custom_overlay_to_vc(node_config) + + if next_el_id != 0: + new_name = name + '_elid' + str(next_el_id) + if new_name not in names: + node_config = parse_config(new_name, config, next_vset) + add_custom_overlay_to_vc(node_config) + else: + + node_config = parse_config(name, config) + add_custom_overlay_to_vc(node_config) diff --git a/mytoncore.py b/mytoncore.py index 82572b5b..cf19e348 100755 --- a/mytoncore.py +++ b/mytoncore.py @@ -8,6 +8,7 @@ import requests import re from mypylib.mypylib import * +from custom_overlays import deploy_custom_overlays local = MyPyClass(__file__) @@ -4297,6 +4298,7 @@ def General(): local.start_cycle(Telemetry, sec=60, args=(ton, )) local.start_cycle(OverlayTelemetry, sec=7200, args=(ton, )) local.start_cycle(ScanLiteServers, sec=60, args=(ton,)) + local.start_cycle(deploy_custom_overlays, sec=60) thr_sleep() #end define diff --git a/mytonctrl.py b/mytonctrl.py index 09a91834..6549afcd 100755 --- a/mytonctrl.py +++ b/mytonctrl.py @@ -4,6 +4,7 @@ from mypylib.mypylib import * from mypyconsole.mypyconsole import * from mytoncore import * +from custom_overlays import add_custom_overlay, list_custom_overlays, delete_custom_overlay import sys, getopt, os local = MyPyClass(__file__) @@ -25,6 +26,10 @@ def Init(argv): console.AddItem("seqno", Seqno, local.translate("seqno_cmd")) console.AddItem("getconfig", GetConfig, local.translate("getconfig_cmd")) + console.AddItem("add_custom_overlay", add_custom_overlay, local.translate("add_custom_overlay_cmd")) + console.AddItem("list_custom_overlays", list_custom_overlays, local.translate("list_custom_overlays_cmd")) + console.AddItem("delete_custom_overlay", delete_custom_overlay, local.translate("delete_custom_overlay_cmd")) + console.AddItem("nw", CreatNewWallet, local.translate("nw_cmd")) console.AddItem("aw", ActivateWallet, local.translate("aw_cmd")) console.AddItem("wl", PrintWalletsList, local.translate("wl_cmd")) From fab1484971a95ce02aae4e109be9c7e1a212a4e5 Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 2 Apr 2024 21:19:30 +0800 Subject: [PATCH 02/40] fix config parsing --- custom_overlays.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/custom_overlays.py b/custom_overlays.py index c6fbcdb3..960629cb 100644 --- a/custom_overlays.py +++ b/custom_overlays.py @@ -1,7 +1,7 @@ import json from mypylib.mypylib import color_print -from mytoncore import local as mytoncore_local, hex2base64, MyTonCore +from mytoncore import local as mytoncore_local, hex2base64 from mytonctrl import ton @@ -18,11 +18,6 @@ def parse_config(name: str, config: dict, vset: list = None): "nodes": [] } for k, v in config.items(): - result["nodes"].append({ - "adnl_id": hex2base64(k), - "msg_sender": v["msg_sender"], - "msg_sender_priority": v["msg_sender_priority"], - }) if k == '@validators' and v: if vset is None: raise Exception("Validators set is not defined but @validators is in config") @@ -31,7 +26,14 @@ def parse_config(name: str, config: dict, vset: list = None): "adnl_id": hex2base64(v_adnl), "msg_sender": False, }) - return result + else: + result["nodes"].append({ + "adnl_id": hex2base64(k), + "msg_sender": v["msg_sender"], + }) + if v["msg_sender"]: + result["nodes"][-1]["msg_sender_priority"] = v["msg_sender_priority"] + return result def add_custom_overlay(args): From e935a4d1fcfbc34a094126867688a5123cef3331 Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 2 Apr 2024 21:21:57 +0800 Subject: [PATCH 03/40] fix circular import fix circular import --- mytoncore.py | 2 +- mytonctrl.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mytoncore.py b/mytoncore.py index cf19e348..b8971427 100755 --- a/mytoncore.py +++ b/mytoncore.py @@ -8,7 +8,6 @@ import requests import re from mypylib.mypylib import * -from custom_overlays import deploy_custom_overlays local = MyPyClass(__file__) @@ -4298,6 +4297,7 @@ def General(): local.start_cycle(Telemetry, sec=60, args=(ton, )) local.start_cycle(OverlayTelemetry, sec=7200, args=(ton, )) local.start_cycle(ScanLiteServers, sec=60, args=(ton,)) + from custom_overlays import deploy_custom_overlays local.start_cycle(deploy_custom_overlays, sec=60) thr_sleep() #end define diff --git a/mytonctrl.py b/mytonctrl.py index 6549afcd..f32cc0ef 100755 --- a/mytonctrl.py +++ b/mytonctrl.py @@ -4,7 +4,6 @@ from mypylib.mypylib import * from mypyconsole.mypyconsole import * from mytoncore import * -from custom_overlays import add_custom_overlay, list_custom_overlays, delete_custom_overlay import sys, getopt, os local = MyPyClass(__file__) @@ -26,6 +25,7 @@ def Init(argv): console.AddItem("seqno", Seqno, local.translate("seqno_cmd")) console.AddItem("getconfig", GetConfig, local.translate("getconfig_cmd")) + from custom_overlays import add_custom_overlay, list_custom_overlays, delete_custom_overlay console.AddItem("add_custom_overlay", add_custom_overlay, local.translate("add_custom_overlay_cmd")) console.AddItem("list_custom_overlays", list_custom_overlays, local.translate("list_custom_overlays_cmd")) console.AddItem("delete_custom_overlay", delete_custom_overlay, local.translate("delete_custom_overlay_cmd")) From d0b2f5917b52777eed98625f46f1ccd5ba9ef0b1 Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 2 Apr 2024 21:28:01 +0800 Subject: [PATCH 04/40] bugfix --- custom_overlays.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/custom_overlays.py b/custom_overlays.py index 960629cb..7e9dca40 100644 --- a/custom_overlays.py +++ b/custom_overlays.py @@ -43,6 +43,8 @@ def add_custom_overlay(args): path = args[1] with open(path, 'r') as f: config = json.load(f) + if 'custom_overlays' not in mytoncore_local.db: + mytoncore_local.db['custom_overlays'] = {} mytoncore_local.db['custom_overlays'][args[0]] = config From 3863ef0b5d622eca9d63b473b38ba95ab8002a12 Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 2 Apr 2024 21:36:53 +0800 Subject: [PATCH 05/40] rm mytoncore_local from custom_overlays --- custom_overlays.py | 21 +++++++++------------ mytoncore.py | 17 +++++++++++++++++ 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/custom_overlays.py b/custom_overlays.py index 7e9dca40..105f0124 100644 --- a/custom_overlays.py +++ b/custom_overlays.py @@ -1,7 +1,7 @@ import json from mypylib.mypylib import color_print -from mytoncore import local as mytoncore_local, hex2base64 +from mytoncore import hex2base64 from mytonctrl import ton @@ -43,13 +43,11 @@ def add_custom_overlay(args): path = args[1] with open(path, 'r') as f: config = json.load(f) - if 'custom_overlays' not in mytoncore_local.db: - mytoncore_local.db['custom_overlays'] = {} - mytoncore_local.db['custom_overlays'][args[0]] = config + ton.set_custom_overlay(args[0], config) def list_custom_overlays(args): - for k, v in mytoncore_local.db['custom_overlays'].items(): + for k, v in ton.get_custom_overlays().items(): color_print(f"Custom overlay {{bold}}{k}{{endc}}:") print(json.dumps(v, indent=4)) @@ -58,7 +56,7 @@ def delete_custom_overlay(args): if len(args) != 1: color_print("{red}Bad args. Usage:{endc} delete_custom_overlay ") return - del mytoncore_local.db['custom_overlays'][args[0]] + ton.delete_custom_overlay(args[0]) def delete_custom_overlay_from_vc(name: str): @@ -67,7 +65,7 @@ def delete_custom_overlay_from_vc(name: str): def add_custom_overlay_to_vc(config: dict): - mytoncore_local.add_log(f"Adding custom overlay {config['name']}", "debug") + ton.add_log(f"Adding custom overlay {config['name']}", "debug") path = ton.tempDir + f'/custom_overlay_{config["name"]}.json' with open(path, 'w') as f: json.dump(config, f) @@ -100,15 +98,15 @@ def deploy_custom_overlays(): pure_name = '_'.join(name.split('_')[:-1]) el_id = int(suffix.split('elid')[-1].isdigit()) if el_id not in (current_el_id, next_el_id): - mytoncore_local.add_log(f"Overlay {name} is not in current or next election, deleting", "debug") + ton.add_log(f"Overlay {name} is not in current or next election, deleting", "debug") delete_custom_overlay_from_vc(name) # delete overlay if election id is not in current or next election continue - if pure_name not in mytoncore_local.db['custom_overlays']: - mytoncore_local.add_log(f"Overlay {name} is not in mtc db, deleting", "debug") + if pure_name not in ton.get_custom_overlays(): + ton.add_log(f"Overlay {name} is not in mtc db, deleting", "debug") delete_custom_overlay_from_vc(name) # delete overlay if it's not in mtc db - for name, config in mytoncore_local.db['custom_overlays'].items(): + for name, config in ton.get_custom_overlays().items(): if name in names: continue if '@validators' in config: @@ -123,6 +121,5 @@ def deploy_custom_overlays(): node_config = parse_config(new_name, config, next_vset) add_custom_overlay_to_vc(node_config) else: - node_config = parse_config(name, config) add_custom_overlay_to_vc(node_config) diff --git a/mytoncore.py b/mytoncore.py index b8971427..3de0304a 100755 --- a/mytoncore.py +++ b/mytoncore.py @@ -3646,6 +3646,23 @@ def GetPoolData(self, addrB64): return poolData #end define + def get_custom_overlays(self): + if 'custom_overlays' not in local.db: + local.db['custom_overlays'] = {} + return local.db['custom_overlays'] + + def set_custom_overlay(self, name: str, config: dict): + overlays = self.get_custom_overlays() + overlays[name] = config + local.save() + + def delete_custom_overlay(self, name: str): + del local.db['custom_overlays'][name] + local.save() + + def add_log(self, text, level="info"): + local.add_log(text, level) + def GetNetworkName(self): mainnetValidatorsElectedFor = 65536 mainnetZerostateRootHash = "x55B13F6D0E1D0C34C9C2160F6F918E92D82BF9DDCF8DE2E4C94A3FDF39D15446" From 8d584bc7e26b0f9bea9f5b85a67e26b1bf7d2df8 Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 2 Apr 2024 21:59:05 +0800 Subject: [PATCH 06/40] improves --- custom_overlays.py | 16 +++++++++++++--- mytoncore.py | 3 --- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/custom_overlays.py b/custom_overlays.py index 105f0124..8f0396d6 100644 --- a/custom_overlays.py +++ b/custom_overlays.py @@ -5,6 +5,11 @@ from mytonctrl import ton +def add_log(message: str, level: str = 'info'): + from mytoncore import local + local.add_log(message, level) + + def parse_config(name: str, config: dict, vset: list = None): """ Converts config to validator-console friendly format @@ -44,9 +49,13 @@ def add_custom_overlay(args): with open(path, 'r') as f: config = json.load(f) ton.set_custom_overlay(args[0], config) + color_print("add_custom_overlay - {green}OK{endc}") def list_custom_overlays(args): + if not ton.get_custom_overlays(): + color_print("{red}No custom overlays{endc}") + return for k, v in ton.get_custom_overlays().items(): color_print(f"Custom overlay {{bold}}{k}{{endc}}:") print(json.dumps(v, indent=4)) @@ -57,6 +66,7 @@ def delete_custom_overlay(args): color_print("{red}Bad args. Usage:{endc} delete_custom_overlay ") return ton.delete_custom_overlay(args[0]) + color_print("delete_custom_overlay - {green}OK{endc}") def delete_custom_overlay_from_vc(name: str): @@ -65,7 +75,7 @@ def delete_custom_overlay_from_vc(name: str): def add_custom_overlay_to_vc(config: dict): - ton.add_log(f"Adding custom overlay {config['name']}", "debug") + add_log(f"Adding custom overlay {config['name']}", "debug") path = ton.tempDir + f'/custom_overlay_{config["name"]}.json' with open(path, 'w') as f: json.dump(config, f) @@ -98,12 +108,12 @@ def deploy_custom_overlays(): pure_name = '_'.join(name.split('_')[:-1]) el_id = int(suffix.split('elid')[-1].isdigit()) if el_id not in (current_el_id, next_el_id): - ton.add_log(f"Overlay {name} is not in current or next election, deleting", "debug") + add_log(f"Overlay {name} is not in current or next election, deleting", "debug") delete_custom_overlay_from_vc(name) # delete overlay if election id is not in current or next election continue if pure_name not in ton.get_custom_overlays(): - ton.add_log(f"Overlay {name} is not in mtc db, deleting", "debug") + add_log(f"Overlay {name} is not in mtc db, deleting", "debug") delete_custom_overlay_from_vc(name) # delete overlay if it's not in mtc db for name, config in ton.get_custom_overlays().items(): diff --git a/mytoncore.py b/mytoncore.py index 3de0304a..532b53df 100755 --- a/mytoncore.py +++ b/mytoncore.py @@ -3660,9 +3660,6 @@ def delete_custom_overlay(self, name: str): del local.db['custom_overlays'][name] local.save() - def add_log(self, text, level="info"): - local.add_log(text, level) - def GetNetworkName(self): mainnetValidatorsElectedFor = 65536 mainnetZerostateRootHash = "x55B13F6D0E1D0C34C9C2160F6F918E92D82BF9DDCF8DE2E4C94A3FDF39D15446" From 1b9ff1c16273c92119d840a5c1469aaf98780d14 Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 2 Apr 2024 22:10:38 +0800 Subject: [PATCH 07/40] rm global variables from custom_overlays --- custom_overlays.py | 40 +++++++++++++++++++++++----------------- mytoncore.py | 4 ++-- mytonctrl.py | 3 ++- 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/custom_overlays.py b/custom_overlays.py index 8f0396d6..94e31342 100644 --- a/custom_overlays.py +++ b/custom_overlays.py @@ -1,13 +1,14 @@ +import base64 import json from mypylib.mypylib import color_print -from mytoncore import hex2base64 -from mytonctrl import ton -def add_log(message: str, level: str = 'info'): - from mytoncore import local - local.add_log(message, level) +def hex2base64(h): + b = bytes.fromhex(h) + b64 = base64.b64encode(b) + s = b64.decode("utf-8") + return s def parse_config(name: str, config: dict, vset: list = None): @@ -42,6 +43,7 @@ def parse_config(name: str, config: dict, vset: list = None): def add_custom_overlay(args): + from mytonctrl import ton if len(args) != 2: color_print("{red}Bad args. Usage:{endc} add_custom_overlay ") return @@ -53,6 +55,7 @@ def add_custom_overlay(args): def list_custom_overlays(args): + from mytonctrl import ton if not ton.get_custom_overlays(): color_print("{red}No custom overlays{endc}") return @@ -62,6 +65,7 @@ def list_custom_overlays(args): def delete_custom_overlay(args): + from mytonctrl import ton if len(args) != 1: color_print("{red}Bad args. Usage:{endc} delete_custom_overlay ") return @@ -69,13 +73,12 @@ def delete_custom_overlay(args): color_print("delete_custom_overlay - {green}OK{endc}") -def delete_custom_overlay_from_vc(name: str): +def delete_custom_overlay_from_vc(ton, name: str): result = ton.validatorConsole.Run(f"delcustomoverlay {name}") return 'success' in result -def add_custom_overlay_to_vc(config: dict): - add_log(f"Adding custom overlay {config['name']}", "debug") +def add_custom_overlay_to_vc(ton, config: dict): path = ton.tempDir + f'/custom_overlay_{config["name"]}.json' with open(path, 'w') as f: json.dump(config, f) @@ -83,7 +86,7 @@ def add_custom_overlay_to_vc(config: dict): return 'success' in result -def deploy_custom_overlays(): +def deploy_custom_overlays(local, ton): result = ton.validatorConsole.Run("showcustomoverlays") if 'unknown command' in result: return # node old version @@ -108,13 +111,13 @@ def deploy_custom_overlays(): pure_name = '_'.join(name.split('_')[:-1]) el_id = int(suffix.split('elid')[-1].isdigit()) if el_id not in (current_el_id, next_el_id): - add_log(f"Overlay {name} is not in current or next election, deleting", "debug") - delete_custom_overlay_from_vc(name) # delete overlay if election id is not in current or next election + local.add_log(f"Overlay {name} is not in current or next election, deleting", "debug") + delete_custom_overlay_from_vc(ton, name) # delete overlay if election id is not in current or next election continue - if pure_name not in ton.get_custom_overlays(): - add_log(f"Overlay {name} is not in mtc db, deleting", "debug") - delete_custom_overlay_from_vc(name) # delete overlay if it's not in mtc db + elif pure_name not in ton.get_custom_overlays(): + local.add_log(f"Overlay {name} is not in mtc db, deleting", "debug") + delete_custom_overlay_from_vc(ton, name) # delete overlay if it's not in mtc db for name, config in ton.get_custom_overlays().items(): if name in names: @@ -123,13 +126,16 @@ def deploy_custom_overlays(): new_name = name + '_elid' + str(current_el_id) if new_name not in names: node_config = parse_config(new_name, config, current_vset) - add_custom_overlay_to_vc(node_config) + local.add_log(f"Adding custom overlay {config['name']}", "debug") + add_custom_overlay_to_vc(ton, node_config) if next_el_id != 0: new_name = name + '_elid' + str(next_el_id) if new_name not in names: node_config = parse_config(new_name, config, next_vset) - add_custom_overlay_to_vc(node_config) + local.add_log(f"Adding custom overlay {config['name']}", "debug") + add_custom_overlay_to_vc(ton, node_config) else: node_config = parse_config(name, config) - add_custom_overlay_to_vc(node_config) + local.add_log(f"Adding custom overlay {config['name']}", "debug") + add_custom_overlay_to_vc(ton, node_config) diff --git a/mytoncore.py b/mytoncore.py index 532b53df..bc7ee518 100755 --- a/mytoncore.py +++ b/mytoncore.py @@ -8,6 +8,7 @@ import requests import re from mypylib.mypylib import * +from custom_overlays import deploy_custom_overlays local = MyPyClass(__file__) @@ -4311,8 +4312,7 @@ def General(): local.start_cycle(Telemetry, sec=60, args=(ton, )) local.start_cycle(OverlayTelemetry, sec=7200, args=(ton, )) local.start_cycle(ScanLiteServers, sec=60, args=(ton,)) - from custom_overlays import deploy_custom_overlays - local.start_cycle(deploy_custom_overlays, sec=60) + local.start_cycle(deploy_custom_overlays, sec=60, args=(local, ton,)) thr_sleep() #end define diff --git a/mytonctrl.py b/mytonctrl.py index f32cc0ef..c05bf9f5 100755 --- a/mytonctrl.py +++ b/mytonctrl.py @@ -3,6 +3,8 @@ from mypylib.mypylib import * from mypyconsole.mypyconsole import * +from custom_overlays import add_custom_overlay, list_custom_overlays, delete_custom_overlay + from mytoncore import * import sys, getopt, os @@ -25,7 +27,6 @@ def Init(argv): console.AddItem("seqno", Seqno, local.translate("seqno_cmd")) console.AddItem("getconfig", GetConfig, local.translate("getconfig_cmd")) - from custom_overlays import add_custom_overlay, list_custom_overlays, delete_custom_overlay console.AddItem("add_custom_overlay", add_custom_overlay, local.translate("add_custom_overlay_cmd")) console.AddItem("list_custom_overlays", list_custom_overlays, local.translate("list_custom_overlays_cmd")) console.AddItem("delete_custom_overlay", delete_custom_overlay, local.translate("delete_custom_overlay_cmd")) From b0c15b362bc6a6809f37057b16e2ccdaf9046042 Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 2 Apr 2024 22:18:15 +0800 Subject: [PATCH 08/40] bugfix bugfix bugfix --- custom_overlays.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/custom_overlays.py b/custom_overlays.py index 94e31342..9a51b00f 100644 --- a/custom_overlays.py +++ b/custom_overlays.py @@ -93,7 +93,7 @@ def deploy_custom_overlays(local, ton): names = [] for line in result.split('\n'): if line.startswith('Overlay'): - names.append(line.split(' ')[1].replace('"', '')) + names.append(line.split(' ')[1].replace('"', '').replace(':', '')) config34 = ton.GetConfig34() current_el_id = config34['startWorkTime'] @@ -109,14 +109,14 @@ def deploy_custom_overlays(local, ton): suffix = name.split('_')[-1] if suffix.startswith('elid') and suffix.split('elid')[-1].isdigit(): # probably election id pure_name = '_'.join(name.split('_')[:-1]) - el_id = int(suffix.split('elid')[-1].isdigit()) + el_id = int(suffix.split('elid')[-1]) if el_id not in (current_el_id, next_el_id): local.add_log(f"Overlay {name} is not in current or next election, deleting", "debug") delete_custom_overlay_from_vc(ton, name) # delete overlay if election id is not in current or next election continue - elif pure_name not in ton.get_custom_overlays(): - local.add_log(f"Overlay {name} is not in mtc db, deleting", "debug") + if pure_name not in ton.get_custom_overlays(): + local.add_log(f"Overlay {name} ({pure_name}) is not in mtc db, deleting", "debug") delete_custom_overlay_from_vc(ton, name) # delete overlay if it's not in mtc db for name, config in ton.get_custom_overlays().items(): @@ -126,16 +126,16 @@ def deploy_custom_overlays(local, ton): new_name = name + '_elid' + str(current_el_id) if new_name not in names: node_config = parse_config(new_name, config, current_vset) - local.add_log(f"Adding custom overlay {config['name']}", "debug") + local.add_log(f"Adding custom overlay {name}", "debug") add_custom_overlay_to_vc(ton, node_config) if next_el_id != 0: new_name = name + '_elid' + str(next_el_id) if new_name not in names: node_config = parse_config(new_name, config, next_vset) - local.add_log(f"Adding custom overlay {config['name']}", "debug") + local.add_log(f"Adding custom overlay {name}", "debug") add_custom_overlay_to_vc(ton, node_config) else: node_config = parse_config(name, config) - local.add_log(f"Adding custom overlay {config['name']}", "debug") + local.add_log(f"Adding custom overlay {name}", "debug") add_custom_overlay_to_vc(ton, node_config) From cb8ab7df4b21e223869b6129fb6d6288e294c766 Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 2 Apr 2024 22:33:51 +0800 Subject: [PATCH 09/40] logging improve --- custom_overlays.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/custom_overlays.py b/custom_overlays.py index 9a51b00f..64dcbdcd 100644 --- a/custom_overlays.py +++ b/custom_overlays.py @@ -126,14 +126,14 @@ def deploy_custom_overlays(local, ton): new_name = name + '_elid' + str(current_el_id) if new_name not in names: node_config = parse_config(new_name, config, current_vset) - local.add_log(f"Adding custom overlay {name}", "debug") + local.add_log(f"Adding custom overlay {new_name}", "debug") add_custom_overlay_to_vc(ton, node_config) if next_el_id != 0: new_name = name + '_elid' + str(next_el_id) if new_name not in names: node_config = parse_config(new_name, config, next_vset) - local.add_log(f"Adding custom overlay {name}", "debug") + local.add_log(f"Adding custom overlay {new_name}", "debug") add_custom_overlay_to_vc(ton, node_config) else: node_config = parse_config(name, config) From af8bc48a1e944bd52791ff5128fddf095e15be10 Mon Sep 17 00:00:00 2001 From: Igroman787 <27614297+igroman787@users.noreply.github.com> Date: Tue, 2 Apr 2024 23:42:41 +0300 Subject: [PATCH 10/40] delete old code --- mytonctrl.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/mytonctrl.py b/mytonctrl.py index c05bf9f5..14e14d50 100755 --- a/mytonctrl.py +++ b/mytonctrl.py @@ -209,28 +209,9 @@ def Upgrade(args): repo = "ton" author, repo, branch = check_git(args, repo, "upgrade") - # bugfix if the files are in the wrong place - liteClient = ton.GetSettings("liteClient") - configPath = liteClient.get("configPath") - pubkeyPath = liteClient.get("liteServer").get("pubkeyPath") - if "ton-lite-client-test1" in configPath: - liteClient["configPath"] = configPath.replace("lite-client/ton-lite-client-test1.config.json", "global.config.json") - if "/usr/bin/ton" in pubkeyPath: - liteClient["liteServer"]["pubkeyPath"] = "/var/ton-work/keys/liteserver.pub" - ton.SetSettings("liteClient", liteClient) - validatorConsole = ton.GetSettings("validatorConsole") - privKeyPath = validatorConsole.get("privKeyPath") - pubKeyPath = validatorConsole.get("pubKeyPath") - if "/usr/bin/ton" in privKeyPath: - validatorConsole["privKeyPath"] = "/var/ton-work/keys/client" - if "/usr/bin/ton" in pubKeyPath: - validatorConsole["pubKeyPath"] = "/var/ton-work/keys/server.pub" - ton.SetSettings("validatorConsole", validatorConsole) - # Run script runArgs = ["bash", "/usr/src/mytonctrl/scripts/upgrade.sh", "-a", author, "-r", repo, "-b", branch] exitCode = run_as_root(runArgs) - exitCode += run_as_root(["python3", "/usr/src/mytonctrl/scripts/upgrade.py"]) if exitCode == 0: text = "Upgrade - {green}OK{endc}" else: From 4f70dfa0adab9c25723211b25483f598705a17a9 Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 3 Apr 2024 21:55:24 +0800 Subject: [PATCH 11/40] add default custom overlays --- custom_overlays.py | 72 +++++++++++++++++++++++++++++++++++++++++++++- mytoncore.py | 4 +-- 2 files changed, 73 insertions(+), 3 deletions(-) diff --git a/custom_overlays.py b/custom_overlays.py index 64dcbdcd..a483bf27 100644 --- a/custom_overlays.py +++ b/custom_overlays.py @@ -51,6 +51,14 @@ def add_custom_overlay(args): with open(path, 'r') as f: config = json.load(f) ton.set_custom_overlay(args[0], config) + if '@validators' in config: + print('Dynamic overlay will be added within 1 minute') + else: + result = add_custom_overlay_to_vc(ton, parse_config(args[0], config)) + if not result: + print('Failed to add overlay to validator console') + color_print("add_custom_overlay - {red}ERROR{endc}") + return color_print("add_custom_overlay - {green}OK{endc}") @@ -69,7 +77,15 @@ def delete_custom_overlay(args): if len(args) != 1: color_print("{red}Bad args. Usage:{endc} delete_custom_overlay ") return - ton.delete_custom_overlay(args[0]) + if '@validators' in ton.get_custom_overlays().get(args[0], {}): + print('Dynamic overlay will be deleted within 1 minute') + else: + ton.delete_custom_overlay(args[0]) + result = delete_custom_overlay_from_vc(ton, args[0]) + if not result: + print('Failed to delete overlay from validator console') + color_print("delete_custom_overlay - {red}ERROR{endc}") + return color_print("delete_custom_overlay - {green}OK{endc}") @@ -86,6 +102,13 @@ def add_custom_overlay_to_vc(ton, config: dict): return 'success' in result +def custom_overlays(local, ton): + config = get_default_custom_overlay(local, ton) + if config is not None: + ton.set_custom_overlay('default', config) + deploy_custom_overlays(local, ton) + + def deploy_custom_overlays(local, ton): result = ton.validatorConsole.Run("showcustomoverlays") if 'unknown command' in result: @@ -139,3 +162,50 @@ def deploy_custom_overlays(local, ton): node_config = parse_config(name, config) local.add_log(f"Adding custom overlay {name}", "debug") add_custom_overlay_to_vc(ton, node_config) + + +MAINNET_DEFAULT_CUSTOM_OVERLAY = { + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": { + "msg_sender": True, + "msg_sender_priority": 15 + }, + "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB": { + "msg_sender": True, + "msg_sender_priority": 10 + }, + "@validators": True +} + + +TESTNET_DEFAULT_CUSTOM_OVERLAY = { + "DF27B30444D07087863B77F8BD27BABA8E57EDECA393605F6610FDCB64FFECD1": { + "msg_sender": True, + "msg_sender_priority": 15 + }, + "B360D229CA597906ADFC522FAC3EB5F8AE9D80981693225E7083577A4F016118": { + "msg_sender": True, + "msg_sender_priority": 10 + }, + "F794DE0B21423B6F4C168C5652758E5743CD977ACE13B3B2BA88E28580D9BEDB": { + "msg_sender": True, + "msg_sender_priority": 10 + }, + "6447CEAC80573AF2ABCA741FC940BB690AC263DC4B779FB6609CE5E9A4B31AE1": { + "msg_sender": True, + "msg_sender_priority": 5, + }, + "@validators": True +} + + +def get_default_custom_overlay(local, ton): + if not local.db.get('useDefaultCustomOverlays', True): + return None + network = ton.GetNetworkName() + if network == 'mainnet': + config = MAINNET_DEFAULT_CUSTOM_OVERLAY + elif network == 'testnet': + config = TESTNET_DEFAULT_CUSTOM_OVERLAY + else: + return None + return config diff --git a/mytoncore.py b/mytoncore.py index bc7ee518..33d1f872 100755 --- a/mytoncore.py +++ b/mytoncore.py @@ -8,7 +8,7 @@ import requests import re from mypylib.mypylib import * -from custom_overlays import deploy_custom_overlays +from custom_overlays import custom_overlays local = MyPyClass(__file__) @@ -4312,7 +4312,7 @@ def General(): local.start_cycle(Telemetry, sec=60, args=(ton, )) local.start_cycle(OverlayTelemetry, sec=7200, args=(ton, )) local.start_cycle(ScanLiteServers, sec=60, args=(ton,)) - local.start_cycle(deploy_custom_overlays, sec=60, args=(local, ton,)) + local.start_cycle(custom_overlays, sec=60, args=(local, ton,)) thr_sleep() #end define From 39f2717c2bd5f05d172da486dc1ad949390f772a Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 3 Apr 2024 21:59:56 +0800 Subject: [PATCH 12/40] bugfix --- custom_overlays.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/custom_overlays.py b/custom_overlays.py index a483bf27..595232a3 100644 --- a/custom_overlays.py +++ b/custom_overlays.py @@ -77,10 +77,10 @@ def delete_custom_overlay(args): if len(args) != 1: color_print("{red}Bad args. Usage:{endc} delete_custom_overlay ") return + ton.delete_custom_overlay(args[0]) if '@validators' in ton.get_custom_overlays().get(args[0], {}): print('Dynamic overlay will be deleted within 1 minute') else: - ton.delete_custom_overlay(args[0]) result = delete_custom_overlay_from_vc(ton, args[0]) if not result: print('Failed to delete overlay from validator console') From 6250661d04fb8209b4d26c5ea3eca9abb35e79ea Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 4 Apr 2024 13:33:19 +0800 Subject: [PATCH 13/40] add set_archive_ttl * fix * add set_archive_ttl to console * add status to set_archive_ttl * add set_archive_ttl --- mytonctrl.py | 14 ++++++++++++++ scripts/set_archive_ttl.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 scripts/set_archive_ttl.py diff --git a/mytonctrl.py b/mytonctrl.py index c05bf9f5..a418a740 100755 --- a/mytonctrl.py +++ b/mytonctrl.py @@ -30,6 +30,7 @@ def Init(argv): console.AddItem("add_custom_overlay", add_custom_overlay, local.translate("add_custom_overlay_cmd")) console.AddItem("list_custom_overlays", list_custom_overlays, local.translate("list_custom_overlays_cmd")) console.AddItem("delete_custom_overlay", delete_custom_overlay, local.translate("delete_custom_overlay_cmd")) + console.AddItem("set_archive_ttl", set_archive_ttl, local.translate("set_archive_ttl_cmd")) console.AddItem("nw", CreatNewWallet, local.translate("nw_cmd")) console.AddItem("aw", ActivateWallet, local.translate("aw_cmd")) @@ -1213,6 +1214,19 @@ def UpdateValidatorSet(args): #end define +def set_archive_ttl(args): + if len(args) != 1: + color_print("{red}Bad args. Usage:{endc} set_archive_ttl ") + return + ttl = args[0] + result = run_as_root(['python3', '/usr/src/mytonctrl/scripts/set_archive_ttl.py', ttl]) + if result: + color_print("set_archive_ttl - {red}Error{endc}") + return + color_print("set_archive_ttl - {green}OK{endc}") +#end define + + ### ### Start of the program ### diff --git a/scripts/set_archive_ttl.py b/scripts/set_archive_ttl.py new file mode 100644 index 00000000..506c2f80 --- /dev/null +++ b/scripts/set_archive_ttl.py @@ -0,0 +1,37 @@ +import subprocess +import sys + +with open('/etc/systemd/system/validator.service', 'r') as file: + service = file.read() + + +for line in service.split('\n'): + if line.startswith('ExecStart'): + exec_start = line + break + + +if '--archive-ttl' in exec_start: + print('Archive TTL is already set') + sys.exit(100) + +default_command = 'ExecStart = /usr/bin/ton/validator-engine/validator-engine --threads --daemonize --global-config /usr/bin/ton/global.config.json --db /var/ton-work/db/ --logname /var/ton-work/log --state-ttl 604800 --verbosity 1' + +# ExecStart = /usr/bin/ton/validator-engine/validator-engine --threads 31 --daemonize --global-config /usr/bin/ton/global.config.json --db /var/ton-work/db/ --logname /var/ton-work/log --state-ttl 604800 --verbosity 1 + +t = exec_start.split(' ') +t.pop(t.index('--threads') + 1) # pop threads value since it's different for each node + +if ' '.join(t) != default_command: + print('ExecStart is not default. Please set archive-ttl manually') + sys.exit(101) + +archive_ttl = sys.argv[1] + +new_exec_start = exec_start + f' --archive-ttl {archive_ttl}' + +with open('/etc/systemd/system/validator.service', 'w') as file: + file.write(service.replace(exec_start, new_exec_start)) + +subprocess.run(['systemctl', 'daemon-reload']) +subprocess.run(['systemctl', 'restart', 'validator']) From c649737b32c22d7a999ba21f159fad06b67a6a26 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 4 Apr 2024 14:46:46 +0800 Subject: [PATCH 14/40] update error message on set_archive_ttl --- scripts/set_archive_ttl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/set_archive_ttl.py b/scripts/set_archive_ttl.py index 506c2f80..afd74064 100644 --- a/scripts/set_archive_ttl.py +++ b/scripts/set_archive_ttl.py @@ -23,7 +23,7 @@ t.pop(t.index('--threads') + 1) # pop threads value since it's different for each node if ' '.join(t) != default_command: - print('ExecStart is not default. Please set archive-ttl manually') + print('ExecStart is not default. Please set archive-ttl manually in `/etc/systemd/system/validator.service`.') sys.exit(101) archive_ttl = sys.argv[1] From 8683e66f1632c1017fc7d115c02f6671b506d1d4 Mon Sep 17 00:00:00 2001 From: yungwine Date: Fri, 5 Apr 2024 11:43:52 +0800 Subject: [PATCH 15/40] add disk usage warning --- mytonctrl.py | 27 ++++++++++++++++++--------- translate.json | 5 +++++ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/mytonctrl.py b/mytonctrl.py index bb7f084c..b0c88148 100755 --- a/mytonctrl.py +++ b/mytonctrl.py @@ -110,6 +110,7 @@ def Init(argv): def PreUp(): CheckMytonctrlUpdate() + CheckDiskUsage() check_vport() # CheckTonUpdate() #end define @@ -140,26 +141,26 @@ def check_git(input_args, default_repo, text): git_path = f"{src_dir}/{default_repo}" default_author = "ton-blockchain" default_branch = "master" - + # Get author, repo, branch local_author, local_repo = get_git_author_and_repo(git_path) local_branch = get_git_branch(git_path) - + # Set author, repo, branch data = GetAuthorRepoBranchFromArgs(input_args) need_author = data.get("author") need_repo = data.get("repo") need_branch = data.get("branch") - + # Check if remote repo is different from default - if ((need_author is None and local_author != default_author) or + if ((need_author is None and local_author != default_author) or (need_repo is None and local_repo != default_repo)): remote_url = f"https://github.com/{local_author}/{local_repo}/tree/{need_branch if need_branch else local_branch}" raise Exception(f"{text} error: You are on {remote_url} remote url, to {text} to the tip use `{text} {remote_url}` command") elif need_branch is None and local_branch != default_branch: raise Exception(f"{text} error: You are on {local_branch} branch, to {text} to the tip of {local_branch} branch use `{text} {local_branch}` command") #end if - + if need_author is None: need_author = local_author if need_repo is None: @@ -167,7 +168,7 @@ def check_git(input_args, default_repo, text): if need_branch is None: need_branch = local_branch #end if - + return need_author, need_repo, need_branch #end define @@ -209,7 +210,7 @@ def Update(args): def Upgrade(args): repo = "ton" author, repo, branch = check_git(args, repo, "upgrade") - + # Run script runArgs = ["bash", "/usr/src/mytonctrl/scripts/upgrade.sh", "-a", author, "-r", repo, "-b", branch] exitCode = run_as_root(runArgs) @@ -227,6 +228,14 @@ def CheckMytonctrlUpdate(): color_print(local.translate("mytonctrl_update_available")) #end define + +def CheckDiskUsage(): + usage = ton.GetDbUsage() + if usage > 90: + color_print(local.translate("disk_usage_warning")) +#end define + + def CheckTonUpdate(): git_path = "/usr/src/ton" result = check_git_update(git_path) @@ -405,7 +414,7 @@ def PrintLocalStatus(adnlAddr, validatorIndex, validatorEfficiency, validatorWal dbSize_text = GetColorInt(dbSize, 1000, logic="less", ending=" Gb") dbUsage_text = GetColorInt(dbUsage, 80, logic="less", ending="%") dbStatus_text = local.translate("local_status_db").format(dbSize_text, dbUsage_text) - + # Mytonctrl and validator git hash mtcGitPath = "/usr/src/mytonctrl" validatorGitPath = "/usr/src/ton" @@ -430,7 +439,7 @@ def PrintLocalStatus(adnlAddr, validatorIndex, validatorEfficiency, validatorWal print(cpuLoad_text) print(netLoad_text) print(memoryLoad_text) - + print(disksLoad_text) print(mytoncoreStatus_text) print(validatorStatus_text) diff --git a/translate.json b/translate.json index 5ce41b98..294ac055 100644 --- a/translate.json +++ b/translate.json @@ -369,6 +369,11 @@ "ru": "{green}Доступно обновление MyTonCtrl. {red}Пожалуйста, обновите его с помощью команды `update`.{endc}", "zh_TW": "{green}MyTonCtrl 有可用更新. {red}請使用 `update` 命令進行更新.{endc}" }, + "disk_usage_warning": { + "en": "{red} Disk is almost full, clean the TON database immediately: https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming {endc}", + "ru": "{red} Диск почти заполнен, немедленно очистите базу данных TON: https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming {endc}", + "zh_TW": "{red} 磁盤幾乎滿了，立即清理 TON 數據庫: https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming {endc}" + }, "ton_update_available": { "en": "{green}TON update available. {red}Please update it with `upgrade` command.{endc}", "ru": "{green}Доступно обновление TON. {red}Пожалуйста, обновите его с помощью команды `upgrade`.{endc}", From 4bc7cebec2e3234f22afdbc211af6844a3cf7a9b Mon Sep 17 00:00:00 2001 From: yungwine Date: Fri, 5 Apr 2024 16:10:20 +0800 Subject: [PATCH 16/40] customize message warning --- mytonctrl.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mytonctrl.py b/mytonctrl.py index b0c88148..046d1676 100755 --- a/mytonctrl.py +++ b/mytonctrl.py @@ -232,7 +232,9 @@ def CheckMytonctrlUpdate(): def CheckDiskUsage(): usage = ton.GetDbUsage() if usage > 90: + print('============================================================================================') color_print(local.translate("disk_usage_warning")) + print('============================================================================================') #end define From 536f0d53675a0d90ba59190ca31dcfb04642ff15 Mon Sep 17 00:00:00 2001 From: yungwine Date: Mon, 8 Apr 2024 13:02:24 +0800 Subject: [PATCH 17/40] improve custom overlays * improve logging * improve custom overlays - Do not let node enter custom overlay if it has not required adnl address - fix bug with wrong message for deleting custom overlay --- custom_overlays.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/custom_overlays.py b/custom_overlays.py index 595232a3..acfadfda 100644 --- a/custom_overlays.py +++ b/custom_overlays.py @@ -43,7 +43,7 @@ def parse_config(name: str, config: dict, vset: list = None): def add_custom_overlay(args): - from mytonctrl import ton + from mytonctrl import ton, local if len(args) != 2: color_print("{red}Bad args. Usage:{endc} add_custom_overlay ") return @@ -54,7 +54,7 @@ def add_custom_overlay(args): if '@validators' in config: print('Dynamic overlay will be added within 1 minute') else: - result = add_custom_overlay_to_vc(ton, parse_config(args[0], config)) + result = add_custom_overlay_to_vc(local, ton, parse_config(args[0], config)) if not result: print('Failed to add overlay to validator console') color_print("add_custom_overlay - {red}ERROR{endc}") @@ -77,10 +77,11 @@ def delete_custom_overlay(args): if len(args) != 1: color_print("{red}Bad args. Usage:{endc} delete_custom_overlay ") return - ton.delete_custom_overlay(args[0]) if '@validators' in ton.get_custom_overlays().get(args[0], {}): + ton.delete_custom_overlay(args[0]) print('Dynamic overlay will be deleted within 1 minute') else: + ton.delete_custom_overlay(args[0]) result = delete_custom_overlay_from_vc(ton, args[0]) if not result: print('Failed to delete overlay from validator console') @@ -89,12 +90,26 @@ def delete_custom_overlay(args): color_print("delete_custom_overlay - {green}OK{endc}") +def check_node_eligible_for_custom_overlay(ton, config: dict): + vconfig = ton.GetValidatorConfig() + my_adnls = vconfig.adnl + node_adnls = [i["adnl_id"] for i in config["nodes"]] + for adnl in my_adnls: + if adnl.id in node_adnls: + return True + return False + + def delete_custom_overlay_from_vc(ton, name: str): result = ton.validatorConsole.Run(f"delcustomoverlay {name}") return 'success' in result -def add_custom_overlay_to_vc(ton, config: dict): +def add_custom_overlay_to_vc(local, ton, config: dict): + if not check_node_eligible_for_custom_overlay(ton, config): + local.add_log(f"Node has no adnl address required for custom overlay {config.get('name')}", "debug") + return False + local.add_log(f"Adding custom overlay {config.get('name')}", "debug") path = ton.tempDir + f'/custom_overlay_{config["name"]}.json' with open(path, 'w') as f: json.dump(config, f) @@ -149,19 +164,16 @@ def deploy_custom_overlays(local, ton): new_name = name + '_elid' + str(current_el_id) if new_name not in names: node_config = parse_config(new_name, config, current_vset) - local.add_log(f"Adding custom overlay {new_name}", "debug") - add_custom_overlay_to_vc(ton, node_config) + add_custom_overlay_to_vc(local, ton, node_config) if next_el_id != 0: new_name = name + '_elid' + str(next_el_id) if new_name not in names: node_config = parse_config(new_name, config, next_vset) - local.add_log(f"Adding custom overlay {new_name}", "debug") - add_custom_overlay_to_vc(ton, node_config) + add_custom_overlay_to_vc(local, ton, node_config) else: node_config = parse_config(name, config) - local.add_log(f"Adding custom overlay {name}", "debug") - add_custom_overlay_to_vc(ton, node_config) + add_custom_overlay_to_vc(local, ton, node_config) MAINNET_DEFAULT_CUSTOM_OVERLAY = { From a0d75b21183cc937573827edb31e053c44b68f0a Mon Sep 17 00:00:00 2001 From: yungwine Date: Mon, 8 Apr 2024 17:18:44 +0800 Subject: [PATCH 18/40] dont check verbosity in set_archive_ttl --- scripts/set_archive_ttl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/set_archive_ttl.py b/scripts/set_archive_ttl.py index afd74064..f11fd436 100644 --- a/scripts/set_archive_ttl.py +++ b/scripts/set_archive_ttl.py @@ -15,12 +15,13 @@ print('Archive TTL is already set') sys.exit(100) -default_command = 'ExecStart = /usr/bin/ton/validator-engine/validator-engine --threads --daemonize --global-config /usr/bin/ton/global.config.json --db /var/ton-work/db/ --logname /var/ton-work/log --state-ttl 604800 --verbosity 1' +default_command = 'ExecStart = /usr/bin/ton/validator-engine/validator-engine --threads --daemonize --global-config /usr/bin/ton/global.config.json --db /var/ton-work/db/ --logname /var/ton-work/log --state-ttl 604800 --verbosity' # ExecStart = /usr/bin/ton/validator-engine/validator-engine --threads 31 --daemonize --global-config /usr/bin/ton/global.config.json --db /var/ton-work/db/ --logname /var/ton-work/log --state-ttl 604800 --verbosity 1 t = exec_start.split(' ') t.pop(t.index('--threads') + 1) # pop threads value since it's different for each node +t.pop(t.index('--verbosity') + 1) # pop verbosity value if ' '.join(t) != default_command: print('ExecStart is not default. Please set archive-ttl manually in `/etc/systemd/system/validator.service`.') From 80c1c37b3cc22d779c573f71bb9c5ea2c8487628 Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 9 Apr 2024 12:59:18 +0800 Subject: [PATCH 19/40] download default custom overlays from github * improves * update default custom overlay * download default custom overlays from github --- custom_overlays.py | 48 ++++++------------------------------ default_custom_overlays.json | 32 ++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 40 deletions(-) create mode 100644 default_custom_overlays.json diff --git a/custom_overlays.py b/custom_overlays.py index acfadfda..fb11bdfd 100644 --- a/custom_overlays.py +++ b/custom_overlays.py @@ -1,5 +1,6 @@ import base64 import json +import requests from mypylib.mypylib import color_print @@ -176,48 +177,15 @@ def deploy_custom_overlays(local, ton): add_custom_overlay_to_vc(local, ton, node_config) -MAINNET_DEFAULT_CUSTOM_OVERLAY = { - "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": { - "msg_sender": True, - "msg_sender_priority": 15 - }, - "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB": { - "msg_sender": True, - "msg_sender_priority": 10 - }, - "@validators": True -} - - -TESTNET_DEFAULT_CUSTOM_OVERLAY = { - "DF27B30444D07087863B77F8BD27BABA8E57EDECA393605F6610FDCB64FFECD1": { - "msg_sender": True, - "msg_sender_priority": 15 - }, - "B360D229CA597906ADFC522FAC3EB5F8AE9D80981693225E7083577A4F016118": { - "msg_sender": True, - "msg_sender_priority": 10 - }, - "F794DE0B21423B6F4C168C5652758E5743CD977ACE13B3B2BA88E28580D9BEDB": { - "msg_sender": True, - "msg_sender_priority": 10 - }, - "6447CEAC80573AF2ABCA741FC940BB690AC263DC4B779FB6609CE5E9A4B31AE1": { - "msg_sender": True, - "msg_sender_priority": 5, - }, - "@validators": True -} - - def get_default_custom_overlay(local, ton): if not local.db.get('useDefaultCustomOverlays', True): return None network = ton.GetNetworkName() - if network == 'mainnet': - config = MAINNET_DEFAULT_CUSTOM_OVERLAY - elif network == 'testnet': - config = TESTNET_DEFAULT_CUSTOM_OVERLAY - else: + default_url = 'https://raw.githubusercontent.com/ton-blockchain/mytonctrl/mytonctrl1_dev/default_custom_overlays.json' # todo: change to master + url = local.db.get('defaultCustomOverlaysUrl', default_url) + resp = requests.get(url) + if resp.status_code != 200: + local.add_log(f"Failed to get default custom overlays from {url}", "error") return None - return config + config = resp.json() + return config.get(network) diff --git a/default_custom_overlays.json b/default_custom_overlays.json new file mode 100644 index 00000000..8d5f5e2e --- /dev/null +++ b/default_custom_overlays.json @@ -0,0 +1,32 @@ +{ + "mainnet": { + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": { + "msg_sender": true, + "msg_sender_priority": 15 + }, + "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB": { + "msg_sender": true, + "msg_sender_priority": 10 + }, + "@validators": true + }, + "testnet": { + "DF27B30444D07087863B77F8BD27BABA8E57EDECA393605F6610FDCB64FFECD1": { + "msg_sender": true, + "msg_sender_priority": 14 + }, + "B360D229CA597906ADFC522FAC3EB5F8AE9D80981693225E7083577A4F016118": { + "msg_sender": true, + "msg_sender_priority": 10 + }, + "F794DE0B21423B6F4C168C5652758E5743CD977ACE13B3B2BA88E28580D9BEDB": { + "msg_sender": true, + "msg_sender_priority": 10 + }, + "6447CEAC80573AF2ABCA741FC940BB690AC263DC4B779FB6609CE5E9A4B31AE1": { + "msg_sender": true, + "msg_sender_priority": 5 + }, + "@validators": true + } +} \ No newline at end of file From 4a258de6e33ba984b1c54467381cbb9f490106d8 Mon Sep 17 00:00:00 2001 From: Igroman787 <27614297+igroman787@users.noreply.github.com> Date: Wed, 10 Apr 2024 09:43:23 +0300 Subject: [PATCH 20/40] Update default_custom_overlays_url --- custom_overlays.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/custom_overlays.py b/custom_overlays.py index fb11bdfd..03cad4fa 100644 --- a/custom_overlays.py +++ b/custom_overlays.py @@ -181,7 +181,7 @@ def get_default_custom_overlay(local, ton): if not local.db.get('useDefaultCustomOverlays', True): return None network = ton.GetNetworkName() - default_url = 'https://raw.githubusercontent.com/ton-blockchain/mytonctrl/mytonctrl1_dev/default_custom_overlays.json' # todo: change to master + default_url = 'https://ton-blockchain.github.io/fallback_custom_overlays.json' url = local.db.get('defaultCustomOverlaysUrl', default_url) resp = requests.get(url) if resp.status_code != 200: From 1a04eea4e21b9e1ca31b777ce20ba9aa8290cc7d Mon Sep 17 00:00:00 2001 From: yungwine Date: Fri, 19 Apr 2024 11:34:33 +0400 Subject: [PATCH 21/40] fix vo --- mytoncore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mytoncore.py b/mytoncore.py index 33d1f872..d6a600ca 100755 --- a/mytoncore.py +++ b/mytoncore.py @@ -2478,12 +2478,12 @@ def VoteOffer(self, offerHash): if validatorIndex in offer.get("votedValidators"): local.add_log("Proposal already has been voted", "debug") return + self.AddSaveOffer(offer) var1 = self.CreateConfigProposalRequest(offerHash, validatorIndex) validatorSignature = self.GetValidatorSignature(validatorKey, var1) resultFilePath = self.SignProposalVoteRequestWithValidator(offerHash, validatorIndex, validatorPubkey_b64, validatorSignature) resultFilePath = self.SignBocWithWallet(wallet, resultFilePath, fullConfigAddr, 1.5) self.SendFile(resultFilePath, wallet) - self.AddSaveOffer(offer) #end define def VoteComplaint(self, electionId, complaintHash): From 138da0a51c957d1fe0bb506dc1f471875899cdbc Mon Sep 17 00:00:00 2001 From: neodiX Date: Sat, 11 May 2024 20:16:22 +0200 Subject: [PATCH 22/40] use jemalloc --- scripts/toninstaller.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/toninstaller.sh b/scripts/toninstaller.sh index 02f865a2..740fd468 100755 --- a/scripts/toninstaller.sh +++ b/scripts/toninstaller.sh @@ -84,7 +84,7 @@ if [ "$OSTYPE" == "linux-gnu" ]; then elif [ -f /etc/debian_version ]; then echo "Ubuntu/Debian Linux detected." apt-get update - apt-get install -y build-essential git cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev python3-pip libsecp256k1-dev libsodium-dev liblz4-dev + apt-get install -y build-essential git cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev python3-pip libsecp256k1-dev libsodium-dev liblz4-dev libjemalloc-dev # Install ninja apt-get install -y ninja-build @@ -172,7 +172,7 @@ if [[ "$OSTYPE" =~ darwin.* ]]; then cmake -DCMAKE_BUILD_TYPE=Release $SOURCES_DIR/ton -GNinja fi else - cmake -DCMAKE_BUILD_TYPE=Release $SOURCES_DIR/ton -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$opensslPath/include -DOPENSSL_CRYPTO_LIBRARY=$opensslPath/libcrypto.a + cmake -DCMAKE_BUILD_TYPE=Release $SOURCES_DIR/ton -GNinja -DTON_USE_JEMALLOC=ON -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$opensslPath/include -DOPENSSL_CRYPTO_LIBRARY=$opensslPath/libcrypto.a fi # Компилируем из исходников From 9310e68ab1f08810468ef9baf847d16a348b7389 Mon Sep 17 00:00:00 2001 From: neodix Date: Sun, 12 May 2024 21:16:03 +0200 Subject: [PATCH 23/40] use jemalloc on upgrade --- scripts/upgrade.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/upgrade.sh b/scripts/upgrade.sh index 117d168b..f381a962 100644 --- a/scripts/upgrade.sh +++ b/scripts/upgrade.sh @@ -29,7 +29,7 @@ COLOR='\033[92m' ENDC='\033[0m' # Установить дополнительные зависимости -apt-get install -y libsecp256k1-dev libsodium-dev ninja-build liblz4-dev +apt-get install -y libsecp256k1-dev libsodium-dev ninja-build liblz4-dev libjemalloc-dev # bugfix if the files are in the wrong place wget "https://ton-blockchain.github.io/global.config.json" -O global.config.json @@ -72,7 +72,7 @@ rm -rf .ninja_* memory=$(cat /proc/meminfo | grep MemAvailable | awk '{print $2}') let "cpuNumber = memory / 2100000" || cpuNumber=1 -cmake -DCMAKE_BUILD_TYPE=Release ${srcdir}/${repo} -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$opensslPath/include -DOPENSSL_CRYPTO_LIBRARY=$opensslPath/libcrypto.a +cmake -DCMAKE_BUILD_TYPE=Release ${srcdir}/${repo} -GNinja -DTON_USE_JEMALLOC=ON -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$opensslPath/include -DOPENSSL_CRYPTO_LIBRARY=$opensslPath/libcrypto.a ninja -j ${cpuNumber} fift validator-engine lite-client pow-miner validator-engine-console generate-random-id dht-server func tonlibjson rldp-http-proxy systemctl restart validator From 0b8bd0149c7828cd7d69a88cb64b580f3b44f437 Mon Sep 17 00:00:00 2001 From: Igroman787 <27614297+igroman787@users.noreply.github.com> Date: Mon, 13 May 2024 15:36:02 +0300 Subject: [PATCH 24/40] Revert "use jemalloc" --- scripts/toninstaller.sh | 4 ++-- scripts/upgrade.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/toninstaller.sh b/scripts/toninstaller.sh index 740fd468..02f865a2 100755 --- a/scripts/toninstaller.sh +++ b/scripts/toninstaller.sh @@ -84,7 +84,7 @@ if [ "$OSTYPE" == "linux-gnu" ]; then elif [ -f /etc/debian_version ]; then echo "Ubuntu/Debian Linux detected." apt-get update - apt-get install -y build-essential git cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev python3-pip libsecp256k1-dev libsodium-dev liblz4-dev libjemalloc-dev + apt-get install -y build-essential git cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev python3-pip libsecp256k1-dev libsodium-dev liblz4-dev # Install ninja apt-get install -y ninja-build @@ -172,7 +172,7 @@ if [[ "$OSTYPE" =~ darwin.* ]]; then cmake -DCMAKE_BUILD_TYPE=Release $SOURCES_DIR/ton -GNinja fi else - cmake -DCMAKE_BUILD_TYPE=Release $SOURCES_DIR/ton -GNinja -DTON_USE_JEMALLOC=ON -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$opensslPath/include -DOPENSSL_CRYPTO_LIBRARY=$opensslPath/libcrypto.a + cmake -DCMAKE_BUILD_TYPE=Release $SOURCES_DIR/ton -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$opensslPath/include -DOPENSSL_CRYPTO_LIBRARY=$opensslPath/libcrypto.a fi # Компилируем из исходников diff --git a/scripts/upgrade.sh b/scripts/upgrade.sh index f381a962..117d168b 100644 --- a/scripts/upgrade.sh +++ b/scripts/upgrade.sh @@ -29,7 +29,7 @@ COLOR='\033[92m' ENDC='\033[0m' # Установить дополнительные зависимости -apt-get install -y libsecp256k1-dev libsodium-dev ninja-build liblz4-dev libjemalloc-dev +apt-get install -y libsecp256k1-dev libsodium-dev ninja-build liblz4-dev # bugfix if the files are in the wrong place wget "https://ton-blockchain.github.io/global.config.json" -O global.config.json @@ -72,7 +72,7 @@ rm -rf .ninja_* memory=$(cat /proc/meminfo | grep MemAvailable | awk '{print $2}') let "cpuNumber = memory / 2100000" || cpuNumber=1 -cmake -DCMAKE_BUILD_TYPE=Release ${srcdir}/${repo} -GNinja -DTON_USE_JEMALLOC=ON -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$opensslPath/include -DOPENSSL_CRYPTO_LIBRARY=$opensslPath/libcrypto.a +cmake -DCMAKE_BUILD_TYPE=Release ${srcdir}/${repo} -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$opensslPath/include -DOPENSSL_CRYPTO_LIBRARY=$opensslPath/libcrypto.a ninja -j ${cpuNumber} fift validator-engine lite-client pow-miner validator-engine-console generate-random-id dht-server func tonlibjson rldp-http-proxy systemctl restart validator From 3257127657b0b55056e874aa4b4e083f2de54835 Mon Sep 17 00:00:00 2001 From: Igroman787 <27614297+igroman787@users.noreply.github.com> Date: Sat, 25 May 2024 14:47:52 +0300 Subject: [PATCH 25/40] use jemmaloc --- scripts/toninstaller.sh | 4 ++-- scripts/upgrade.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/toninstaller.sh b/scripts/toninstaller.sh index 02f865a2..740fd468 100755 --- a/scripts/toninstaller.sh +++ b/scripts/toninstaller.sh @@ -84,7 +84,7 @@ if [ "$OSTYPE" == "linux-gnu" ]; then elif [ -f /etc/debian_version ]; then echo "Ubuntu/Debian Linux detected." apt-get update - apt-get install -y build-essential git cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev python3-pip libsecp256k1-dev libsodium-dev liblz4-dev + apt-get install -y build-essential git cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev python3-pip libsecp256k1-dev libsodium-dev liblz4-dev libjemalloc-dev # Install ninja apt-get install -y ninja-build @@ -172,7 +172,7 @@ if [[ "$OSTYPE" =~ darwin.* ]]; then cmake -DCMAKE_BUILD_TYPE=Release $SOURCES_DIR/ton -GNinja fi else - cmake -DCMAKE_BUILD_TYPE=Release $SOURCES_DIR/ton -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$opensslPath/include -DOPENSSL_CRYPTO_LIBRARY=$opensslPath/libcrypto.a + cmake -DCMAKE_BUILD_TYPE=Release $SOURCES_DIR/ton -GNinja -DTON_USE_JEMALLOC=ON -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$opensslPath/include -DOPENSSL_CRYPTO_LIBRARY=$opensslPath/libcrypto.a fi # Компилируем из исходников diff --git a/scripts/upgrade.sh b/scripts/upgrade.sh index 117d168b..f381a962 100644 --- a/scripts/upgrade.sh +++ b/scripts/upgrade.sh @@ -29,7 +29,7 @@ COLOR='\033[92m' ENDC='\033[0m' # Установить дополнительные зависимости -apt-get install -y libsecp256k1-dev libsodium-dev ninja-build liblz4-dev +apt-get install -y libsecp256k1-dev libsodium-dev ninja-build liblz4-dev libjemalloc-dev # bugfix if the files are in the wrong place wget "https://ton-blockchain.github.io/global.config.json" -O global.config.json @@ -72,7 +72,7 @@ rm -rf .ninja_* memory=$(cat /proc/meminfo | grep MemAvailable | awk '{print $2}') let "cpuNumber = memory / 2100000" || cpuNumber=1 -cmake -DCMAKE_BUILD_TYPE=Release ${srcdir}/${repo} -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$opensslPath/include -DOPENSSL_CRYPTO_LIBRARY=$opensslPath/libcrypto.a +cmake -DCMAKE_BUILD_TYPE=Release ${srcdir}/${repo} -GNinja -DTON_USE_JEMALLOC=ON -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$opensslPath/include -DOPENSSL_CRYPTO_LIBRARY=$opensslPath/libcrypto.a ninja -j ${cpuNumber} fift validator-engine lite-client pow-miner validator-engine-console generate-random-id dht-server func tonlibjson rldp-http-proxy systemctl restart validator From 93b5580cf72c8edd9ea116bc10ef321cca23fa4c Mon Sep 17 00:00:00 2001 From: yungwine Date: Fri, 7 Jun 2024 17:42:22 +0700 Subject: [PATCH 26/40] add set_state_ttl script --- scripts/set_state_ttl.py | 30 ++++++++++++++++++++++++++++++ scripts/upgrade.sh | 8 ++++++++ 2 files changed, 38 insertions(+) create mode 100644 scripts/set_state_ttl.py diff --git a/scripts/set_state_ttl.py b/scripts/set_state_ttl.py new file mode 100644 index 00000000..4838fe29 --- /dev/null +++ b/scripts/set_state_ttl.py @@ -0,0 +1,30 @@ +import subprocess +import sys + +with open('/etc/systemd/system/validator.service', 'r') as file: + service = file.read() + + +for line in service.split('\n'): + if line.startswith('ExecStart'): + exec_start = line + break + + +if exec_start.split(' ')[2] != '/usr/bin/ton/validator-engine/validator-engine': + raise Exception('Invalid node start command in service file') + + +if '--state-ttl 604800' not in exec_start: + print('No state-ttl or custom one found in ExecStart') + sys.exit(0) + +new_exec_start = exec_start.replace('--state-ttl 604800', '') + +with open('/etc/systemd/system/validator.service', 'w') as file: + file.write(service.replace(exec_start, new_exec_start)) + +subprocess.run(['systemctl', 'daemon-reload']) +subprocess.run(['systemctl', 'restart', 'validator']) + +print('Removed state-ttl from service file.') diff --git a/scripts/upgrade.sh b/scripts/upgrade.sh index f381a962..8c5cae14 100644 --- a/scripts/upgrade.sh +++ b/scripts/upgrade.sh @@ -76,6 +76,14 @@ cmake -DCMAKE_BUILD_TYPE=Release ${srcdir}/${repo} -GNinja -DTON_USE_JEMALLOC=ON ninja -j ${cpuNumber} fift validator-engine lite-client pow-miner validator-engine-console generate-random-id dht-server func tonlibjson rldp-http-proxy systemctl restart validator +if [ -e /usr/src/mytonctrl/scripts/set_state_ttl.py ] +then + /usr/bin/python3 /usr/src/mytonctrl/scripts/set_state_ttl.py +else + echo "Set state ttl script is not found!" +fi + + # Конец echo -e "${COLOR}[1/1]${ENDC} TON components update completed" exit 0 From c92c6d6a12ff323472d7d9de1676ee262e0e6a1a Mon Sep 17 00:00:00 2001 From: "Dr. Awesome Doge" Date: Tue, 18 Jun 2024 17:27:37 +0800 Subject: [PATCH 27/40] Exclude deletion of `local.config.json` https://github.com/ton-blockchain/mytonctrl/issues/205 --- scripts/upgrade.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/upgrade.sh b/scripts/upgrade.sh index 8c5cae14..ca896434 100644 --- a/scripts/upgrade.sh +++ b/scripts/upgrade.sh @@ -67,7 +67,7 @@ make build_libs -j12 # Update binary cd ${bindir}/${repo} -ls --hide=global.config.json | xargs -d '\n' rm -rf +ls --hide=global.config.json --hide=local.config.json | xargs -d '\n' rm -rf rm -rf .ninja_* memory=$(cat /proc/meminfo | grep MemAvailable | awk '{print $2}') let "cpuNumber = memory / 2100000" || cpuNumber=1 From d7e879bd79cf05712f2f9bee7b092fa0f5e685a2 Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 19 Jun 2024 13:27:26 +0700 Subject: [PATCH 28/40] add warning to update to mtc2.0 for non-validators --- mytonctrl.py | 11 ++++++++++- translate.json | 5 +++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/mytonctrl.py b/mytonctrl.py index 046d1676..6804a30b 100755 --- a/mytonctrl.py +++ b/mytonctrl.py @@ -109,7 +109,8 @@ def Init(argv): #end define def PreUp(): - CheckMytonctrlUpdate() + CheckMytonctrlUpdate() # check mtc current branch update only if there wasnt warning about mtc2 + CheckMytonctrl2Update() CheckDiskUsage() check_vport() # CheckTonUpdate() @@ -229,6 +230,14 @@ def CheckMytonctrlUpdate(): #end define +def CheckMytonctrl2Update(): + if not ton.find_myself_in_vl(): # we are not validator in current and prev rounds + print('============================================================================================') + color_print(local.translate("update_mtc2_warning")) + print('============================================================================================') +# end define + + def CheckDiskUsage(): usage = ton.GetDbUsage() if usage > 90: diff --git a/translate.json b/translate.json index 294ac055..16e5377d 100644 --- a/translate.json +++ b/translate.json @@ -369,6 +369,11 @@ "ru": "{green}Доступно обновление MyTonCtrl. {red}Пожалуйста, обновите его с помощью команды `update`.{endc}", "zh_TW": "{green}MyTonCtrl 有可用更新. {red}請使用 `update` 命令進行更新.{endc}" }, + "update_mtc2_warning": { + "en": "If you are not a validator and using node only as liteserver, {bold} update to the MyTonCtrl2.0:{endc} `update mytonctrl2`. After updating turn \"liteserver\" mode on: `disable_mode validator`; `enable_mode liteserver`", + "ru": "Если вы не валидатор и используете ноду только как лайтсервер, {bold} обновитесь до MyTonCtrl2.0:{endc} `update mytonctrl2`. После обновления включите режим \"Лайтсервера\": `disable_mode validator`; `enable_mode liteserver`", + "zh_TW": "如果您不是驗證者，僅將節點用作 liteserver，{bold} 更新到 MyTonCtrl2.0:{endc} `update mytonctrl2`。更新後，將 \"liteserver\" 模式打開: `disable_mode validator`; `enable_mode liteserver`" + }, "disk_usage_warning": { "en": "{red} Disk is almost full, clean the TON database immediately: https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming {endc}", "ru": "{red} Диск почти заполнен, немедленно очистите базу данных TON: https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming {endc}", From f62863646a70aea0a9fe853f3b2c388e85f67f12 Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 19 Jun 2024 13:33:12 +0700 Subject: [PATCH 29/40] add find_myself_in_vl --- mytoncore.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mytoncore.py b/mytoncore.py index d6a600ca..236f14a0 100755 --- a/mytoncore.py +++ b/mytoncore.py @@ -2677,6 +2677,15 @@ def GetValidatorsList(self, past=False): return validators #end define + def find_myself_in_vl(self): + current_vl = self.GetValidatorsList() + past_vl = self.GetValidatorsList(past=True) + my_adnl = self.GetAdnlAddr() + for validator in current_vl + past_vl: + if validator["adnlAddr"] == my_adnl: + return True + return False + def CheckValidators(self, start, end): local.add_log("start CheckValidators function", "debug") electionId = start From c563ed5587ffb345082ed1c8599d703585b6332b Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 19 Jun 2024 13:48:25 +0700 Subject: [PATCH 30/40] make mtc2 warning red --- translate.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/translate.json b/translate.json index 16e5377d..326a67b8 100644 --- a/translate.json +++ b/translate.json @@ -370,9 +370,9 @@ "zh_TW": "{green}MyTonCtrl 有可用更新. {red}請使用 `update` 命令進行更新.{endc}" }, "update_mtc2_warning": { - "en": "If you are not a validator and using node only as liteserver, {bold} update to the MyTonCtrl2.0:{endc} `update mytonctrl2`. After updating turn \"liteserver\" mode on: `disable_mode validator`; `enable_mode liteserver`", - "ru": "Если вы не валидатор и используете ноду только как лайтсервер, {bold} обновитесь до MyTonCtrl2.0:{endc} `update mytonctrl2`. После обновления включите режим \"Лайтсервера\": `disable_mode validator`; `enable_mode liteserver`", - "zh_TW": "如果您不是驗證者，僅將節點用作 liteserver，{bold} 更新到 MyTonCtrl2.0:{endc} `update mytonctrl2`。更新後，將 \"liteserver\" 模式打開: `disable_mode validator`; `enable_mode liteserver`" + "en": "{red} If you are not a validator and using node only as liteserver, update to the MyTonCtrl2.0:{endc} `update mytonctrl2`. After updating turn \"liteserver\" mode on: `disable_mode validator`; `enable_mode liteserver` {endc}", + "ru": "{red} Если вы не валидатор и используете ноду только как лайтсервер, обновитесь до MyTonCtrl2.0:{endc} `update mytonctrl2`. После обновления включите режим \"Лайтсервера\": `disable_mode validator`; `enable_mode liteserver` {endc}", + "zh_TW": "{red} 如果您不是驗證者，僅將節點用作 liteserver，{bold} 更新到 MyTonCtrl2.0: `update mytonctrl2`。更新後，將 \"liteserver\" 模式打開: `disable_mode validator`; `enable_mode liteserver` {endc}" }, "disk_usage_warning": { "en": "{red} Disk is almost full, clean the TON database immediately: https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming {endc}", From 4629d3f7700b5855e3b7a3c445f038e4309533ed Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 19 Jun 2024 13:49:52 +0700 Subject: [PATCH 31/40] fix color --- translate.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/translate.json b/translate.json index 326a67b8..613039dc 100644 --- a/translate.json +++ b/translate.json @@ -370,9 +370,9 @@ "zh_TW": "{green}MyTonCtrl 有可用更新. {red}請使用 `update` 命令進行更新.{endc}" }, "update_mtc2_warning": { - "en": "{red} If you are not a validator and using node only as liteserver, update to the MyTonCtrl2.0:{endc} `update mytonctrl2`. After updating turn \"liteserver\" mode on: `disable_mode validator`; `enable_mode liteserver` {endc}", - "ru": "{red} Если вы не валидатор и используете ноду только как лайтсервер, обновитесь до MyTonCtrl2.0:{endc} `update mytonctrl2`. После обновления включите режим \"Лайтсервера\": `disable_mode validator`; `enable_mode liteserver` {endc}", - "zh_TW": "{red} 如果您不是驗證者，僅將節點用作 liteserver，{bold} 更新到 MyTonCtrl2.0: `update mytonctrl2`。更新後，將 \"liteserver\" 模式打開: `disable_mode validator`; `enable_mode liteserver` {endc}" + "en": "{red} If you are not a validator and using node only as liteserver, update to the MyTonCtrl2.0: `update mytonctrl2`. After updating turn \"liteserver\" mode on: `disable_mode validator`; `enable_mode liteserver` {endc}", + "ru": "{red} Если вы не валидатор и используете ноду только как лайтсервер, обновитесь до MyTonCtrl2.0: `update mytonctrl2`. После обновления включите режим \"Лайтсервера\": `disable_mode validator`; `enable_mode liteserver` {endc}", + "zh_TW": "{red} 如果您不是驗證者，僅將節點用作 liteserver，更新到 MyTonCtrl2.0: `update mytonctrl2`。更新後，將 \"liteserver\" 模式打開: `disable_mode validator`; `enable_mode liteserver` {endc}" }, "disk_usage_warning": { "en": "{red} Disk is almost full, clean the TON database immediately: https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming {endc}", From 86dcdbb954b75f8cd869de8790f87108ca60e650 Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 19 Jun 2024 13:58:03 +0700 Subject: [PATCH 32/40] add try --- mytonctrl.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/mytonctrl.py b/mytonctrl.py index 6804a30b..02072eb3 100755 --- a/mytonctrl.py +++ b/mytonctrl.py @@ -231,10 +231,13 @@ def CheckMytonctrlUpdate(): def CheckMytonctrl2Update(): - if not ton.find_myself_in_vl(): # we are not validator in current and prev rounds - print('============================================================================================') - color_print(local.translate("update_mtc2_warning")) - print('============================================================================================') + try: + if not ton.find_myself_in_vl(): # we are not validator in current and prev rounds + print('============================================================================================') + color_print(local.translate("update_mtc2_warning")) + print('============================================================================================') + except Exception as err: + print(f'Failed to check node as validator: {err}') # end define From 41e03ccaaecf8a6570aa77b774c329f44012b7b0 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 11 Jul 2024 21:59:11 +0700 Subject: [PATCH 33/40] handle list saved offers --- mytoncore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mytoncore.py b/mytoncore.py index d6a600ca..a24c59c6 100755 --- a/mytoncore.py +++ b/mytoncore.py @@ -3066,7 +3066,7 @@ def WriteBookmarkData(self, bookmark): def GetSaveOffers(self): bname = "saveOffers" saveOffers = local.db.get(bname) - if saveOffers is None: + if saveOffers is None or isinstance(saveOffers, list): saveOffers = dict() local.db[bname] = saveOffers return saveOffers From e9ea856521a9fcbe97e46c82af3f6bf5e36059ca Mon Sep 17 00:00:00 2001 From: Igroman787 <27614297+igroman787@users.noreply.github.com> Date: Sun, 14 Jul 2024 13:23:46 +0300 Subject: [PATCH 34/40] replace `find_myself_in_vl` with `find_myself_in_el` --- default_custom_overlays.json | 32 -------------------------------- mytoncore.py | 13 +++++++------ mytonctrl.py | 4 ++-- translate.json | 6 +++--- 4 files changed, 12 insertions(+), 43 deletions(-) delete mode 100644 default_custom_overlays.json diff --git a/default_custom_overlays.json b/default_custom_overlays.json deleted file mode 100644 index 8d5f5e2e..00000000 --- a/default_custom_overlays.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "mainnet": { - "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": { - "msg_sender": true, - "msg_sender_priority": 15 - }, - "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB": { - "msg_sender": true, - "msg_sender_priority": 10 - }, - "@validators": true - }, - "testnet": { - "DF27B30444D07087863B77F8BD27BABA8E57EDECA393605F6610FDCB64FFECD1": { - "msg_sender": true, - "msg_sender_priority": 14 - }, - "B360D229CA597906ADFC522FAC3EB5F8AE9D80981693225E7083577A4F016118": { - "msg_sender": true, - "msg_sender_priority": 10 - }, - "F794DE0B21423B6F4C168C5652758E5743CD977ACE13B3B2BA88E28580D9BEDB": { - "msg_sender": true, - "msg_sender_priority": 10 - }, - "6447CEAC80573AF2ABCA741FC940BB690AC263DC4B779FB6609CE5E9A4B31AE1": { - "msg_sender": true, - "msg_sender_priority": 5 - }, - "@validators": true - } -} \ No newline at end of file diff --git a/mytoncore.py b/mytoncore.py index 04a3071d..ed132f58 100755 --- a/mytoncore.py +++ b/mytoncore.py @@ -2677,14 +2677,15 @@ def GetValidatorsList(self, past=False): return validators #end define - def find_myself_in_vl(self): - current_vl = self.GetValidatorsList() - past_vl = self.GetValidatorsList(past=True) + def find_myself_in_el(self): + save_elections = self.GetSaveElections() my_adnl = self.GetAdnlAddr() - for validator in current_vl + past_vl: - if validator["adnlAddr"] == my_adnl: - return True + for election_id, election in save_elections.items(): + for adnl in election: + if adnl == my_adnl: + return True return False + #end define def CheckValidators(self, start, end): local.add_log("start CheckValidators function", "debug") diff --git a/mytonctrl.py b/mytonctrl.py index 02072eb3..1cbb1683 100755 --- a/mytonctrl.py +++ b/mytonctrl.py @@ -232,12 +232,12 @@ def CheckMytonctrlUpdate(): def CheckMytonctrl2Update(): try: - if not ton.find_myself_in_vl(): # we are not validator in current and prev rounds + if not ton.find_myself_in_el(): # we are not validator in current and prev rounds print('============================================================================================') color_print(local.translate("update_mtc2_warning")) print('============================================================================================') except Exception as err: - print(f'Failed to check node as validator: {err}') + local.add_log(f'Failed to check node as validator: {err}', "error") # end define diff --git a/translate.json b/translate.json index 613039dc..e2aa23ed 100644 --- a/translate.json +++ b/translate.json @@ -370,9 +370,9 @@ "zh_TW": "{green}MyTonCtrl 有可用更新. {red}請使用 `update` 命令進行更新.{endc}" }, "update_mtc2_warning": { - "en": "{red} If you are not a validator and using node only as liteserver, update to the MyTonCtrl2.0: `update mytonctrl2`. After updating turn \"liteserver\" mode on: `disable_mode validator`; `enable_mode liteserver` {endc}", - "ru": "{red} Если вы не валидатор и используете ноду только как лайтсервер, обновитесь до MyTonCtrl2.0: `update mytonctrl2`. После обновления включите режим \"Лайтсервера\": `disable_mode validator`; `enable_mode liteserver` {endc}", - "zh_TW": "{red} 如果您不是驗證者，僅將節點用作 liteserver，更新到 MyTonCtrl2.0: `update mytonctrl2`。更新後，將 \"liteserver\" 模式打開: `disable_mode validator`; `enable_mode liteserver` {endc}" + "en": "{red}This version is outdated. Please update to the second version: `update mytonctrl2`{endc}", + "ru": "{red}Данная версия устарела. Пожалуйста обновитесь на вторую версию: `update mytonctrl2`{endc}", + "zh_TW": "{red}這個版本已經過時了。請更新至第二版本: `update mytonctrl2`{endc}" }, "disk_usage_warning": { "en": "{red} Disk is almost full, clean the TON database immediately: https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming {endc}", From 622c3f69195277b91e438cd47ff575c4f582004e Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 18 Jul 2024 17:37:40 +0800 Subject: [PATCH 35/40] add collator config --- modules/collator_config.py | 92 ++++++++++++++++++++++++++++++++++++++ mytoncore/mytoncore.py | 10 +++++ mytonctrl/mytonctrl.py | 4 ++ 3 files changed, 106 insertions(+) create mode 100644 modules/collator_config.py diff --git a/modules/collator_config.py b/modules/collator_config.py new file mode 100644 index 00000000..9596520c --- /dev/null +++ b/modules/collator_config.py @@ -0,0 +1,92 @@ +import json +import requests + +from mypylib.mypylib import color_print +from modules.module import MtcModule +from mytoncore.utils import hex2base64 + + +class CollatorConfigModule(MtcModule): + + @staticmethod + def check_config_url(url): + try: + r = requests.get(url, timeout=3) + if r.status_code != 200: + print(f'Failed to get config from {url}: {r.status_code} code; {r.text}') + return + return r.json() + except Exception as e: + print(f'Failed to get config from {url}: {e}') + return + + @staticmethod + def check_config_file(path): + try: + with open(path, 'r') as f: + return json.load(f) + except Exception as e: + print(f'Failed to read config from {path}: {e}') + return + + @staticmethod + def get_config(path): + if 'http' in path: + config = CollatorConfigModule.check_config_url(path) + else: + config = CollatorConfigModule.check_config_file(path) + if config is None: + raise Exception(f'Failed to get config') + return config + + def add_collator_config_to_vc(self, config: dict): + self.local.add_log(f"Adding collator options config to validator console", "debug") + path = self.ton.tempDir + f'/collator_config.json' + with open(path, 'w') as f: + json.dump(config, f) + result = self.ton.validatorConsole.Run(f"setcollatoroptionsjson {path}") + return 'success' in result, result + + def set_collator_config(self, args): + if len(args) != 1: + color_print("{red}Bad args. Usage:{endc} set_collator_config ") + return + location = args[0] + config = self.get_config(location) + self.ton.set_collator_config(location) + added, msg = self.add_collator_config_to_vc(config) + if not added: + print(f'Failed to add collator config to validator console: {msg}') + color_print("set_collator_config - {red}ERROR{endc}") + return + color_print("set_collator_config - {green}OK{endc}") + + def get_collator_config(self, args): + location = self.ton.get_collator_config_location() + print(f'Collator config location: {location}') + path = self.ton.tempDir + f'/current_collator_config.json' + output = self.ton.validatorConsole.Run(f'getcollatoroptionsjson {path}') + if 'saved config to' not in output: + print(f'Failed to get collator config: {output}') + color_print("get_collator_config - {red}ERROR{endc}") + return + with open(path, 'r') as f: + config = json.load(f) + print(f'Collator config:') + print(json.dumps(config, indent=4)) + color_print("get_collator_config - {green}OK{endc}") + + def update_collator_config(self, args): + location = self.ton.get_collator_config() + config = self.get_config(location) + added, msg = self.add_collator_config_to_vc(config) + if not added: + print(f'Failed to add collator config to validator console: {msg}') + color_print("update_collator_config - {red}ERROR{endc}") + return + color_print("update_collator_config - {green}OK{endc}") + + def add_console_commands(self, console): + console.AddItem("set_collator_config", self.set_collator_config, self.local.translate("set_collator_config_cmd")) + console.AddItem("update_collator_config", self.update_collator_config, self.local.translate("update_collator_config_cmd")) + console.AddItem("get_collator_config", self.get_collator_config, self.local.translate("get_collator_config_cmd")) diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 726b0d2f..098d6ec9 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -3986,6 +3986,16 @@ def delete_custom_overlay(self, name: str): del self.local.db['custom_overlays'][name] self.local.save() + def set_collator_config(self, location: str): + self.local.db['collator_config'] = location + self.local.save() + + def get_collator_config_location(self): + default = 'https://raw.githubusercontent.com/ton-blockchain/ton-blockchain.github.io/main/default_collator_options.json' + location = self.local.db.get('collator_config', default) + if location is None: + location = default + return location def GetNetworkName(self): data = self.local.read_db(self.liteClient.configPath) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 9598716f..47e06721 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -128,6 +128,10 @@ def inject_globals(func): module = CustomOverlayModule(ton, local) module.add_console_commands(console) + from modules.collator_config import CollatorConfigModule + module = CollatorConfigModule(ton, local) + module.add_console_commands(console) + if ton.using_validator(): from modules.validator import ValidatorModule module = ValidatorModule(ton, local) From ca211e9aa747a90e8b90a26b085857501cb62509 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 18 Jul 2024 21:41:39 +0800 Subject: [PATCH 36/40] bugfix --- modules/collator_config.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/collator_config.py b/modules/collator_config.py index 9596520c..9234b08f 100644 --- a/modules/collator_config.py +++ b/modules/collator_config.py @@ -3,7 +3,6 @@ from mypylib.mypylib import color_print from modules.module import MtcModule -from mytoncore.utils import hex2base64 class CollatorConfigModule(MtcModule): @@ -77,7 +76,7 @@ def get_collator_config(self, args): color_print("get_collator_config - {green}OK{endc}") def update_collator_config(self, args): - location = self.ton.get_collator_config() + location = self.ton.get_collator_config_location() config = self.get_config(location) added, msg = self.add_collator_config_to_vc(config) if not added: From 7c8a5c9f9f02e2a8c267aa93b9eb391c4157a3c9 Mon Sep 17 00:00:00 2001 From: yungwine Date: Mon, 5 Aug 2024 11:27:36 +0800 Subject: [PATCH 37/40] fix incorrect warning for --state-ttl changing --- mytonctrl/mytonctrl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 47e06721..ba60f84b 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -388,7 +388,7 @@ def Upgrade(ton, args): try: from mytoninstaller.mytoninstaller import set_node_argument, get_node_args node_args = get_node_args() - if node_args['--state-ttl'] == '604800': + if node_args.get('--state-ttl') == '604800': set_node_argument(ton.local, ["--state-ttl", "-d"]) except Exception as e: color_print(f"{{red}}Failed to set node argument: {e} {{endc}}") From c34a459023c2510363926b1813a83980a58fd39c Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 6 Aug 2024 14:18:31 +0800 Subject: [PATCH 38/40] check validator wallet status before activating pool --- modules/__init__.py | 1 + modules/nominator_pool.py | 2 ++ modules/single_pool.py | 1 + mytoncore/mytoncore.py | 27 +++++++++++++++++++-------- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/modules/__init__.py b/modules/__init__.py index 298302b0..625ecc79 100644 --- a/modules/__init__.py +++ b/modules/__init__.py @@ -34,6 +34,7 @@ class Setting: 'stake': Setting('validator', None, 'Stake amount'), 'stakePercent': Setting('validator', 99, 'Stake percent if `stake` is null'), 'isSlashing': Setting('validator', None, 'Create complaints to validators'), + 'walletName': Setting('validator', None, 'Wallet name'), 'maxFactor': Setting('validator', None, 'Param send to Elector. if null will be taken from 17 config param'), 'participateBeforeEnd': Setting('validator', None, 'Amount of seconds before start of round to participate'), 'liquid_pool_addr': Setting('liquid-staking', None, 'Liquid staking pool address'), diff --git a/modules/nominator_pool.py b/modules/nominator_pool.py index a5adcc4b..3ca8647b 100644 --- a/modules/nominator_pool.py +++ b/modules/nominator_pool.py @@ -62,6 +62,8 @@ def do_activate_pool(self, pool, ex=True): elif account.status == "active": self.local.add_log("do_activate_pool warning: account status is active", "warning") else: + validator_wallet = self.ton.GetValidatorWallet() + self.ton.check_account_status(validator_wallet.addrB64) self.ton.SendFile(pool.bocFilePath, pool, timeout=False, remove=False) #end define diff --git a/modules/single_pool.py b/modules/single_pool.py index d2721c5a..7e33a260 100644 --- a/modules/single_pool.py +++ b/modules/single_pool.py @@ -51,6 +51,7 @@ def do_activate_single_pool(self, pool): self.local.add_log("start activate_single_pool function", "debug") boc_mode = "--with-init" validator_wallet = self.ton.GetValidatorWallet() + self.ton.check_account_status(validator_wallet.addrB64) result_file_path = self.ton.SignBocWithWallet(validator_wallet, pool.bocFilePath, pool.addrB64_init, 1, boc_mode=boc_mode) self.ton.SendFile(result_file_path, validator_wallet) diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 098d6ec9..fffede66 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -1180,9 +1180,7 @@ def SignBocWithWallet(self, wallet, boc_path, dest, coins, **kwargs): # Balance checking account = self.GetAccount(wallet.addrB64) - if account.balance < coins + 0.1: - raise Exception("Wallet balance is less than requested coins") - #end if + self.check_account_balance(account, coins + 0.1) # Bounceable checking destAccount = self.GetAccount(dest) @@ -1864,6 +1862,22 @@ def GetWalletId(self, wallet): return subwallet #end define + def check_account_balance(self, account, coins): + if not isinstance(account, Account): + account = self.GetAccount(account) + if account.balance < coins: + raise Exception(f"Wallet {account.addrB64} balance is less than requested coins. Balance: {account.balance}, requested amount: {coins} (need {coins - account.balance} more)") + # end if + # end define + + def check_account_status(self, account): + if not isinstance(account, Account): + account = self.GetAccount(account) + if account.status != "active": + raise Exception(f"Wallet {account.addrB64} account is uninitialized") + # end if + # end define + def MoveCoins(self, wallet, dest, coins, **kwargs): self.local.add_log("start MoveCoins function", "debug") flags = kwargs.get("flags", list()) @@ -1884,11 +1898,8 @@ def MoveCoins(self, wallet, dest, coins, **kwargs): # Balance checking account = self.GetAccount(wallet.addrB64) - if account.balance < coins + 0.1: - raise Exception("Wallet balance is less than requested coins") - if account.status != "active": - raise Exception("Wallet account is uninitialized") - #end if + self.check_account_balance(account, coins + 0.1) + self.check_account_status(account) # Bounceable checking destAccount = self.GetAccount(dest) From 968437a4b598974a5c72d78c7343cf0fd45d0f54 Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 6 Aug 2024 14:20:32 +0800 Subject: [PATCH 39/40] fixes fix exceptions rename method fix name --- modules/__init__.py | 2 +- modules/nominator_pool.py | 2 +- modules/single_pool.py | 2 +- mytoncore/mytoncore.py | 11 +++++++---- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/modules/__init__.py b/modules/__init__.py index 625ecc79..8b92c260 100644 --- a/modules/__init__.py +++ b/modules/__init__.py @@ -34,7 +34,7 @@ class Setting: 'stake': Setting('validator', None, 'Stake amount'), 'stakePercent': Setting('validator', 99, 'Stake percent if `stake` is null'), 'isSlashing': Setting('validator', None, 'Create complaints to validators'), - 'walletName': Setting('validator', None, 'Wallet name'), + 'validatorWalletName': Setting('validator', 'wallet_001', 'Validator\'s wallet name'), 'maxFactor': Setting('validator', None, 'Param send to Elector. if null will be taken from 17 config param'), 'participateBeforeEnd': Setting('validator', None, 'Amount of seconds before start of round to participate'), 'liquid_pool_addr': Setting('liquid-staking', None, 'Liquid staking pool address'), diff --git a/modules/nominator_pool.py b/modules/nominator_pool.py index 3ca8647b..14f4fbdd 100644 --- a/modules/nominator_pool.py +++ b/modules/nominator_pool.py @@ -63,7 +63,7 @@ def do_activate_pool(self, pool, ex=True): self.local.add_log("do_activate_pool warning: account status is active", "warning") else: validator_wallet = self.ton.GetValidatorWallet() - self.ton.check_account_status(validator_wallet.addrB64) + self.ton.check_account_active(validator_wallet.addrB64) self.ton.SendFile(pool.bocFilePath, pool, timeout=False, remove=False) #end define diff --git a/modules/single_pool.py b/modules/single_pool.py index 7e33a260..5b5c6e2b 100644 --- a/modules/single_pool.py +++ b/modules/single_pool.py @@ -51,7 +51,7 @@ def do_activate_single_pool(self, pool): self.local.add_log("start activate_single_pool function", "debug") boc_mode = "--with-init" validator_wallet = self.ton.GetValidatorWallet() - self.ton.check_account_status(validator_wallet.addrB64) + self.ton.check_account_active(validator_wallet.addrB64) result_file_path = self.ton.SignBocWithWallet(validator_wallet, pool.bocFilePath, pool.addrB64_init, 1, boc_mode=boc_mode) self.ton.SendFile(result_file_path, validator_wallet) diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index fffede66..ed643083 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -1866,15 +1866,18 @@ def check_account_balance(self, account, coins): if not isinstance(account, Account): account = self.GetAccount(account) if account.balance < coins: - raise Exception(f"Wallet {account.addrB64} balance is less than requested coins. Balance: {account.balance}, requested amount: {coins} (need {coins - account.balance} more)") + raise Exception(f"Account {account.addrB64} balance is less than requested coins. Balance: {account.balance}, requested amount: {coins} (need {coins - account.balance} more)") # end if # end define - def check_account_status(self, account): + def check_account_active(self, account): if not isinstance(account, Account): + address = account account = self.GetAccount(account) + else: + address = account.addrB64 if account.status != "active": - raise Exception(f"Wallet {account.addrB64} account is uninitialized") + raise Exception(f"Account {address} account is uninitialized") # end if # end define @@ -1899,7 +1902,7 @@ def MoveCoins(self, wallet, dest, coins, **kwargs): # Balance checking account = self.GetAccount(wallet.addrB64) self.check_account_balance(account, coins + 0.1) - self.check_account_status(account) + self.check_account_active(account) # Bounceable checking destAccount = self.GetAccount(dest) From ed83c22ccdf09caa0fff98737fd28f45cc058d45 Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 6 Aug 2024 19:32:56 +0800 Subject: [PATCH 40/40] delete unused files form mtc1 --- custom_overlays.py | 191 ------------------------------------- scripts/set_archive_ttl.py | 38 -------- scripts/set_state_ttl.py | 30 ------ 3 files changed, 259 deletions(-) delete mode 100644 custom_overlays.py delete mode 100644 scripts/set_archive_ttl.py delete mode 100644 scripts/set_state_ttl.py diff --git a/custom_overlays.py b/custom_overlays.py deleted file mode 100644 index 03cad4fa..00000000 --- a/custom_overlays.py +++ /dev/null @@ -1,191 +0,0 @@ -import base64 -import json -import requests - -from mypylib.mypylib import color_print - - -def hex2base64(h): - b = bytes.fromhex(h) - b64 = base64.b64encode(b) - s = b64.decode("utf-8") - return s - - -def parse_config(name: str, config: dict, vset: list = None): - """ - Converts config to validator-console friendly format - :param name: custom overlay name - :param config: config - :param vset: list of validators adnl addresses, can be None if `@validators` not in config - :return: - """ - result = { - "name": name, - "nodes": [] - } - for k, v in config.items(): - if k == '@validators' and v: - if vset is None: - raise Exception("Validators set is not defined but @validators is in config") - for v_adnl in vset: - result["nodes"].append({ - "adnl_id": hex2base64(v_adnl), - "msg_sender": False, - }) - else: - result["nodes"].append({ - "adnl_id": hex2base64(k), - "msg_sender": v["msg_sender"], - }) - if v["msg_sender"]: - result["nodes"][-1]["msg_sender_priority"] = v["msg_sender_priority"] - return result - - -def add_custom_overlay(args): - from mytonctrl import ton, local - if len(args) != 2: - color_print("{red}Bad args. Usage:{endc} add_custom_overlay ") - return - path = args[1] - with open(path, 'r') as f: - config = json.load(f) - ton.set_custom_overlay(args[0], config) - if '@validators' in config: - print('Dynamic overlay will be added within 1 minute') - else: - result = add_custom_overlay_to_vc(local, ton, parse_config(args[0], config)) - if not result: - print('Failed to add overlay to validator console') - color_print("add_custom_overlay - {red}ERROR{endc}") - return - color_print("add_custom_overlay - {green}OK{endc}") - - -def list_custom_overlays(args): - from mytonctrl import ton - if not ton.get_custom_overlays(): - color_print("{red}No custom overlays{endc}") - return - for k, v in ton.get_custom_overlays().items(): - color_print(f"Custom overlay {{bold}}{k}{{endc}}:") - print(json.dumps(v, indent=4)) - - -def delete_custom_overlay(args): - from mytonctrl import ton - if len(args) != 1: - color_print("{red}Bad args. Usage:{endc} delete_custom_overlay ") - return - if '@validators' in ton.get_custom_overlays().get(args[0], {}): - ton.delete_custom_overlay(args[0]) - print('Dynamic overlay will be deleted within 1 minute') - else: - ton.delete_custom_overlay(args[0]) - result = delete_custom_overlay_from_vc(ton, args[0]) - if not result: - print('Failed to delete overlay from validator console') - color_print("delete_custom_overlay - {red}ERROR{endc}") - return - color_print("delete_custom_overlay - {green}OK{endc}") - - -def check_node_eligible_for_custom_overlay(ton, config: dict): - vconfig = ton.GetValidatorConfig() - my_adnls = vconfig.adnl - node_adnls = [i["adnl_id"] for i in config["nodes"]] - for adnl in my_adnls: - if adnl.id in node_adnls: - return True - return False - - -def delete_custom_overlay_from_vc(ton, name: str): - result = ton.validatorConsole.Run(f"delcustomoverlay {name}") - return 'success' in result - - -def add_custom_overlay_to_vc(local, ton, config: dict): - if not check_node_eligible_for_custom_overlay(ton, config): - local.add_log(f"Node has no adnl address required for custom overlay {config.get('name')}", "debug") - return False - local.add_log(f"Adding custom overlay {config.get('name')}", "debug") - path = ton.tempDir + f'/custom_overlay_{config["name"]}.json' - with open(path, 'w') as f: - json.dump(config, f) - result = ton.validatorConsole.Run(f"addcustomoverlay {path}") - return 'success' in result - - -def custom_overlays(local, ton): - config = get_default_custom_overlay(local, ton) - if config is not None: - ton.set_custom_overlay('default', config) - deploy_custom_overlays(local, ton) - - -def deploy_custom_overlays(local, ton): - result = ton.validatorConsole.Run("showcustomoverlays") - if 'unknown command' in result: - return # node old version - names = [] - for line in result.split('\n'): - if line.startswith('Overlay'): - names.append(line.split(' ')[1].replace('"', '').replace(':', '')) - - config34 = ton.GetConfig34() - current_el_id = config34['startWorkTime'] - current_vset = [i["adnlAddr"] for i in config34['validators']] - - config36 = ton.GetConfig36() - next_el_id = config36['startWorkTime'] if config36['validators'] else 0 - next_vset = [i["adnlAddr"] for i in config36['validators']] - - for name in names: - # check that overlay still exists in mtc db - pure_name = name - suffix = name.split('_')[-1] - if suffix.startswith('elid') and suffix.split('elid')[-1].isdigit(): # probably election id - pure_name = '_'.join(name.split('_')[:-1]) - el_id = int(suffix.split('elid')[-1]) - if el_id not in (current_el_id, next_el_id): - local.add_log(f"Overlay {name} is not in current or next election, deleting", "debug") - delete_custom_overlay_from_vc(ton, name) # delete overlay if election id is not in current or next election - continue - - if pure_name not in ton.get_custom_overlays(): - local.add_log(f"Overlay {name} ({pure_name}) is not in mtc db, deleting", "debug") - delete_custom_overlay_from_vc(ton, name) # delete overlay if it's not in mtc db - - for name, config in ton.get_custom_overlays().items(): - if name in names: - continue - if '@validators' in config: - new_name = name + '_elid' + str(current_el_id) - if new_name not in names: - node_config = parse_config(new_name, config, current_vset) - add_custom_overlay_to_vc(local, ton, node_config) - - if next_el_id != 0: - new_name = name + '_elid' + str(next_el_id) - if new_name not in names: - node_config = parse_config(new_name, config, next_vset) - add_custom_overlay_to_vc(local, ton, node_config) - else: - node_config = parse_config(name, config) - add_custom_overlay_to_vc(local, ton, node_config) - - -def get_default_custom_overlay(local, ton): - if not local.db.get('useDefaultCustomOverlays', True): - return None - network = ton.GetNetworkName() - default_url = 'https://ton-blockchain.github.io/fallback_custom_overlays.json' - url = local.db.get('defaultCustomOverlaysUrl', default_url) - resp = requests.get(url) - if resp.status_code != 200: - local.add_log(f"Failed to get default custom overlays from {url}", "error") - return None - config = resp.json() - return config.get(network) diff --git a/scripts/set_archive_ttl.py b/scripts/set_archive_ttl.py deleted file mode 100644 index f11fd436..00000000 --- a/scripts/set_archive_ttl.py +++ /dev/null @@ -1,38 +0,0 @@ -import subprocess -import sys - -with open('/etc/systemd/system/validator.service', 'r') as file: - service = file.read() - - -for line in service.split('\n'): - if line.startswith('ExecStart'): - exec_start = line - break - - -if '--archive-ttl' in exec_start: - print('Archive TTL is already set') - sys.exit(100) - -default_command = 'ExecStart = /usr/bin/ton/validator-engine/validator-engine --threads --daemonize --global-config /usr/bin/ton/global.config.json --db /var/ton-work/db/ --logname /var/ton-work/log --state-ttl 604800 --verbosity' - -# ExecStart = /usr/bin/ton/validator-engine/validator-engine --threads 31 --daemonize --global-config /usr/bin/ton/global.config.json --db /var/ton-work/db/ --logname /var/ton-work/log --state-ttl 604800 --verbosity 1 - -t = exec_start.split(' ') -t.pop(t.index('--threads') + 1) # pop threads value since it's different for each node -t.pop(t.index('--verbosity') + 1) # pop verbosity value - -if ' '.join(t) != default_command: - print('ExecStart is not default. Please set archive-ttl manually in `/etc/systemd/system/validator.service`.') - sys.exit(101) - -archive_ttl = sys.argv[1] - -new_exec_start = exec_start + f' --archive-ttl {archive_ttl}' - -with open('/etc/systemd/system/validator.service', 'w') as file: - file.write(service.replace(exec_start, new_exec_start)) - -subprocess.run(['systemctl', 'daemon-reload']) -subprocess.run(['systemctl', 'restart', 'validator']) diff --git a/scripts/set_state_ttl.py b/scripts/set_state_ttl.py deleted file mode 100644 index 4838fe29..00000000 --- a/scripts/set_state_ttl.py +++ /dev/null @@ -1,30 +0,0 @@ -import subprocess -import sys - -with open('/etc/systemd/system/validator.service', 'r') as file: - service = file.read() - - -for line in service.split('\n'): - if line.startswith('ExecStart'): - exec_start = line - break - - -if exec_start.split(' ')[2] != '/usr/bin/ton/validator-engine/validator-engine': - raise Exception('Invalid node start command in service file') - - -if '--state-ttl 604800' not in exec_start: - print('No state-ttl or custom one found in ExecStart') - sys.exit(0) - -new_exec_start = exec_start.replace('--state-ttl 604800', '') - -with open('/etc/systemd/system/validator.service', 'w') as file: - file.write(service.replace(exec_start, new_exec_start)) - -subprocess.run(['systemctl', 'daemon-reload']) -subprocess.run(['systemctl', 'restart', 'validator']) - -print('Removed state-ttl from service file.')