diff --git a/modules/custom_overlays.py b/modules/custom_overlays.py new file mode 100644 index 00000000..e951318c --- /dev/null +++ b/modules/custom_overlays.py @@ -0,0 +1,181 @@ +import json +import requests + +from mypylib.mypylib import color_print +from modules.module import MtcModule +from mytoncore.utils import hex2base64 + + +class CustomOverlayModule(MtcModule): + + @staticmethod + def parse_config(name: str, config: dict, vset: list = None): + """ + Converts config to validator-console friendly format + :param name: custom overlay name + :param config: config + :param vset: list of validators adnl addresses, can be None if `@validators` not in config + :return: + """ + result = { + "name": name, + "nodes": [] + } + for k, v in config.items(): + if k == '@validators' and v: + if vset is None: + raise Exception("Validators set is not defined but @validators is in config") + for v_adnl in vset: + result["nodes"].append({ + "adnl_id": hex2base64(v_adnl), + "msg_sender": False, + }) + else: + result["nodes"].append({ + "adnl_id": hex2base64(k), + "msg_sender": v["msg_sender"], + }) + if v["msg_sender"]: + result["nodes"][-1]["msg_sender_priority"] = v["msg_sender_priority"] + return result + + def add_custom_overlay(self, args): + if len(args) != 2: + color_print("{red}Bad args. Usage:{endc} add_custom_overlay ") + return + path = args[1] + with open(path, 'r') as f: + config = json.load(f) + self.ton.set_custom_overlay(args[0], config) + if '@validators' in config: + print('Dynamic overlay will be added within 1 minute') + else: + result = self.add_custom_overlay_to_vc(self.parse_config(args[0], config)) + if not result: + print('Failed to add overlay to validator console') + color_print("add_custom_overlay - {red}ERROR{endc}") + return + color_print("add_custom_overlay - {green}OK{endc}") + + def list_custom_overlays(self, args): + if not self.ton.get_custom_overlays(): + color_print("{red}No custom overlays{endc}") + return + for k, v in self.ton.get_custom_overlays().items(): + color_print(f"Custom overlay {{bold}}{k}{{endc}}:") + print(json.dumps(v, indent=4)) + + def delete_custom_overlay(self, args): + if len(args) != 1: + color_print("{red}Bad args. Usage:{endc} delete_custom_overlay ") + return + if '@validators' in self.ton.get_custom_overlays().get(args[0], {}): + self.ton.delete_custom_overlay(args[0]) + print('Dynamic overlay will be deleted within 1 minute') + else: + self.ton.delete_custom_overlay(args[0]) + result = self.delete_custom_overlay_from_vc(args[0]) + if not result: + print('Failed to delete overlay from validator console') + color_print("delete_custom_overlay - {red}ERROR{endc}") + return + color_print("delete_custom_overlay - {green}OK{endc}") + + def check_node_eligible_for_custom_overlay(self, config: dict): + vconfig = self.ton.GetValidatorConfig() + my_adnls = vconfig.adnl + node_adnls = [i["adnl_id"] for i in config["nodes"]] + for adnl in my_adnls: + if adnl.id in node_adnls: + return True + return False + + def delete_custom_overlay_from_vc(self, name: str): + result = self.ton.validatorConsole.Run(f"delcustomoverlay {name}") + return 'success' in result + + def add_custom_overlay_to_vc(self, config: dict): + if not self.check_node_eligible_for_custom_overlay(config): + self.ton.local.add_log(f"Node has no adnl address required for custom overlay {config.get('name')}", "debug") + return False + self.ton.local.add_log(f"Adding custom overlay {config.get('name')}", "debug") + path = self.ton.tempDir + f'/custom_overlay_{config["name"]}.json' + with open(path, 'w') as f: + json.dump(config, f) + result = self.ton.validatorConsole.Run(f"addcustomoverlay {path}") + return 'success' in result + + def custom_overlays(self): + config = self.get_default_custom_overlay() + if config is not None: + self.ton.set_custom_overlay('default', config) + self.deploy_custom_overlays() + + def deploy_custom_overlays(self): + result = self.ton.validatorConsole.Run("showcustomoverlays") + if 'unknown command' in result: + return # node old version + names = [] + for line in result.split('\n'): + if line.startswith('Overlay'): + names.append(line.split(' ')[1].replace('"', '').replace(':', '')) + + config34 = self.ton.GetConfig34() + current_el_id = config34['startWorkTime'] + current_vset = [i["adnlAddr"] for i in config34['validators']] + + config36 = self.ton.GetConfig36() + next_el_id = config36['startWorkTime'] if config36['validators'] else 0 + next_vset = [i["adnlAddr"] for i in config36['validators']] + + for name in names: + # check that overlay still exists in mtc db + pure_name = name + suffix = name.split('_')[-1] + if suffix.startswith('elid') and suffix.split('elid')[-1].isdigit(): # probably election id + pure_name = '_'.join(name.split('_')[:-1]) + el_id = int(suffix.split('elid')[-1]) + if el_id not in (current_el_id, next_el_id): + self.ton.local.add_log(f"Overlay {name} is not in current or next election, deleting", "debug") + self.delete_custom_overlay_from_vc(name) # delete overlay if election id is not in current or next election + continue + + if pure_name not in self.ton.get_custom_overlays(): + self.ton.local.add_log(f"Overlay {name} ({pure_name}) is not in mtc db, deleting", "debug") + self.delete_custom_overlay_from_vc(name) # delete overlay if it's not in mtc db + + for name, config in self.ton.get_custom_overlays().items(): + if name in names: + continue + if '@validators' in config: + new_name = name + '_elid' + str(current_el_id) + if new_name not in names: + node_config = self.parse_config(new_name, config, current_vset) + self.add_custom_overlay_to_vc(node_config) + + if next_el_id != 0: + new_name = name + '_elid' + str(next_el_id) + if new_name not in names: + node_config = self.parse_config(new_name, config, next_vset) + self.add_custom_overlay_to_vc(node_config) + else: + node_config = self.parse_config(name, config) + self.add_custom_overlay_to_vc(node_config) + + def get_default_custom_overlay(self): + if not self.ton.local.db.get('useDefaultCustomOverlays', True): + return None + network = self.ton.GetNetworkName() + default_url = 'https://ton-blockchain.github.io/fallback_custom_overlays.json' + url = self.ton.local.db.get('defaultCustomOverlaysUrl', default_url) + resp = requests.get(url) + if resp.status_code != 200: + self.ton.local.add_log(f"Failed to get default custom overlays from {url}", "error") + return None + config = resp.json() + return config.get(network) + + def add_console_commands(self, console): + console.AddItem("add_custom_overlay", self.add_custom_overlay, self.local.translate("add_custom_overlay_cmd")) + console.AddItem("list_custom_overlays", self.list_custom_overlays, self.local.translate("list_custom_overlays_cmd")) + console.AddItem("delete_custom_overlay", self.delete_custom_overlay, self.local.translate("delete_custom_overlay_cmd")) diff --git a/mytoncore/functions.py b/mytoncore/functions.py index f90e9038..c7a3d373 100755 --- a/mytoncore/functions.py +++ b/mytoncore/functions.py @@ -690,6 +690,10 @@ def General(local): local.start_cycle(Telemetry, sec=60, args=(local, ton, )) local.start_cycle(OverlayTelemetry, sec=7200, args=(local, ton, )) local.start_cycle(ScanLiteServers, sec=60, args=(local, ton,)) + + from modules.custom_overlays import CustomOverlayModule + local.start_cycle(CustomOverlayModule(ton, local).custom_overlays, sec=60, args=()) + thr_sleep() # end define diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 6498a191..133d5c60 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -3910,6 +3910,21 @@ def ControllerRecoverStake(self, controllerAddr): self.local.add_log("ControllerRecoverStake completed") #end define + def get_custom_overlays(self): + if 'custom_overlays' not in self.local.db: + self.local.db['custom_overlays'] = {} + return self.local.db['custom_overlays'] + + def set_custom_overlay(self, name: str, config: dict): + overlays = self.get_custom_overlays() + overlays[name] = config + self.local.save() + + def delete_custom_overlay(self, name: str): + del self.local.db['custom_overlays'][name] + self.local.save() + + def GetNetworkName(self): data = self.local.read_db(self.liteClient.configPath) mainnet_zero_state_root_hash = "F6OpKZKqvqeFp6CQmFomXNMfMj2EnaUSOXN+Mh+wVWk=" diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 9c407dd7..7d9865c8 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -120,6 +120,10 @@ def inject_globals(func): #console.AddItem("ssoc", inject_globals(SignShardOverlayCert), local.translate("ssoc_cmd")) #console.AddItem("isoc", inject_globals(ImportShardOverlayCert), local.translate("isoc_cmd")) + from modules.custom_overlays import CustomOverlayModule + module = CustomOverlayModule(ton, local) + module.add_console_commands(console) + if ton.using_validator(): from modules.validator import ValidatorModule module = ValidatorModule(ton, local)