From 8cec21b18944fd65c4b345cf490add26f6b06d82 Mon Sep 17 00:00:00 2001 From: Andreas Wicenec Date: Mon, 17 Oct 2022 15:59:05 +0800 Subject: [PATCH] Additional modifications for ehtim --- tools/xml2palette/xml2palette.py | 135 +++++++++++++++++++++++-------- 1 file changed, 100 insertions(+), 35 deletions(-) diff --git a/tools/xml2palette/xml2palette.py b/tools/xml2palette/xml2palette.py index 653c2a199..0f7def9dc 100755 --- a/tools/xml2palette/xml2palette.py +++ b/tools/xml2palette/xml2palette.py @@ -758,7 +758,7 @@ def process_greatgrandchild(self, ggchild: dict) -> dict: :returns member dict """ - logger.debug("Initialized ggchild member: %s", self.member) + # logger.debug("Initialized ggchild member: %s", self.member) logger.debug("greatgrandchild element: %s", ggchild.tag) if ggchild.tag == "name": self.func_name = ggchild.text if self.func_name == "Unknown" else self.func_name @@ -767,23 +767,27 @@ def process_greatgrandchild(self, ggchild: dict) -> dict: if len(ggchild) > 0 and len(ggchild[0]) > 0 and ggchild[0][0].text != None: # get detailed description text - dd = ggchild[0][0].text + detailed_description = ggchild[0][0].text - # check if a return type exists in the detailed description - hasReturn = dd.rfind(":return:") != -1 or dd.rfind(":returns:") != -1 + dd = tokenize_description(detailed_description) # get return type, if it exists - if hasReturn: - return_part = dd[dd.rfind(":return:")+8:].strip().replace('\n', ' ') - output_port_name = "output" - logger.debug("Add output port:" + str(output_port_name) + "/" + str(self.return_type) + "/" + str(return_part)) - self.member["params"].append({"key": str(output_port_name), "direction": "out", "value": str(output_port_name) + "//" + str(self.return_type) + "/OutputPort/readwrite//False/False/" + str(return_part) }) + if dd['returns']: + for output in dd['returns']: + port_name = output[0] + output_desc = output[1] + logger.debug("Add output port:" + str(port_name) + "/" + str(self.return_type) + "/" + str(output_desc)) + self.member["params"].append( + {"key": str(port_name), + "direction": "out", + "value": str(port_name) + "//" + str(self.return_type) + "/OutputPort/readwrite//False/False/" + str(output_desc) + }) # get first part of description, up until when the param are mentioned - description = dd[:dd.find(":param")].strip() + description = dd['desc'] # find the list of param names and descriptions in the tag - params = parse_params(dd) + params = dd['params'] # use the params above for p in params: @@ -904,10 +908,11 @@ def process_greatgrandchild(self, ggchild: dict) -> dict: self.member = None # else: # self.func_name = f"{self.func_path}.{self.func_name}" - self.return_type = "None" if self.return_type == "def" else self.return_type - self.member["params"].append({"key": "func_name", "direction": None, "value": "Function Name/" + f"{self.func_path}.{self.func_name}" + "/String/ApplicationArgument/readonly//False/True/Python function name"}) - self.member["params"].append({"key": "input_parser", "direction": None, "value": "Input Parser/pickle/Select/ApplicationArgument/readwrite/pickle,eval,npy,path,dataurl/False/False/Input port parsing technique"}) - self.member["params"].append({"key": "output_parser", "direction": None, "value": "Output Parser/pickle/Select/ApplicationArgument/readwrite/pickle,eval,npy,path,dataurl/False/False/Output port parsing technique"}) + if self.member: + self.return_type = "None" if self.return_type == "def" else self.return_type + self.member["params"].append({"key": "func_name", "direction": None, "value": "Function Name/" + f"{self.func_path}.{self.func_name}" + "/String/ApplicationArgument/readonly//False/True/Python function name"}) + self.member["params"].append({"key": "input_parser", "direction": None, "value": "Input Parser/pickle/Select/ApplicationArgument/readwrite/pickle,eval,npy,path,dataurl/False/False/Input port parsing technique"}) + self.member["params"].append({"key": "output_parser", "direction": None, "value": "Output Parser/pickle/Select/ApplicationArgument/readwrite/pickle,eval,npy,path,dataurl/False/False/Output port parsing technique"}) def process_compounddef(compounddef:dict) -> list: """ @@ -1098,7 +1103,7 @@ def _process_grandchild(gchild, hold_name, language): return None if gg.member != member and gg.member["params"] not in [None, []]: member["params"].extend(gg.member["params"]) - logger.debug("member after adding gg_members: %s", member) + # logger.debug("member after adding gg_members: %s", member) logger.debug("Finished processing great grand children") del(gg) @@ -1120,42 +1125,102 @@ def process_compounddef_default(compounddef, language): continue return result +def tokenize_description(detailed_description: str) -> dict: + """ + Parse the detailed description and split it up into description + text, parameters and return value (if applicable) + + :param detailed_description: str; the text contained in the XML node + + :returns: dict of form {'desc':(str), 'params':(list), 'returns':(str)} + """ + args_index = 0 + ret_index = -1 + p_format = "Args:" + r_format = "Returns:" # assume Args format + dd = [d.strip() for d in detailed_description.split("\n")] + + if dd.count("Args:") > 0: + args_index = dd.index("Args:") + 1 + if dd.count("Returns:") > 0: + ret_index = dd.index("Returns:") + if detailed_description.find(":param") >= 0: + p_format = ":param" + r_format = ":returns:" + ret_index = detailed_description.find(r_format) + param_lines = [d for d in dd if d.startswith(p_format)] + else: + param_lines = [d for d in dd[args_index:ret_index] if d.find(":") > -1] + desc = detailed_description.split(p_format)[0] + returns = detailed_description.split(r_format)[1] if ret_index >= 0 else None + + params = parse_params(param_lines, p_format=p_format) + returns = parse_returns(returns) if ret_index > -1 else None + return {'desc': desc, 'params': params, 'returns': returns} # find the list of param names and descriptions in the tag -def parse_params(detailed_description: str) -> list: +def parse_params(param_lines: list, p_format:str='Args:') -> list: """ Parse parameter descirptions found in a detailed_description tag. This assumes rEST style documentation. - :param detailed_description: str, the content of the description XML node + :param param_lines: list, the content of the description XML node + :param p_format: str, the format of the paramter lines - :returns list of parameter descriptions + :returns list of tuples of form (param_name, parameter description) """ - result = [] - - if detailed_description.find("Returns:") >= 0: - split_str = "Returns:" - elif detailed_description.find(":returns") >= 0: - split_str = ":returns" - detailed_description = detailed_description.split(split_str)[0] - param_lines = [p.replace('\n','').strip() for p in detailed_description.split(":param")[1:]] - # param_lines = [line.strip() for line in detailed_description] + params = [] for p_line in param_lines: - logger.debug("p_line: %s" + p_line) + logger.debug("p_line: %s; p_format: %s", p_line, p_format) try: - index_of_second_colon = p_line.index(':', 0) + if p_format == ':param': + if p_line.startswith(p_format): + index_of_colon = p_line.index(':', 1) + name_start = len(p_format) + 1 + else: + continue + else: + index_of_colon = p_line.index(':') + name_start = 0 except: - # didnt find second colon, skip + # didnt find colon, skip continue - param_name = p_line[1:index_of_second_colon].strip() - param_description = p_line[index_of_second_colon+2:].strip() + param_name = p_line[name_start:index_of_colon].strip() + param_name = param_name.split('(')[0].strip() # get rid of type if exists + param_description = p_line[index_of_colon+2:].strip() + logger.debug("param_name: %s; param_description: %s", param_name, param_description) + params.append((param_name, param_description)) - result.append((param_name, param_description)) + return params - return result +def parse_returns(returns:list) -> list: + """ + Parse the return specs into a list. Deals with multiple returns. + + :param returns: list of tuples of form (return_name, return_description) + + :returns: list of tuples of return items with descriptions + """ + rr = [r for r in returns.split("\n") if r != ""] + r_names = [] + r_descs = [] + for r in rr: + r_name = "" + r_desc = "" + if r.find(':') > -1: + r_name, r_desc = r.split(":") + else: + r_desc += r + r_names.append(r_name) + r_descs.append(r_desc) + r_names = [r.split(":")[0].strip() for r in rr] + r_names = [r.replace("(", "") for r in r_names] + r_names = [r.replace(")", "") for r in r_names] + returns = tuple(zip(r_names, r_descs)) + return returns # find the named aparam in params, and update the description