From c0605fb7ceb98f8f01189d1eb7c447455a77f80d Mon Sep 17 00:00:00 2001 From: Viktor Kreschenski Date: Fri, 10 Jan 2020 17:24:37 +0100 Subject: [PATCH 1/2] Implemented rules2yml converter --- rules2yml.py | 179 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 rules2yml.py diff --git a/rules2yml.py b/rules2yml.py new file mode 100644 index 0000000..003918d --- /dev/null +++ b/rules2yml.py @@ -0,0 +1,179 @@ +import sys +import unicodedata +import re +from glob import * +import os + + +# Define grammar rules for rule definitions +rules_dict = {'in_range': r'\b(in_range)\b: \[\d+(\.\d+)?, \d+(\.\d+)?\]', + 'is_greater_than': r'\b(is_greater_than)\b: \d+(\.\d+)?', + 'is_greater_than_or_equal_to': r'\b(is_greater_than_or_equal_to)\b', + 'is_less_than_or_equal_to': r'\b(is_less_than_or_equal_to)\b', + 'is_less_than': r'\b(is_less_than)\b: \d+(\.\d+)?', + 'is_equal_to': r'\b(is_equal_to)\b', + 'is_different_to': r'\b(is_different_to)\b \d+(\.\d+)?', + 'is_globally_unique': r'\b(is_globally_unique)\b', + 'refers_to': r'\b(refers_to)\b', + 'is_iso_country_code': r'\b(is_iso_country_code)\b', + 'first_element': r'\b(first_element)\b', + 'last_element': r'\b(last_element)\b', + 'is_optional': r'\b(is_optional)\b', + 'check_if': r'\b(check_if)\b \[\{.*: \d+(\.\d+)?, target: .*}, \{do_check: \{.*: \d+(\.\d+)?}}]'} + +rule_dir = "testreq" + +if not os.path.exists(rule_dir): + os.makedirs(rule_dir) + +for file in glob("open-simulation-interface/*.proto*"): + filename = file.split("open-simulation-interface/")[1].split(".proto")[0] + + if os.path.exists(f"{rule_dir}/{filename}.yml"): + continue + + with open(f"{rule_dir}/{filename}.yml", "a") as yml_file: + with open(file, "rt") as fin: + isEnum = False + enumName = "" + numMessage = 0 + shiftCounter = False + saveStatement = "" + rules = [] + + for line in fin: + if file.find(".proto") != -1: + + # Search for comment ("//"). + matchComment = re.search("//", line) + if matchComment is not None: + statement = line[:matchComment.start()] + comment = line[matchComment.end():] + else: + statement = line + comment = "" + + # Add part of the statement from last line. + statement = saveStatement + " " + statement + saveStatement = "" + + # New line is not necessary. Remove for a better output. + statement = statement.replace("\n", "") + comment = comment.replace("\n", "") + + # Is statement complete + matchSep = re.search(r"[{};]", statement) + if matchSep is None: + saveStatement = statement + statement = "" + else: + saveStatement = statement[matchSep.end():] + statement = statement[:matchSep.end()] + + if isEnum is True: + matchName = re.search(r"\b\w[\S:]+\b", statement) + if matchName is not None: + checkName = statement[matchName.start():matchName.end()] + + # Search for "enum". + matchEnum = re.search(r"\benum\b", statement) + if matchEnum is not None: + isEnum = True + # print(f"Matched enum {isEnum}") + endOfLine = statement[matchEnum.end():] + matchName = re.search(r"\b\w[\S]*\b", endOfLine) + if matchName is not None: + # Test case 8: Check name - no special char + matchNameConv = re.search(r"\b[A-Z][a-zA-Z0-9]*\b",endOfLine[matchName.start():matchName.end()]) + enumName = convert(endOfLine[matchName.start():matchName.end()]) + "_" + + # Search for a closing brace. + matchClosingBrace = re.search("}", statement) + if isEnum is True and matchClosingBrace is not None: + isEnum = False + enumName = "" + continue + + def convert(name): + s1 = re.sub(r'(.)([A-Z][a-z]+)', r'\1_\2', name) + return re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', s1).upper() + + # Check if not inside an enum. + if isEnum is False: + # Search for "message". + matchMessage = re.search(r"\bmessage\b", statement) + if matchMessage is not None: + # a new message or a new nested message + numMessage += 1 + endOfLine = statement[matchMessage.end():] + matchName = re.search(r"\b\w[\S]*\b", endOfLine) + if matchName is not None: + # Test case 10: Check name - no special char - + # start with a capital letter + matchNameConv = re.search(r"\b[A-Z][a-zA-Z0-9]*\b",endOfLine[matchName.start():matchName.end()]) + # print(matchNameConv.group(0)) + yml_file.write(2*(numMessage-1)*" "+f"{matchNameConv.group(0)}:\n") + + elif re.search(r"\bextend\b", statement) is not None: + # treat extend as message + numMessage += 1 + + else: + # Check field names + if numMessage > 0: + matchName = re.search(r"\b\w[\S]*\b\s*=", statement) + if matchName is not None: + checkName = statement[matchName.start():matchName.end()-1] + # Check field message type (remove field name) + type = statement.replace(checkName, "") + matchName = re.search(r"\b\w[\S\.]*\s*=", type) + + # Search for a closing brace. + matchClosingBrace = re.search("}", statement) + if numMessage > 0 and matchClosingBrace is not None: + numMessage -= 1 + + + if matchComment is not None: + if comment != '': + for rulename, ruleregex in rules_dict.items(): + if re.search(ruleregex, comment): + rules.append(comment) + shiftCounter = True + + elif len(saveStatement) == 0: + if numMessage > 0 or isEnum == True: + if statement.find(";") != -1: + field = statement.strip().split()[2] + yml_file.write((2*numMessage)*" "+f"{field}:\n") + + if shiftCounter: + for rule in rules: + + rule_list = rule.split() + # Check if syntax + if "check_if" in rule_list: + yml_file.write((2*numMessage+2)*" "+f"- {rule_list[0]}:\n") + yml_file.write((2*numMessage+4)*" "+f"- {rule_list[2]}: {rule_list[3]}\n") + yml_file.write((2*numMessage+6)*" "+f"target: {rule_list[1]}\n") + yml_file.write((2*numMessage+4)*" "+f"{rule_list[5]}:\n") + yml_file.write((2*numMessage+4)*" "+f"- {rule_list[6]}:\n") + + # First element syntax + elif "first_element" in rule_list: + yml_file.write((2*numMessage+2)*" "+f"- {rule_list[0]}:\n") + yml_file.write((2*numMessage+6)*" "+f"{rule_list[1]}:\n") + yml_file.write((2*numMessage+8)*" "+f"- {rule_list[2]}: {rule_list[3]}\n") + + # Last element syntax + elif "last_element" in rule_list: + yml_file.write((2*numMessage+2)*" "+f"- {rule_list[0]}:\n") + yml_file.write((2*numMessage+6)*" "+f"{rule_list[1]}:\n") + yml_file.write((2*numMessage+8)*" "+f"- {rule_list[2]}: {rule_list[3]}\n") + elif "is_globally_unique" in rule_list: + yml_file.write((2*numMessage+2)*" "+f"-{rule}:\n") + else: + yml_file.write((2*numMessage+2)*" "+f"-{rule}\n") + + shiftCounter = False + rules = [] From 4357bb43c726365a43b2fd6e739eccde7af550f2 Mon Sep 17 00:00:00 2001 From: Viktor Kreschenski Date: Mon, 13 Jan 2020 12:26:25 +0100 Subject: [PATCH 2/2] Updated travis ci with parsed rules --- .travis.yml | 23 +++- rules2yml.py | 344 ++++++++++++++++++++++++++------------------------- 2 files changed, 198 insertions(+), 169 deletions(-) diff --git a/.travis.yml b/.travis.yml index c7d8c56..38aca0d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,6 +40,9 @@ script: - cd .. - pip install . + # Generate parsed rules + python rules2yml.py -d rules + # Check if rule syntax in osi is correct # - python test_cases.py @@ -49,16 +52,30 @@ script: # Show validator usage - osivalidator -h - # Run validator on a small test + # Run validator on a small test with already existing rules - osivalidator data/small_test.osi.lzma - osivalidator data/small_test.osi.lzma -p - osivalidator data/small_test.txt.lzma -f separated - osivalidator data/small_test.txt.lzma -f separated -p - # Decompress both traces and then run the validator again + # Run validator on a small test with parsed rules + - osivalidator data/small_test.osi.lzma -r rules + - osivalidator data/small_test.osi.lzma -p -r rules + - osivalidator data/small_test.txt.lzma -f separated -r rules + - osivalidator data/small_test.txt.lzma -f separated -p -r rules + + # Decompress both traces - lzma -d data/small_test.osi.lzma - lzma -d data/small_test.txt.lzma + + # Run the validator on decompressed data - osivalidator data/small_test.osi - osivalidator data/small_test.osi -p - osivalidator data/small_test.txt -f separated - - osivalidator data/small_test.txt -f separated -p + - osivalidator data/small_test.txt -f separated -p + + # Run the validator on decompressed data with parsed rules + - osivalidator data/small_test.osi -r rules + - osivalidator data/small_test.osi -p -r rules + - osivalidator data/small_test.txt -f separated -r rules + - osivalidator data/small_test.txt -f separated -p -r rules diff --git a/rules2yml.py b/rules2yml.py index 003918d..403011d 100644 --- a/rules2yml.py +++ b/rules2yml.py @@ -1,179 +1,191 @@ import sys import unicodedata +import argparse import re from glob import * import os +import yaml +def command_line_arguments(): + """ Define and handle command line interface """ -# Define grammar rules for rule definitions -rules_dict = {'in_range': r'\b(in_range)\b: \[\d+(\.\d+)?, \d+(\.\d+)?\]', - 'is_greater_than': r'\b(is_greater_than)\b: \d+(\.\d+)?', - 'is_greater_than_or_equal_to': r'\b(is_greater_than_or_equal_to)\b', - 'is_less_than_or_equal_to': r'\b(is_less_than_or_equal_to)\b', - 'is_less_than': r'\b(is_less_than)\b: \d+(\.\d+)?', - 'is_equal_to': r'\b(is_equal_to)\b', - 'is_different_to': r'\b(is_different_to)\b \d+(\.\d+)?', - 'is_globally_unique': r'\b(is_globally_unique)\b', - 'refers_to': r'\b(refers_to)\b', - 'is_iso_country_code': r'\b(is_iso_country_code)\b', - 'first_element': r'\b(first_element)\b', - 'last_element': r'\b(last_element)\b', - 'is_optional': r'\b(is_optional)\b', - 'check_if': r'\b(check_if)\b \[\{.*: \d+(\.\d+)?, target: .*}, \{do_check: \{.*: \d+(\.\d+)?}}]'} - -rule_dir = "testreq" - -if not os.path.exists(rule_dir): - os.makedirs(rule_dir) - -for file in glob("open-simulation-interface/*.proto*"): - filename = file.split("open-simulation-interface/")[1].split(".proto")[0] - - if os.path.exists(f"{rule_dir}/{filename}.yml"): - continue - - with open(f"{rule_dir}/{filename}.yml", "a") as yml_file: - with open(file, "rt") as fin: - isEnum = False - enumName = "" - numMessage = 0 - shiftCounter = False - saveStatement = "" - rules = [] - - for line in fin: - if file.find(".proto") != -1: - - # Search for comment ("//"). - matchComment = re.search("//", line) - if matchComment is not None: - statement = line[:matchComment.start()] - comment = line[matchComment.end():] - else: - statement = line - comment = "" - - # Add part of the statement from last line. - statement = saveStatement + " " + statement - saveStatement = "" - - # New line is not necessary. Remove for a better output. - statement = statement.replace("\n", "") - comment = comment.replace("\n", "") - - # Is statement complete - matchSep = re.search(r"[{};]", statement) - if matchSep is None: - saveStatement = statement - statement = "" - else: - saveStatement = statement[matchSep.end():] - statement = statement[:matchSep.end()] - - if isEnum is True: - matchName = re.search(r"\b\w[\S:]+\b", statement) - if matchName is not None: - checkName = statement[matchName.start():matchName.end()] - - # Search for "enum". - matchEnum = re.search(r"\benum\b", statement) - if matchEnum is not None: - isEnum = True - # print(f"Matched enum {isEnum}") - endOfLine = statement[matchEnum.end():] - matchName = re.search(r"\b\w[\S]*\b", endOfLine) - if matchName is not None: - # Test case 8: Check name - no special char - matchNameConv = re.search(r"\b[A-Z][a-zA-Z0-9]*\b",endOfLine[matchName.start():matchName.end()]) - enumName = convert(endOfLine[matchName.start():matchName.end()]) + "_" - - # Search for a closing brace. - matchClosingBrace = re.search("}", statement) - if isEnum is True and matchClosingBrace is not None: - isEnum = False - enumName = "" - continue - - def convert(name): - s1 = re.sub(r'(.)([A-Z][a-z]+)', r'\1_\2', name) - return re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', s1).upper() - - # Check if not inside an enum. - if isEnum is False: - # Search for "message". - matchMessage = re.search(r"\bmessage\b", statement) - if matchMessage is not None: - # a new message or a new nested message - numMessage += 1 - endOfLine = statement[matchMessage.end():] - matchName = re.search(r"\b\w[\S]*\b", endOfLine) - if matchName is not None: - # Test case 10: Check name - no special char - - # start with a capital letter - matchNameConv = re.search(r"\b[A-Z][a-zA-Z0-9]*\b",endOfLine[matchName.start():matchName.end()]) - # print(matchNameConv.group(0)) - yml_file.write(2*(numMessage-1)*" "+f"{matchNameConv.group(0)}:\n") + dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + + parser = argparse.ArgumentParser( + description='Export the rules of *.proto files into the *.yml format so it can be used by the validator.', + prog='python3 rules2yml.py') + parser.add_argument('--dir', '-d', + help='Name of the directory where the yml rules will be stored.', + default='rules', + required=False, + type=str) + + return parser.parse_args() + +def gen_yml_rules(dir_name='rules'): - elif re.search(r"\bextend\b", statement) is not None: - # treat extend as message - numMessage += 1 + with open(r'open-simulation-interface/rules.yml') as file: + rules_dict = yaml.load(file, Loader=yaml.FullLoader) + if not os.path.exists(dir_name): + os.makedirs(dir_name) + + for file in glob("open-simulation-interface/*.proto*"): + filename = file.split("open-simulation-interface/")[1].split(".proto")[0] + + if os.path.exists(f"{dir_name}/{filename}.yml"): + continue + + with open(f"{dir_name}/{filename}.yml", "a") as yml_file: + with open(file, "rt") as fin: + isEnum = False + enumName = "" + numMessage = 0 + shiftCounter = False + saveStatement = "" + rules = [] + + for line in fin: + if file.find(".proto") != -1: + + # Search for comment ("//"). + matchComment = re.search("//", line) + if matchComment is not None: + statement = line[:matchComment.start()] + comment = line[matchComment.end():] else: - # Check field names - if numMessage > 0: - matchName = re.search(r"\b\w[\S]*\b\s*=", statement) - if matchName is not None: - checkName = statement[matchName.start():matchName.end()-1] - # Check field message type (remove field name) - type = statement.replace(checkName, "") - matchName = re.search(r"\b\w[\S\.]*\s*=", type) + statement = line + comment = "" + + # Add part of the statement from last line. + statement = saveStatement + " " + statement + saveStatement = "" + + # New line is not necessary. Remove for a better output. + statement = statement.replace("\n", "") + comment = comment.replace("\n", "") + + # Is statement complete + matchSep = re.search(r"[{};]", statement) + if matchSep is None: + saveStatement = statement + statement = "" + else: + saveStatement = statement[matchSep.end():] + statement = statement[:matchSep.end()] + + if isEnum is True: + matchName = re.search(r"\b\w[\S:]+\b", statement) + if matchName is not None: + checkName = statement[matchName.start():matchName.end()] + + # Search for "enum". + matchEnum = re.search(r"\benum\b", statement) + if matchEnum is not None: + isEnum = True + # print(f"Matched enum {isEnum}") + endOfLine = statement[matchEnum.end():] + matchName = re.search(r"\b\w[\S]*\b", endOfLine) + if matchName is not None: + # Test case 8: Check name - no special char + matchNameConv = re.search(r"\b[A-Z][a-zA-Z0-9]*\b",endOfLine[matchName.start():matchName.end()]) + enumName = convert(endOfLine[matchName.start():matchName.end()]) + "_" # Search for a closing brace. matchClosingBrace = re.search("}", statement) - if numMessage > 0 and matchClosingBrace is not None: - numMessage -= 1 - - - if matchComment is not None: - if comment != '': - for rulename, ruleregex in rules_dict.items(): - if re.search(ruleregex, comment): - rules.append(comment) - shiftCounter = True - - elif len(saveStatement) == 0: - if numMessage > 0 or isEnum == True: - if statement.find(";") != -1: - field = statement.strip().split()[2] - yml_file.write((2*numMessage)*" "+f"{field}:\n") - - if shiftCounter: - for rule in rules: - - rule_list = rule.split() - # Check if syntax - if "check_if" in rule_list: - yml_file.write((2*numMessage+2)*" "+f"- {rule_list[0]}:\n") - yml_file.write((2*numMessage+4)*" "+f"- {rule_list[2]}: {rule_list[3]}\n") - yml_file.write((2*numMessage+6)*" "+f"target: {rule_list[1]}\n") - yml_file.write((2*numMessage+4)*" "+f"{rule_list[5]}:\n") - yml_file.write((2*numMessage+4)*" "+f"- {rule_list[6]}:\n") - - # First element syntax - elif "first_element" in rule_list: - yml_file.write((2*numMessage+2)*" "+f"- {rule_list[0]}:\n") - yml_file.write((2*numMessage+6)*" "+f"{rule_list[1]}:\n") - yml_file.write((2*numMessage+8)*" "+f"- {rule_list[2]}: {rule_list[3]}\n") - - # Last element syntax - elif "last_element" in rule_list: - yml_file.write((2*numMessage+2)*" "+f"- {rule_list[0]}:\n") - yml_file.write((2*numMessage+6)*" "+f"{rule_list[1]}:\n") - yml_file.write((2*numMessage+8)*" "+f"- {rule_list[2]}: {rule_list[3]}\n") - elif "is_globally_unique" in rule_list: - yml_file.write((2*numMessage+2)*" "+f"-{rule}:\n") - else: - yml_file.write((2*numMessage+2)*" "+f"-{rule}\n") - - shiftCounter = False - rules = [] + if isEnum is True and matchClosingBrace is not None: + isEnum = False + enumName = "" + continue + + def convert(name): + s1 = re.sub(r'(.)([A-Z][a-z]+)', r'\1_\2', name) + return re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', s1).upper() + + # Check if not inside an enum. + if isEnum is False: + # Search for "message". + matchMessage = re.search(r"\bmessage\b", statement) + if matchMessage is not None: + # a new message or a new nested message + numMessage += 1 + endOfLine = statement[matchMessage.end():] + matchName = re.search(r"\b\w[\S]*\b", endOfLine) + if matchName is not None: + # Test case 10: Check name - no special char - + # start with a capital letter + matchNameConv = re.search(r"\b[A-Z][a-zA-Z0-9]*\b",endOfLine[matchName.start():matchName.end()]) + # print(matchNameConv.group(0)) + yml_file.write(2*(numMessage-1)*" "+f"{matchNameConv.group(0)}:\n") + + elif re.search(r"\bextend\b", statement) is not None: + # treat extend as message + numMessage += 1 + + else: + # Check field names + if numMessage > 0: + matchName = re.search(r"\b\w[\S]*\b\s*=", statement) + if matchName is not None: + checkName = statement[matchName.start():matchName.end()-1] + # Check field message type (remove field name) + type = statement.replace(checkName, "") + matchName = re.search(r"\b\w[\S\.]*\s*=", type) + + # Search for a closing brace. + matchClosingBrace = re.search("}", statement) + if numMessage > 0 and matchClosingBrace is not None: + numMessage -= 1 + + + if matchComment is not None: + if comment != '': + for rulename, ruleregex in rules_dict.items(): + if re.search(ruleregex, comment): + rules.append(comment) + shiftCounter = True + + elif len(saveStatement) == 0: + if numMessage > 0 or isEnum == True: + if statement.find(";") != -1: + field = statement.strip().split()[2] + yml_file.write((2*numMessage)*" "+f"{field}:\n") + + if shiftCounter: + for rule in rules: + + rule_list = rule.split() + # Check if syntax + if "check_if" in rule_list: + yml_file.write((2*numMessage+2)*" "+f"- {rule_list[0]}:\n") + yml_file.write((2*numMessage+4)*" "+f"- {rule_list[2]}: {rule_list[3]}\n") + yml_file.write((2*numMessage+6)*" "+f"target: {rule_list[1]}\n") + yml_file.write((2*numMessage+4)*" "+f"{rule_list[5]}:\n") + yml_file.write((2*numMessage+4)*" "+f"- {rule_list[6]}:\n") + + # First element syntax + elif "first_element" in rule_list: + yml_file.write((2*numMessage+2)*" "+f"- {rule_list[0]}:\n") + yml_file.write((2*numMessage+6)*" "+f"{rule_list[1]}:\n") + yml_file.write((2*numMessage+8)*" "+f"- {rule_list[2]}: {rule_list[3]}\n") + + # Last element syntax + elif "last_element" in rule_list: + yml_file.write((2*numMessage+2)*" "+f"- {rule_list[0]}:\n") + yml_file.write((2*numMessage+6)*" "+f"{rule_list[1]}:\n") + yml_file.write((2*numMessage+8)*" "+f"- {rule_list[2]}: {rule_list[3]}\n") + elif "is_globally_unique" in rule_list: + yml_file.write((2*numMessage+2)*" "+f"-{rule}:\n") + else: + yml_file.write((2*numMessage+2)*" "+f"-{rule}\n") + + shiftCounter = False + rules = [] + +def main(): + # Handling of command line arguments + args = command_line_arguments() + gen_yml_rules(args.dir) + +if __name__ == "__main__": + main() \ No newline at end of file