diff --git a/.travis.yml b/.travis.yml index c7d8c56..38aca0d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,6 +40,9 @@ script: - cd .. - pip install . + # Generate parsed rules + python rules2yml.py -d rules + # Check if rule syntax in osi is correct # - python test_cases.py @@ -49,16 +52,30 @@ script: # Show validator usage - osivalidator -h - # Run validator on a small test + # Run validator on a small test with already existing rules - osivalidator data/small_test.osi.lzma - osivalidator data/small_test.osi.lzma -p - osivalidator data/small_test.txt.lzma -f separated - osivalidator data/small_test.txt.lzma -f separated -p - # Decompress both traces and then run the validator again + # Run validator on a small test with parsed rules + - osivalidator data/small_test.osi.lzma -r rules + - osivalidator data/small_test.osi.lzma -p -r rules + - osivalidator data/small_test.txt.lzma -f separated -r rules + - osivalidator data/small_test.txt.lzma -f separated -p -r rules + + # Decompress both traces - lzma -d data/small_test.osi.lzma - lzma -d data/small_test.txt.lzma + + # Run the validator on decompressed data - osivalidator data/small_test.osi - osivalidator data/small_test.osi -p - osivalidator data/small_test.txt -f separated - - osivalidator data/small_test.txt -f separated -p + - osivalidator data/small_test.txt -f separated -p + + # Run the validator on decompressed data with parsed rules + - osivalidator data/small_test.osi -r rules + - osivalidator data/small_test.osi -p -r rules + - osivalidator data/small_test.txt -f separated -r rules + - osivalidator data/small_test.txt -f separated -p -r rules diff --git a/rules2yml.py b/rules2yml.py new file mode 100644 index 0000000..403011d --- /dev/null +++ b/rules2yml.py @@ -0,0 +1,191 @@ +import sys +import unicodedata +import argparse +import re +from glob import * +import os +import yaml + +def command_line_arguments(): + """ Define and handle command line interface """ + + dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + + parser = argparse.ArgumentParser( + description='Export the rules of *.proto files into the *.yml format so it can be used by the validator.', + prog='python3 rules2yml.py') + parser.add_argument('--dir', '-d', + help='Name of the directory where the yml rules will be stored.', + default='rules', + required=False, + type=str) + + return parser.parse_args() + +def gen_yml_rules(dir_name='rules'): + + with open(r'open-simulation-interface/rules.yml') as file: + rules_dict = yaml.load(file, Loader=yaml.FullLoader) + + if not os.path.exists(dir_name): + os.makedirs(dir_name) + + for file in glob("open-simulation-interface/*.proto*"): + filename = file.split("open-simulation-interface/")[1].split(".proto")[0] + + if os.path.exists(f"{dir_name}/{filename}.yml"): + continue + + with open(f"{dir_name}/{filename}.yml", "a") as yml_file: + with open(file, "rt") as fin: + isEnum = False + enumName = "" + numMessage = 0 + shiftCounter = False + saveStatement = "" + rules = [] + + for line in fin: + if file.find(".proto") != -1: + + # Search for comment ("//"). + matchComment = re.search("//", line) + if matchComment is not None: + statement = line[:matchComment.start()] + comment = line[matchComment.end():] + else: + statement = line + comment = "" + + # Add part of the statement from last line. + statement = saveStatement + " " + statement + saveStatement = "" + + # New line is not necessary. Remove for a better output. + statement = statement.replace("\n", "") + comment = comment.replace("\n", "") + + # Is statement complete + matchSep = re.search(r"[{};]", statement) + if matchSep is None: + saveStatement = statement + statement = "" + else: + saveStatement = statement[matchSep.end():] + statement = statement[:matchSep.end()] + + if isEnum is True: + matchName = re.search(r"\b\w[\S:]+\b", statement) + if matchName is not None: + checkName = statement[matchName.start():matchName.end()] + + # Search for "enum". + matchEnum = re.search(r"\benum\b", statement) + if matchEnum is not None: + isEnum = True + # print(f"Matched enum {isEnum}") + endOfLine = statement[matchEnum.end():] + matchName = re.search(r"\b\w[\S]*\b", endOfLine) + if matchName is not None: + # Test case 8: Check name - no special char + matchNameConv = re.search(r"\b[A-Z][a-zA-Z0-9]*\b",endOfLine[matchName.start():matchName.end()]) + enumName = convert(endOfLine[matchName.start():matchName.end()]) + "_" + + # Search for a closing brace. + matchClosingBrace = re.search("}", statement) + if isEnum is True and matchClosingBrace is not None: + isEnum = False + enumName = "" + continue + + def convert(name): + s1 = re.sub(r'(.)([A-Z][a-z]+)', r'\1_\2', name) + return re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', s1).upper() + + # Check if not inside an enum. + if isEnum is False: + # Search for "message". + matchMessage = re.search(r"\bmessage\b", statement) + if matchMessage is not None: + # a new message or a new nested message + numMessage += 1 + endOfLine = statement[matchMessage.end():] + matchName = re.search(r"\b\w[\S]*\b", endOfLine) + if matchName is not None: + # Test case 10: Check name - no special char - + # start with a capital letter + matchNameConv = re.search(r"\b[A-Z][a-zA-Z0-9]*\b",endOfLine[matchName.start():matchName.end()]) + # print(matchNameConv.group(0)) + yml_file.write(2*(numMessage-1)*" "+f"{matchNameConv.group(0)}:\n") + + elif re.search(r"\bextend\b", statement) is not None: + # treat extend as message + numMessage += 1 + + else: + # Check field names + if numMessage > 0: + matchName = re.search(r"\b\w[\S]*\b\s*=", statement) + if matchName is not None: + checkName = statement[matchName.start():matchName.end()-1] + # Check field message type (remove field name) + type = statement.replace(checkName, "") + matchName = re.search(r"\b\w[\S\.]*\s*=", type) + + # Search for a closing brace. + matchClosingBrace = re.search("}", statement) + if numMessage > 0 and matchClosingBrace is not None: + numMessage -= 1 + + + if matchComment is not None: + if comment != '': + for rulename, ruleregex in rules_dict.items(): + if re.search(ruleregex, comment): + rules.append(comment) + shiftCounter = True + + elif len(saveStatement) == 0: + if numMessage > 0 or isEnum == True: + if statement.find(";") != -1: + field = statement.strip().split()[2] + yml_file.write((2*numMessage)*" "+f"{field}:\n") + + if shiftCounter: + for rule in rules: + + rule_list = rule.split() + # Check if syntax + if "check_if" in rule_list: + yml_file.write((2*numMessage+2)*" "+f"- {rule_list[0]}:\n") + yml_file.write((2*numMessage+4)*" "+f"- {rule_list[2]}: {rule_list[3]}\n") + yml_file.write((2*numMessage+6)*" "+f"target: {rule_list[1]}\n") + yml_file.write((2*numMessage+4)*" "+f"{rule_list[5]}:\n") + yml_file.write((2*numMessage+4)*" "+f"- {rule_list[6]}:\n") + + # First element syntax + elif "first_element" in rule_list: + yml_file.write((2*numMessage+2)*" "+f"- {rule_list[0]}:\n") + yml_file.write((2*numMessage+6)*" "+f"{rule_list[1]}:\n") + yml_file.write((2*numMessage+8)*" "+f"- {rule_list[2]}: {rule_list[3]}\n") + + # Last element syntax + elif "last_element" in rule_list: + yml_file.write((2*numMessage+2)*" "+f"- {rule_list[0]}:\n") + yml_file.write((2*numMessage+6)*" "+f"{rule_list[1]}:\n") + yml_file.write((2*numMessage+8)*" "+f"- {rule_list[2]}: {rule_list[3]}\n") + elif "is_globally_unique" in rule_list: + yml_file.write((2*numMessage+2)*" "+f"-{rule}:\n") + else: + yml_file.write((2*numMessage+2)*" "+f"-{rule}\n") + + shiftCounter = False + rules = [] + +def main(): + # Handling of command line arguments + args = command_line_arguments() + gen_yml_rules(args.dir) + +if __name__ == "__main__": + main() \ No newline at end of file