From 72fb660a7e042a9882ac3e9d4682d044dc347ada Mon Sep 17 00:00:00 2001 From: Viktor Kreschenski Date: Wed, 12 Feb 2020 17:22:57 +0100 Subject: [PATCH 1/7] Change testing trace --- data/small_test.txt.lzma | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/data/small_test.txt.lzma b/data/small_test.txt.lzma index 4259125..ca6f6f1 100644 --- a/data/small_test.txt.lzma +++ b/data/small_test.txt.lzma @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b572e8d38c1ff03c6e58168e99be5560ca310c1052a6e9c20f74432b46547314 -size 130010 +oid sha256:34a169163c86aa8ff1b8ef87881585b3ef166a776f37064dbd95d325271f825c +size 6836 diff --git a/setup.py b/setup.py index 0857df5..6434a65 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ install_requires=[ 'iso3166', 'ruamel.yaml', - 'PyYaml', + 'PyYaml==3.9.1', 'asteval', 'sphinx_rtd_theme', 'recommonmark', From 688444d9f125b4b9879a2d155536084f1cf50fae Mon Sep 17 00:00:00 2001 From: Viktor Kreschenski Date: Wed, 12 Feb 2020 17:43:45 +0100 Subject: [PATCH 2/7] Removed version definition PyYaml --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6434a65..0857df5 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ install_requires=[ 'iso3166', 'ruamel.yaml', - 'PyYaml==3.9.1', + 'PyYaml', 'asteval', 'sphinx_rtd_theme', 'recommonmark', From aa9b6888d349feb922a74726619ea5a4f2dd99e6 Mon Sep 17 00:00:00 2001 From: Viktor Kreschenski Date: Wed, 12 Feb 2020 18:53:13 +0100 Subject: [PATCH 3/7] Use trace with 50 messages --- data/small_test.txt.lzma | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/small_test.txt.lzma b/data/small_test.txt.lzma index ca6f6f1..9568ad3 100644 --- a/data/small_test.txt.lzma +++ b/data/small_test.txt.lzma @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:34a169163c86aa8ff1b8ef87881585b3ef166a776f37064dbd95d325271f825c -size 6836 +oid sha256:97d4efda5888898a82044e2093ee605bade75290038afaf9a03d7bceecc91d98 +size 442 From 05377e2244849940fa914c7f7efcfb4691a52e7d Mon Sep 17 00:00:00 2001 From: Viktor Kreschenski Date: Tue, 18 Feb 2020 19:19:55 +0100 Subject: [PATCH 4/7] Update osi version --- open-simulation-interface | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/open-simulation-interface b/open-simulation-interface index c853a1c..c287831 160000 --- a/open-simulation-interface +++ b/open-simulation-interface @@ -1 +1 @@ -Subproject commit c853a1ca2686598b3c1b8c088b9bb5557b722f4a +Subproject commit c2878311f628bf394ed612417e41a9c2371b4f41 From 697205e18b601e7932305f1f5fa8f425de93d6fc Mon Sep 17 00:00:00 2001 From: Viktor Kreschenski Date: Tue, 18 Feb 2020 19:33:40 +0100 Subject: [PATCH 5/7] Added black code formatter --- .travis.yml | 4 + osivalidator/linked_proto_field.py | 6 +- osivalidator/osi_doxygen_xml.py | 35 +- osivalidator/osi_general_validator.py | 157 +++++--- osivalidator/osi_id_manager.py | 58 +-- osivalidator/osi_rules.py | 71 ++-- osivalidator/osi_rules_checker.py | 5 +- osivalidator/osi_rules_implementations.py | 82 ++-- osivalidator/osi_trace.py | 137 +++++-- osivalidator/osi_validator_logger.py | 57 +-- pyosi/conversions.py | 2 +- pyosi/reading.py | 12 +- pyosi/testing_script.py | 2 +- rules2yml.py | 164 +++++--- setup.py | 53 +-- test_cases.py | 368 +++++++++++++----- tests/test_trace.py | 17 +- tests/test_validation_rules.py | 32 +- tests/tests_rules/test_check_if.py | 45 ++- tests/tests_rules/test_first_element_of.py | 54 +-- tests/tests_rules/test_is_globally_unique.py | 17 +- tests/tests_rules/test_is_greater_than.py | 56 +-- .../test_is_greater_than_or_equal_to.py | 56 +-- tests/tests_rules/test_is_iso_country_code.py | 8 +- tests/tests_rules/test_is_less_than.py | 56 +-- .../test_is_less_than_or_equal_to.py | 56 +-- tests/tests_rules/test_is_optional.py | 6 +- tests/tests_rules/test_is_set.py | 1 - tests/tests_rules/test_is_valid.py | 61 ++- tests/tests_rules/test_last_element_of.py | 57 +-- tests/tests_rules/test_refers_to.py | 33 +- 31 files changed, 1131 insertions(+), 637 deletions(-) diff --git a/.travis.yml b/.travis.yml index f6efef9..1ae0128 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,6 +34,10 @@ script: - python -m site - protoc --version + # Install black code formatter and check if it is applied + - pip install black + - black --check --exclude "(open-simulation-interface|proto2cpp)" . + # Install osi-validation - cd open-simulation-interface - pip install . diff --git a/osivalidator/linked_proto_field.py b/osivalidator/linked_proto_field.py index 1347f1c..c4f6da4 100644 --- a/osivalidator/linked_proto_field.py +++ b/osivalidator/linked_proto_field.py @@ -93,8 +93,10 @@ def get_field(self, field_name): field = getattr(self.value, field_name) - if (hasattr(self.value, 'DESCRIPTOR') and - self.value.DESCRIPTOR.fields_by_name[field_name].label == 3): + if ( + hasattr(self.value, "DESCRIPTOR") + and self.value.DESCRIPTOR.fields_by_name[field_name].label == 3 + ): return [ LinkedProtoField(u_field, parent=self, name=field_name) for u_field in field diff --git a/osivalidator/osi_doxygen_xml.py b/osivalidator/osi_doxygen_xml.py index ffcfe3d..3d782de 100644 --- a/osivalidator/osi_doxygen_xml.py +++ b/osivalidator/osi_doxygen_xml.py @@ -21,11 +21,10 @@ class OSIDoxygenXML: def __init__(self): dir_path = os.path.dirname(os.path.realpath(__file__)) osivalidator_path = os.path.dirname(dir_path) - self.osi_path = os.path.join( - osivalidator_path, 'open-simulation-interface') - self.osi_doc_path = os.path.join(self.osi_path, 'doc') + self.osi_path = os.path.join(osivalidator_path, "open-simulation-interface") + self.osi_doc_path = os.path.join(self.osi_path, "doc") - proto2cpp_path = os.path.join(osivalidator_path, 'proto2cpp') + proto2cpp_path = os.path.join(osivalidator_path, "proto2cpp") self.proto2cpp_file_path = os.path.join(proto2cpp_path, "proto2cpp.py") def generate_osi_doxygen_xml(self): @@ -33,7 +32,7 @@ def generate_osi_doxygen_xml(self): Generate the Doxygen XML documentation in the OSI path """ - configuration = f''' + configuration = f""" PROJECT_NAME = osi-validation INPUT = {self.osi_path} OUTPUT_DIRECTORY = {self.osi_doc_path} @@ -48,10 +47,10 @@ def generate_osi_doxygen_xml(self): XML_PROGRAMLISTING = NO ALIASES = rules="
"
         ALIASES               += endrules="
" - ''' + """ - doxyfile_path = os.path.join(self.osi_path, 'Doxyfile_validation') - doxyfile = open(doxyfile_path, 'w') + doxyfile_path = os.path.join(self.osi_path, "Doxyfile_validation") + doxyfile = open(doxyfile_path, "w") doxyfile.write(configuration) doxyfile.close() @@ -65,7 +64,7 @@ def get_files(self): """ Return the path of the fields in OSI """ - return glob.glob(os.path.join(self.osi_path, 'doc', 'xml', '*.xml')) + return glob.glob(os.path.join(self.osi_path, "doc", "xml", "*.xml")) def parse_rules(self): """ @@ -78,19 +77,20 @@ def parse_rules(self): for xml_file in xml_files: tree = ET.parse(xml_file) memberdefs = tree.findall( - "./compounddef/sectiondef/memberdef[@kind='variable']") + "./compounddef/sectiondef/memberdef[@kind='variable']" + ) for memberdef in memberdefs: - attr_path = memberdef.findtext( - 'definition').split()[-1].split('::') - if attr_path[0] != 'osi3': + attr_path = memberdef.findtext("definition").split()[-1].split("::") + if attr_path[0] != "osi3": continue attr_path.pop(0) attr_rule = memberdef.findtext( - './detaileddescription//preformatted[last()]') + "./detaileddescription//preformatted[last()]" + ) if not attr_rule: continue - rules_lines = attr_rule.split('\n') + rules_lines = attr_rule.split("\n") for line_no, line in enumerate(rules_lines): if line.find(":") == -1 and line: @@ -100,14 +100,13 @@ def parse_rules(self): try: dict_rules = yaml.safe_load(attr_rule) - except (yaml.parser.ParserError, - yaml.parser.ScannerError) as error: + except (yaml.parser.ParserError, yaml.parser.ScannerError) as error: print(attr_path, attr_rule, error) else: rules.append((attr_path, dict_rules)) return rules -if __name__ == '__main__': +if __name__ == "__main__": osidx = OSIDoxygenXML() osidx.generate_osi_doxygen_xml() diff --git a/osivalidator/osi_general_validator.py b/osivalidator/osi_general_validator.py index 69a8b55..2f61d70 100755 --- a/osivalidator/osi_general_validator.py +++ b/osivalidator/osi_general_validator.py @@ -13,70 +13,95 @@ from osivalidator.osi_trace import OSITrace from osivalidator.osi_rules_checker import OSIRulesChecker + def check_positive_int(value): ivalue = int(value) if ivalue < 0: raise argparse.ArgumentTypeError("%s is an invalid positive int value" % value) return ivalue + def command_line_arguments(): """ Define and handle command line interface """ dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) parser = argparse.ArgumentParser( - description='Validate data defined at the input', - prog='osivalidator') - parser.add_argument('data', - help='Path to the file with OSI-serialized data.', - type=str) - parser.add_argument('--rules', '-r', - help='Directory with text files containig rules. ', - default=os.path.join(dir_path, 'requirements-osi-3'), - type=str) - parser.add_argument('--type', '-t', - help='Name of the type used to serialize data.', - choices=['SensorView', 'GroundTruth', 'SensorData'], - default='SensorView', - type=str, - required=False) - parser.add_argument('--output', '-o', - help='Output folder of the log files.', - default='output_logs', - type=str, - required=False) - parser.add_argument('--timesteps', - help='Number of timesteps to analyze. If -1, all.', - type=int, - default=-1, - required=False) - parser.add_argument('--debug', - help='Set the debug mode to ON.', - action="store_true") - parser.add_argument('--verbose', '-v', - help='Set the verbose mode to ON.', - action="store_true") - parser.add_argument('--parallel', '-p', - help='Set parallel mode to ON.', - default=False, - required=False, - action="store_true") - parser.add_argument('--format', '-f', - help='Set the format type of the trace.', - choices=['separated', None], - default=None, - type=str, - required=False) - parser.add_argument('--blast', '-bl', - help='Set the in-memory storage count of OSI messages during validation.', - default=500, - type=check_positive_int, - required=False) - parser.add_argument('--buffer', '-bu', - help='Set the buffer size to retrieve OSI messages from trace file. Set it to 0 if you do not want to use buffering at all.', - default=1000000, - type=check_positive_int, - required=False) + description="Validate data defined at the input", prog="osivalidator" + ) + parser.add_argument( + "data", help="Path to the file with OSI-serialized data.", type=str + ) + parser.add_argument( + "--rules", + "-r", + help="Directory with text files containig rules. ", + default=os.path.join(dir_path, "requirements-osi-3"), + type=str, + ) + parser.add_argument( + "--type", + "-t", + help="Name of the type used to serialize data.", + choices=["SensorView", "GroundTruth", "SensorData"], + default="SensorView", + type=str, + required=False, + ) + parser.add_argument( + "--output", + "-o", + help="Output folder of the log files.", + default="output_logs", + type=str, + required=False, + ) + parser.add_argument( + "--timesteps", + help="Number of timesteps to analyze. If -1, all.", + type=int, + default=-1, + required=False, + ) + parser.add_argument( + "--debug", help="Set the debug mode to ON.", action="store_true" + ) + parser.add_argument( + "--verbose", "-v", help="Set the verbose mode to ON.", action="store_true" + ) + parser.add_argument( + "--parallel", + "-p", + help="Set parallel mode to ON.", + default=False, + required=False, + action="store_true", + ) + parser.add_argument( + "--format", + "-f", + help="Set the format type of the trace.", + choices=["separated", None], + default=None, + type=str, + required=False, + ) + parser.add_argument( + "--blast", + "-bl", + help="Set the in-memory storage count of OSI messages during validation.", + default=500, + type=check_positive_int, + required=False, + ) + parser.add_argument( + "--buffer", + "-bu", + help="Set the buffer size to retrieve OSI messages from trace file. Set it to 0 if you do not want to use buffering at all.", + default=1000000, + type=check_positive_int, + required=False, + ) return parser.parse_args() @@ -88,10 +113,11 @@ def command_line_arguments(): LOGGER = OSIValidatorLogger() VALIDATION_RULES = OSIRules() ID_TO_TS = MANAGER.dict() -BAR_SUFFIX = '%(index)d/%(max)d [%(elapsed_td)s]' -BAR = Bar('', suffix=BAR_SUFFIX) +BAR_SUFFIX = "%(index)d/%(max)d [%(elapsed_td)s]" +BAR = Bar("", suffix=BAR_SUFFIX) MESSAGE_CACHE = MANAGER.dict() + def main(): """Main method""" @@ -112,7 +138,12 @@ def main(): # Read data print("Reading data ...") DATA = OSITrace(buffer_size=args.buffer) - DATA.from_file(path=args.data, type_name=args.type, max_index=args.timesteps, format_type=args.format) + DATA.from_file( + path=args.data, + type_name=args.type, + max_index=args.timesteps, + format_type=args.format, + ) # Collect Validation Rules print("Collect validation rules ...") @@ -138,7 +169,7 @@ def main(): # Increment the max-timestep to analyze max_timestep_blast += args.blast - first_of_blast = (max_timestep_blast-args.blast) + first_of_blast = max_timestep_blast - args.blast last_of_blast = min(max_timestep_blast, max_timestep) # Cache messages @@ -148,9 +179,9 @@ def main(): if args.parallel: # Launch parallel computation # Recreate the pool - try: + try: pool = Pool() - pool.map(process_timestep, range(first_of_blast, last_of_blast)) + pool.map(process_timestep, range(first_of_blast, last_of_blast)) except Exception as e: print(str(e)) @@ -163,7 +194,7 @@ def main(): try: for i in range(first_of_blast, last_of_blast): process_timestep(i) - + except Exception as e: print(str(e)) @@ -176,6 +207,7 @@ def main(): # Synthetize LOGGER.synthetize_results_from_sqlite() + def close_pool(pool): """Cleanly close a pool to free the memory""" pool.close() @@ -192,7 +224,7 @@ def process_timestep(timestep): LOGGER.log_messages[timestep] = [] LOGGER.debug_messages[timestep] = [] - LOGGER.info(None, f'Analyze message of timestamp {timestamp}', False) + LOGGER.info(None, f"Analyze message of timestamp {timestamp}", False) # Check if timestamp already exists if timestamp in TIMESTAMP_ANALYZED: @@ -202,8 +234,9 @@ def process_timestep(timestep): BAR.goto(len(TIMESTAMP_ANALYZED)) # Check common rules - getattr(rule_checker, 'is_valid')( - message, VALIDATION_RULES.get_rules().get_type(MESSAGE_TYPE.value)) + getattr(rule_checker, "is_valid")( + message, VALIDATION_RULES.get_rules().get_type(MESSAGE_TYPE.value) + ) LOGS.extend(LOGGER.log_messages[timestep]) diff --git a/osivalidator/osi_id_manager.py b/osivalidator/osi_id_manager.py index 1a2da58..593690a 100644 --- a/osivalidator/osi_id_manager.py +++ b/osivalidator/osi_id_manager.py @@ -26,13 +26,15 @@ def __init__(self, logger): def get_all_messages_by_id(self, message_id, message_t=None): """Retrieve all the message by giving an id""" - return list(filter(lambda m: message_t_filter(m, message_t), - self._index[message_id])) + return list( + filter(lambda m: message_t_filter(m, message_t), self._index[message_id]) + ) def get_message_by_id(self, message_id, message_t=None): """Retrieve only one message by giving an id""" - return next(filter(lambda m: message_t_filter(m, message_t), - self._index[message_id])) + return next( + filter(lambda m: message_t_filter(m, message_t), self._index[message_id]) + ) def register_message(self, message_id, message): """Register one message in the ID manager""" @@ -55,49 +57,59 @@ def resolve_unicity(self, timestamp): types_counter = None if len(objects) > 1: types_list = list(map(type, objects)) - types_str_list = ", ".join( - map(lambda o: o.__name__, types_list)) + types_str_list = ", ".join(map(lambda o: o.__name__, types_list)) self.logger.warning( timestamp, - f"Several objects of type {types_str_list} have the ID " + - str(identifier)) + f"Several objects of type {types_str_list} have the ID " + + str(identifier), + ) if len(objects) != len(set(types_list)): types_counter = list(map(str, Counter(list(types_list)))) obj_list_str = ", ".join( - map(lambda t, c=types_counter: c[t] + " " + t.__name__, - filter(lambda t, c=types_counter: c[t] != 1, - types_counter))) + map( + lambda t, c=types_counter: c[t] + " " + t.__name__, + filter(lambda t, c=types_counter: c[t] != 1, types_counter), + ) + ) self.logger.error( timestamp, - f"Several objects of the same type have the ID " + - str(identifier) + ":" + obj_list_str) + f"Several objects of the same type have the ID " + + str(identifier) + + ":" + + obj_list_str, + ) def resolve_references(self, timestamp): """Check if references are compliant""" for reference in self._references: referer, identifier, expected_type, condition = reference try: - found_object = next(filter( - lambda o, et=expected_type: type(o).__name__ == et, - self._index[identifier])) + found_object = next( + filter( + lambda o, et=expected_type: type(o).__name__ == et, + self._index[identifier], + ) + ) except (StopIteration, KeyError): self.logger.error( timestamp, - f'Reference unresolved: {referer.DESCRIPTOR.name} ' + - f'to {expected_type} ' + - f'(ID: {identifier})') + f"Reference unresolved: {referer.DESCRIPTOR.name} " + + f"to {expected_type} " + + f"(ID: {identifier})", + ) else: self.logger.debug( - timestamp, f'Reference resolved: {expected_type} ' + - f'(ID: {identifier})') + timestamp, + f"Reference resolved: {expected_type} " + f"(ID: {identifier})", + ) if condition is not None: if condition(found_object): - self.logger.debug(timestamp, f'Condition OK') + self.logger.debug(timestamp, f"Condition OK") else: - self.logger.error(timestamp, f'Condition not OK') + self.logger.error(timestamp, f"Condition not OK") def reset(self): """Erase all data in the ID manager""" diff --git a/osivalidator/osi_rules.py b/osivalidator/osi_rules.py index 689e9c1..cb9138e 100644 --- a/osivalidator/osi_rules.py +++ b/osivalidator/osi_rules.py @@ -24,30 +24,28 @@ def from_yaml_directory(self, path=None): if not path: dir_path = dir_path = os.path.dirname(os.path.realpath(__file__)) - path = os.path.join(dir_path, 'requirements-osi-3') + path = os.path.join(dir_path, "requirements-osi-3") - exts = ('.yml', '.yaml') + exts = (".yml", ".yaml") try: for filename in os.listdir(path): - if filename.startswith('osi_') and filename.endswith(exts): + if filename.startswith("osi_") and filename.endswith(exts): self.from_yaml_file(os.path.join(path, filename)) except FileNotFoundError: - print('Error while reading files OSI-rules. Exiting!') + print("Error while reading files OSI-rules. Exiting!") def from_yaml_file(self, path): """Import from a file """ rules_file = open(path) - self.from_dict(rules_dict=yaml.load( - rules_file, Loader=yaml.SafeLoader)) + self.from_dict(rules_dict=yaml.load(rules_file, Loader=yaml.SafeLoader)) rules_file.close() def from_yaml(self, yaml_content): """Import from a string """ - self.from_dict(rules_dict=yaml.load( - yaml_content, Loader=yaml.SafeLoader)) + self.from_dict(rules_dict=yaml.load(yaml_content, Loader=yaml.SafeLoader)) def from_xml_doxygen(self): """Parse the Doxygen XML documentation to get the rules @@ -63,8 +61,7 @@ def from_xml_doxygen(self): message_t = self.rules.add_type_from_path(message_t_path) for field_rule in field_rules: - message_t.add_field(FieldRules( - name=field_name, rules=field_rule)) + message_t.add_field(FieldRules(name=field_name, rules=field_rule)) def get_rules(self): """Return the rules @@ -78,8 +75,7 @@ def from_dict(self, rules_dict=None, rules_container=None): for key, value in rules_dict.items(): if key[0].isupper() and isinstance(value, dict): # it's a nested type - new_message_t = rules_container.add_type( - MessageTypeRules(name=key)) + new_message_t = rules_container.add_type(MessageTypeRules(name=key)) if value is not None: self.from_dict(value, new_message_t) @@ -90,8 +86,8 @@ def from_dict(self, rules_dict=None, rules_container=None): elif value is not None: sys.stderr.write( - 'must be dict or list, got ' - + type(rules_dict).__name__ + '\n') + "must be dict or list, got " + type(rules_dict).__name__ + "\n" + ) class ProtoMessagePath: @@ -99,8 +95,7 @@ class ProtoMessagePath: def __init__(self, path=None): if path and not all(isinstance(component, str) for component in path): - sys.stderr.write( - 'Path must be str list, found ' + str(path) + '\n') + sys.stderr.write("Path must be str list, found " + str(path) + "\n") self.path = deepcopy(path) or [] def __repr__(self): @@ -111,8 +106,11 @@ def __getitem__(self, parent): def pretty_html(self): """Return a pretty html version of the message path""" - return ".".join(map(lambda l: ""+l+"", self.path[:-1])) \ - + "." + self.path[-1] + return ( + ".".join(map(lambda l: "" + l + "", self.path[:-1])) + + "." + + self.path[-1] + ) def child_path(self, child): """Return a new path for the child""" @@ -191,19 +189,20 @@ def get_type(self, message_path): try: message_t = message_t.nested_types[component] except KeyError: - raise KeyError('Type not found: ' + str(message_path)) + raise KeyError("Type not found: " + str(message_path)) return message_t if isinstance(message_path, str): return self.nested_types[message_path] - sys.stderr.write('Type must be ProtoMessagePath or str' + '\n') + sys.stderr.write("Type must be ProtoMessagePath or str" + "\n") def __getitem__(self, name): return self.nested_types[name] def __repr__(self): - return f'TypeContainer({len(self.nested_types)}):\n' + \ - ','.join(map(str, self.nested_types)) + return f"TypeContainer({len(self.nested_types)}):\n" + ",".join( + map(str, self.nested_types) + ) class MessageTypeRules(TypeRulesContainer): @@ -233,11 +232,12 @@ def __getitem__(self, field_name): return self.get_field(field_name) def __repr__(self): - return f'{self.type_name}:' + \ - f'MessageType({len(self.fields)}):{self.fields},' + \ - f'Nested types({len(self.nested_types)})' + \ - (':' + ','.join(self.nested_types.keys()) if self.nested_types - else '') + return ( + f"{self.type_name}:" + + f"MessageType({len(self.fields)}):{self.fields}," + + f"Nested types({len(self.nested_types)})" + + (":" + ",".join(self.nested_types.keys()) if self.nested_types else "") + ) class FieldRules(OSIRuleNode): @@ -320,7 +320,7 @@ def __init__(self, **kwargs): import osivalidator.osi_rules_implementations as rule_implementations if not hasattr(rule_implementations, self.verb): - sys.stderr.write(self.verb + ' rule does not exist\n') + sys.stderr.write(self.verb + " rule does not exist\n") def from_dict(self, rule_dict: dict): """Instantiate Rule object from a dictionary""" @@ -329,12 +329,11 @@ def from_dict(self, rule_dict: dict): self.verb = verb self.params = params self.extra_params = dict(extra_params) - self.target = self.extra_params.pop('target', None) + self.target = self.extra_params.pop("target", None) return True except AttributeError: - sys.stderr.write( - 'rule must be YAML mapping, got: ' + str(rule_dict) + '\n') + sys.stderr.write("rule must be YAML mapping, got: " + str(rule_dict) + "\n") return False @property @@ -344,7 +343,7 @@ def path(self): @property def targeted_field(self): if self.target: - return self.target.split('.')[-1] + return self.target.split(".")[-1] return self.field_name @path.setter @@ -359,12 +358,16 @@ def __repr__(self): return f"{self.verb}({self.params}) target={self.target}" def __eq__(self, other): - return (self.verb == other.verb and self.params == other.params - and self.severity == other.severity) + return ( + self.verb == other.verb + and self.params == other.params + and self.severity == other.severity + ) class Severity(Enum): """Description of the severity of the raised error if a rule does not comply.""" + INFO = 20 WARN = 30 ERROR = 40 diff --git a/osivalidator/osi_rules_checker.py b/osivalidator/osi_rules_checker.py index 17957df..716d2e2 100644 --- a/osivalidator/osi_rules_checker.py +++ b/osivalidator/osi_rules_checker.py @@ -38,7 +38,7 @@ def log(self, severity, message): elif isinstance(severity, str): severity_method = severity else: - raise TypeError('type not accepted: must be Severity enum or str') + raise TypeError("type not accepted: must be Severity enum or str") return getattr(self.logger, severity_method)(self.timestamp, message) @@ -53,8 +53,7 @@ def check_rule(self, parent_field, rule): try: rule_method = getattr(self, rule.verb) except AttributeError: - raise AttributeError( - 'Rule ' + rule.verb + ' not implemented yet\n') + raise AttributeError("Rule " + rule.verb + " not implemented yet\n") if rule.target is not None: parent_field = parent_field.query(rule.target, parent=True) diff --git a/osivalidator/osi_rules_implementations.py b/osivalidator/osi_rules_implementations.py index 27ac2ea..120c47b 100644 --- a/osivalidator/osi_rules_implementations.py +++ b/osivalidator/osi_rules_implementations.py @@ -15,18 +15,20 @@ def add_default_rules_to_subfields(message, type_rules): """Add default rules to fields of message fields (subfields) """ for descriptor in message.all_field_descriptors: - field_rules = (type_rules.get_field(descriptor.name) - if descriptor.name in type_rules.fields - else type_rules.add_field(FieldRules(descriptor.name))) + field_rules = ( + type_rules.get_field(descriptor.name) + if descriptor.name in type_rules.fields + else type_rules.add_field(FieldRules(descriptor.name)) + ) if descriptor.message_type: - field_rules.add_rule(Rule(verb='is_valid')) + field_rules.add_rule(Rule(verb="is_valid")) - is_set_severity = (Severity.WARN - if field_rules.has_rule('is_optional') - else Severity.ERROR) + is_set_severity = ( + Severity.WARN if field_rules.has_rule("is_optional") else Severity.ERROR + ) - field_rules.add_rule(Rule(verb='is_set', severity=is_set_severity)) + field_rules.add_rule(Rule(verb="is_set", severity=is_set_severity)) # DECORATORS @@ -54,12 +56,13 @@ def rule_implementation(func): """Decorator to label rules method implementations """ func.is_rule = True + @wraps(func) def wrapper(self, field, rule, **kwargs): if isinstance(rule, FieldRules): rule = rule.rules[func.__name__] - if isinstance(field, list) and not getattr(func, 'repeated_selector', False): + if isinstance(field, list) and not getattr(func, "repeated_selector", False): result = all([func(self, unique_field, rule) for unique_field in field]) else: result = func(self, field, rule, **kwargs) @@ -69,9 +72,15 @@ def wrapper(self, field, rule, **kwargs): path = field[0].path else: path = field.path - self.log(rule.severity, str(rule.path) - + '(' + str(rule.params) + ')' - + ' does not comply in ' + str(path)) + self.log( + rule.severity, + str(rule.path) + + "(" + + str(rule.params) + + ")" + + " does not comply in " + + str(path), + ) return result @@ -81,6 +90,7 @@ def wrapper(self, field, rule, **kwargs): # RULES # TODO Refactor this code into a seperate class so it can be easy parsed by sphinx + @rule_implementation def is_valid(self, field, rule): """Check if a field message is valid, that is all the inner rules of the @@ -234,14 +244,16 @@ def first_element(self, field, rule): # Convert parsed yaml file to dictonary rules nested_fields_rules_list = [] for key_field, nested_rule in nested_fields_rules.items(): - nested_rule[0].update({'target': 'this.'+key_field}) + nested_rule[0].update({"target": "this." + key_field}) nested_fields_rules_list.append(nested_rule[0]) rules_checker_list = [] for nested_fields_rule in nested_fields_rules_list: - statement_rule = Rule(dictionary=nested_fields_rule, - field_name=rule.field_name, - severity=Severity.ERROR) + statement_rule = Rule( + dictionary=nested_fields_rule, + field_name=rule.field_name, + severity=Severity.ERROR, + ) statement_rule.path = rule.path.child_path(statement_rule.verb) statement_true = self.check_rule(field[0], statement_rule) and statement_true rules_checker_list.append(statement_true) @@ -264,14 +276,16 @@ def last_element(self, field, rule): # Convert parsed yaml file to dictonary rules nested_fields_rules_list = [] for key_field, nested_rule in nested_fields_rules.items(): - nested_rule[0].update({'target': 'this.'+key_field}) + nested_rule[0].update({"target": "this." + key_field}) nested_fields_rules_list.append(nested_rule[0]) rules_checker_list = [] for nested_fields_rule in nested_fields_rules_list: - statement_rule = Rule(dictionary=nested_fields_rule, - field_name=rule.field_name, - severity=Severity.ERROR) + statement_rule = Rule( + dictionary=nested_fields_rule, + field_name=rule.field_name, + severity=Severity.ERROR, + ) statement_rule.path = rule.path.child_path(statement_rule.verb) statement_true = self.check_rule(field[-1], statement_rule) and statement_true rules_checker_list.append(statement_true) @@ -329,23 +343,31 @@ def check_if(self, field, rule): """ statements = rule.params - do_checks = rule.extra_params['do_check'] + do_checks = rule.extra_params["do_check"] statement_true = True # Check if all the statements are true for statement in statements: - statement_rule = Rule(dictionary=statement, - field_name=rule.field_name, - severity=Severity.INFO) + statement_rule = Rule( + dictionary=statement, field_name=rule.field_name, severity=Severity.INFO + ) statement_rule.path = rule.path.child_path(statement_rule.verb) - statement_true = self.check_rule( - field, statement_rule) and statement_true + statement_true = self.check_rule(field, statement_rule) and statement_true # If the statements are true, check the do_check rules if not statement_true: return True - return all((self.check_rule(field, Rule(path=rule.path.child_path(next(iter(check.keys()))), - dictionary=check, - field_name=rule.field_name)) - for check in do_checks)) + return all( + ( + self.check_rule( + field, + Rule( + path=rule.path.child_path(next(iter(check.keys()))), + dictionary=check, + field_name=rule.field_name, + ), + ) + for check in do_checks + ) + ) diff --git a/osivalidator/osi_trace.py b/osivalidator/osi_trace.py index ff9002e..f4a285c 100644 --- a/osivalidator/osi_trace.py +++ b/osivalidator/osi_trace.py @@ -12,11 +12,12 @@ from osi3.osi_groundtruth_pb2 import GroundTruth from osi3.osi_sensordata_pb2 import SensorData import warnings -warnings.simplefilter('default') + +warnings.simplefilter("default") from osivalidator.linked_proto_field import LinkedProtoField -SEPARATOR = b'$$__$$' +SEPARATOR = b"$$__$$" SEPARATOR_LENGTH = len(SEPARATOR) @@ -34,14 +35,16 @@ def get_size_from_file_stream(file_object): MESSAGES_TYPE = { "SensorView": SensorView, "GroundTruth": GroundTruth, - "SensorData": SensorData + "SensorData": SensorData, } class OSITrace: """This class wrap OSI data. It can import and decode OSI traces.""" - def __init__(self, buffer_size, show_progress=True, path=None, type_name="SensorView"): + def __init__( + self, buffer_size, show_progress=True, path=None, type_name="SensorView" + ): self.trace_file = None self.message_offsets = None self.buffer_size = buffer_size @@ -56,7 +59,7 @@ def __init__(self, buffer_size, show_progress=True, path=None, type_name="Sensor # Open and Read text file def from_file(self, path, type_name="SensorView", max_index=-1, format_type=None): """Import a trace from a file""" - if path.lower().endswith(('.lzma', '.xz')): + if path.lower().endswith((".lzma", ".xz")): self.trace_file = lzma.open(path, "rb") else: self.trace_file = open(path, "rb") @@ -64,8 +67,11 @@ def from_file(self, path, type_name="SensorView", max_index=-1, format_type=None self.type_name = type_name self.format_type = format_type - if self.format_type == 'separated': - warnings.warn("The separated trace files will be completely removed in the near future. Please convert them to *.osi files with the converter in the main OSI repository.", PendingDeprecationWarning) + if self.format_type == "separated": + warnings.warn( + "The separated trace files will be completely removed in the near future. Please convert them to *.osi files with the converter in the main OSI repository.", + PendingDeprecationWarning, + ) self.timestep_count = self.retrieve_message_offsets(max_index) else: self.timestep_count = self.retrieve_message() @@ -81,7 +87,11 @@ def retrieve_message(self): if self.show_progress: progress_bar = Bar(max=trace_size) - print("Retrieving messages in osi trace file until " + str(trace_size) + " ...") + print( + "Retrieving messages in osi trace file until " + + str(trace_size) + + " ..." + ) else: progress_bar = None @@ -91,11 +101,11 @@ def retrieve_message(self): start_time = time.time() self.trace_file.seek(0) - + self.message_offsets = [0] message_offset = 0 - message_length = 0 - counter = 0 # Counter is needed to enable correct buffer parsing of serialized messages + message_length = 0 + counter = 0 # Counter is needed to enable correct buffer parsing of serialized messages # Check if user decided to use buffer if self.buffer_size != 0 and type(self.buffer_size) == int: @@ -106,9 +116,17 @@ def retrieve_message(self): self.trace_file.seek(self.message_offsets[-1]) while not eof: - + # Unpack the message size relative to the current buffer - message_length = struct.unpack(" self.buffer_size * (counter + 1) - + # Check if reached end of file if self.trace_file.tell() == trace_size: self.retrieved_trace_size = self.message_offsets[-1] - self.message_offsets.pop() # Remove the last element since after that there is no message coming + self.message_offsets.pop() # Remove the last element since after that there is no message coming break while eof: # Counter increment and cursor placement update. The cursor is set absolute in the file. - if message_offset >= len(serialized_message): - self.update_bar(progress_bar, message_offset) - counter += 1 - self.trace_file.seek(counter*self.buffer_size) + if message_offset >= len(serialized_message): + self.update_bar(progress_bar, message_offset) + counter += 1 + self.trace_file.seek(counter * self.buffer_size) eof = False else: serialized_message = self.trace_file.read() - while message_offset < trace_size: - message_length = struct.unpack(" 0: matchName = re.search(r"\b\w[\S]*\b\s*=", statement) if matchName is not None: - checkName = statement[matchName.start():matchName.end()-1] + checkName = statement[ + matchName.start() : matchName.end() - 1 + ] # Check field message type (remove field name) type = statement.replace(checkName, "") matchName = re.search(r"\b\w[\S\.]*\s*=", type) @@ -137,19 +161,20 @@ def convert(name): if numMessage > 0 and matchClosingBrace is not None: numMessage -= 1 - - if matchComment is not None: - if comment != '': + if matchComment is not None: + if comment != "": for rulename, ruleregex in rules_dict.items(): if re.search(ruleregex, comment): - rules.append(comment) + rules.append(comment) shiftCounter = True - elif len(saveStatement) == 0: + elif len(saveStatement) == 0: if numMessage > 0 or isEnum == True: if statement.find(";") != -1: field = statement.strip().split()[2] - yml_file.write((2*numMessage)*" "+f"{field}:\n") + yml_file.write( + (2 * numMessage) * " " + f"{field}:\n" + ) if shiftCounter: for rule in rules: @@ -157,35 +182,76 @@ def convert(name): rule_list = rule.split() # Check if syntax if "check_if" in rule_list: - yml_file.write((2*numMessage+2)*" "+f"- {rule_list[0]}:\n") - yml_file.write((2*numMessage+4)*" "+f"- {rule_list[2]}: {rule_list[3]}\n") - yml_file.write((2*numMessage+6)*" "+f"target: {rule_list[1]}\n") - yml_file.write((2*numMessage+4)*" "+f"{rule_list[5]}:\n") - yml_file.write((2*numMessage+4)*" "+f"- {rule_list[6]}:\n") + yml_file.write( + (2 * numMessage + 2) * " " + + f"- {rule_list[0]}:\n" + ) + yml_file.write( + (2 * numMessage + 4) * " " + + f"- {rule_list[2]}: {rule_list[3]}\n" + ) + yml_file.write( + (2 * numMessage + 6) * " " + + f"target: {rule_list[1]}\n" + ) + yml_file.write( + (2 * numMessage + 4) * " " + + f"{rule_list[5]}:\n" + ) + yml_file.write( + (2 * numMessage + 4) * " " + + f"- {rule_list[6]}:\n" + ) # First element syntax elif "first_element" in rule_list: - yml_file.write((2*numMessage+2)*" "+f"- {rule_list[0]}:\n") - yml_file.write((2*numMessage+6)*" "+f"{rule_list[1]}:\n") - yml_file.write((2*numMessage+8)*" "+f"- {rule_list[2]}: {rule_list[3]}\n") + yml_file.write( + (2 * numMessage + 2) * " " + + f"- {rule_list[0]}:\n" + ) + yml_file.write( + (2 * numMessage + 6) * " " + + f"{rule_list[1]}:\n" + ) + yml_file.write( + (2 * numMessage + 8) * " " + + f"- {rule_list[2]}: {rule_list[3]}\n" + ) # Last element syntax elif "last_element" in rule_list: - yml_file.write((2*numMessage+2)*" "+f"- {rule_list[0]}:\n") - yml_file.write((2*numMessage+6)*" "+f"{rule_list[1]}:\n") - yml_file.write((2*numMessage+8)*" "+f"- {rule_list[2]}: {rule_list[3]}\n") + yml_file.write( + (2 * numMessage + 2) * " " + + f"- {rule_list[0]}:\n" + ) + yml_file.write( + (2 * numMessage + 6) * " " + + f"{rule_list[1]}:\n" + ) + yml_file.write( + (2 * numMessage + 8) * " " + + f"- {rule_list[2]}: {rule_list[3]}\n" + ) elif "is_globally_unique" in rule_list: - yml_file.write((2*numMessage+2)*" "+f"-{rule}:\n") + yml_file.write( + (2 * numMessage + 2) * " " + + f"-{rule}:\n" + ) else: - yml_file.write((2*numMessage+2)*" "+f"-{rule}\n") - + yml_file.write( + (2 * numMessage + 2) * " " + + f"-{rule}\n" + ) + shiftCounter = False rules = [] + def main(): # Handling of command line arguments args = command_line_arguments() gen_yml_rules(args.dir) + if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/setup.py b/setup.py index 0857df5..eb113d8 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ import setuptools -AUTHOR = 'Altran Germany / BMW' +AUTHOR = "Altran Germany / BMW" if __name__ == "__main__": with open("README.md", "r") as fh: @@ -26,35 +26,38 @@ "Operating System :: OS Independent", ], data_files=[ - ('open-simulation-interface', - glob.glob('open-simulation-interface/*.proto')), - ('proto2cpp', ['proto2cpp/proto2cpp.py']), - ('lib/python3.6/site-packages/requirements-osi-3', - glob.glob('requirements-osi-3/*.yml')) + ( + "open-simulation-interface", + glob.glob("open-simulation-interface/*.proto"), + ), + ("proto2cpp", ["proto2cpp/proto2cpp.py"]), + ( + "lib/python3.6/site-packages/requirements-osi-3", + glob.glob("requirements-osi-3/*.yml"), + ), ], include_package_data=True, install_requires=[ - 'iso3166', - 'ruamel.yaml', - 'PyYaml', - 'asteval', - 'sphinx_rtd_theme', - 'recommonmark', - 'open-simulation-interface', - 'doxygen-interface', - 'defusedxml', - 'colorama', - 'tabulate', - 'progress', - 'protobuf==3.9.1' + "iso3166", + "ruamel.yaml", + "PyYaml", + "asteval", + "sphinx_rtd_theme", + "recommonmark", + "open-simulation-interface", + "doxygen-interface", + "defusedxml", + "colorama", + "tabulate", + "progress", + "protobuf==3.9.1", ], dependency_links=[ - 'git+https://github.com/OpenSimulationInterface/' + - 'open-simulation-interface.git' + - '@master#egg=open-simulation-interface', + "git+https://github.com/OpenSimulationInterface/" + + "open-simulation-interface.git" + + "@master#egg=open-simulation-interface", ], entry_points={ - 'console_scripts': - ['osivalidator=osivalidator.osi_general_validator:main'], - } + "console_scripts": ["osivalidator=osivalidator.osi_general_validator:main"], + }, ) diff --git a/test_cases.py b/test_cases.py index 0c71a56..1f1f9c5 100644 --- a/test_cases.py +++ b/test_cases.py @@ -6,21 +6,23 @@ state = 0 # Define grammar rules for rule definitions -rules_dict = {'in_range': r'\b(in_range)\b: \[\d+(\.\d+)?, \d+(\.\d+)?\]', - 'is_greater_than': r'\b(is_greater_than)\b: \d+(\.\d+)?', - 'is_greater_than_or_equal_to': r'\b(is_greater_than_or_equal_to)\b: \d+(\.\d+)?', - 'is_less_than_or_equal_to': r'\b(is_less_than_or_equal_to)\b: \d+(\.\d+)?', - 'is_less_than': r'\b(is_less_than)\b: \d+(\.\d+)?', - 'is_equal_to': r'\b(is_equal_to)\b: \d+(\.\d+)?', - 'is_different_to': r'\b(is_different_to)\b: \d+(\.\d+)?', - 'is_globally_unique': r'\b(is_globally_unique)\b', - 'refers_to': r'\b(refers_to)\b', - 'is_iso_country_code': r'\b(is_iso_country_code)\b', - 'first_element': r'\b(first_element)\b: \{.*: \d+\.\d+\}', - 'last_element': r'\b(last_element)\b: \{.*: \d+\.\d+\}', - 'is_optional': r'\b(is_optional)\b', - 'check_if': r'\b(check_if)\b: \[\{.*: \d+(\.\d+)?, target: .*}, \{do_check: \{.*: \d+(\.\d+)?}}]', - 'is_set': r'\b(is_set)\b'} +rules_dict = { + "in_range": r"\b(in_range)\b: \[\d+(\.\d+)?, \d+(\.\d+)?\]", + "is_greater_than": r"\b(is_greater_than)\b: \d+(\.\d+)?", + "is_greater_than_or_equal_to": r"\b(is_greater_than_or_equal_to)\b: \d+(\.\d+)?", + "is_less_than_or_equal_to": r"\b(is_less_than_or_equal_to)\b: \d+(\.\d+)?", + "is_less_than": r"\b(is_less_than)\b: \d+(\.\d+)?", + "is_equal_to": r"\b(is_equal_to)\b: \d+(\.\d+)?", + "is_different_to": r"\b(is_different_to)\b: \d+(\.\d+)?", + "is_globally_unique": r"\b(is_globally_unique)\b", + "refers_to": r"\b(refers_to)\b", + "is_iso_country_code": r"\b(is_iso_country_code)\b", + "first_element": r"\b(first_element)\b: \{.*: \d+\.\d+\}", + "last_element": r"\b(last_element)\b: \{.*: \d+\.\d+\}", + "is_optional": r"\b(is_optional)\b", + "check_if": r"\b(check_if)\b: \[\{.*: \d+(\.\d+)?, target: .*}, \{do_check: \{.*: \d+(\.\d+)?}}]", + "is_set": r"\b(is_set)\b", +} for file in glob("open-simulation-interface/*.proto"): @@ -47,13 +49,24 @@ # -------------------------------------------------------------- # Test case 2 is checking if there is an "Umlaut" etc. - if (sys.version_info >= (3, 0)): - if line != unicodedata.normalize('NFKD', line).encode('ASCII', 'ignore').decode(): - print(file + " in line " + str(i) + ": a none ASCII char is present") + if sys.version_info >= (3, 0): + if ( + line + != unicodedata.normalize("NFKD", line) + .encode("ASCII", "ignore") + .decode() + ): + print( + file + " in line " + str(i) + ": a none ASCII char is present" + ) state = 1 else: - if line != unicodedata.normalize('NFKD', unicode(line, 'ISO-8859-1')).encode('ASCII', 'ignore'): - print(file + " in line " + str(i) + ": a none ASCII char is present") + if line != unicodedata.normalize( + "NFKD", unicode(line, "ISO-8859-1") + ).encode("ASCII", "ignore"): + print( + file + " in line " + str(i) + ": a none ASCII char is present" + ) state = 1 if file.find(".proto") != -1: @@ -87,29 +100,29 @@ # Search for comment ("//"). matchComment = re.search("//", line) if matchComment is not None: - statement = line[:matchComment.start()] - comment = line[matchComment.end():] + statement = line[: matchComment.start()] + comment = line[matchComment.end() :] else: statement = line comment = "" - + # Add part of the statement from last line. statement = saveStatement + " " + statement saveStatement = "" - + # New line is not necessary. Remove for a better output. statement = statement.replace("\n", "") comment = comment.replace("\n", "") - # Is statement complete + # Is statement complete matchSep = re.search(r"[{};]", statement) if matchSep is None: saveStatement = statement statement = "" else: - saveStatement = statement[matchSep.end():] - statement = statement[:matchSep.end()] - + saveStatement = statement[matchSep.end() :] + statement = statement[: matchSep.end()] + # -------------------------------------------------------------- # Test case 6-8 camelcase for enums and check enum name? @@ -117,29 +130,58 @@ if isEnum is True: matchName = re.search(r"\b\w[\S:]+\b", statement) if matchName is not None: - checkName = statement[matchName.start():matchName.end()] + checkName = statement[matchName.start() : matchName.end()] # Test case 6: Check correct name if checkName.find(enumName) != 0: - print(file + " in line " + str(i) + ": enum type wrong. '"+checkName+"' should start with '"+enumName+"'") + print( + file + + " in line " + + str(i) + + ": enum type wrong. '" + + checkName + + "' should start with '" + + enumName + + "'" + ) state = 1 # Test case 7: Check upper case elif checkName != checkName.upper(): - print(file + " in line " + str(i) + ": enum type wrong. '"+checkName+"' should use upper case") + print( + file + + " in line " + + str(i) + + ": enum type wrong. '" + + checkName + + "' should use upper case" + ) state = 1 # Search for "enum". matchEnum = re.search(r"\benum\b", statement) if matchEnum is not None: isEnum = True - endOfLine = statement[matchEnum.end():] + endOfLine = statement[matchEnum.end() :] matchName = re.search(r"\b\w[\S]*\b", endOfLine) if matchName is not None: # Test case 8: Check name - no special char - matchNameConv = re.search(r"\b[A-Z][a-zA-Z0-9]*\b",endOfLine[matchName.start():matchName.end()]) + matchNameConv = re.search( + r"\b[A-Z][a-zA-Z0-9]*\b", + endOfLine[matchName.start() : matchName.end()], + ) if matchNameConv is None: - print(file + " in line " + str(i) + ": enum name wrong. '"+endOfLine[matchName.start():matchName.end()]+"'") + print( + file + + " in line " + + str(i) + + ": enum name wrong. '" + + endOfLine[matchName.start() : matchName.end()] + + "'" + ) state = 1 - enumName = convert(endOfLine[matchName.start():matchName.end()])+"_" + enumName = ( + convert(endOfLine[matchName.start() : matchName.end()]) + + "_" + ) # Search for a closing brace. matchClosingBrace = re.search("}", statement) @@ -148,8 +190,8 @@ enumName = "" def convert(name): - s1 = re.sub(r'(.)([A-Z][a-z]+)', r'\1_\2', name) - return re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', s1).upper() + s1 = re.sub(r"(.)([A-Z][a-z]+)", r"\1_\2", name) + return re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", s1).upper() # -------------------------------------------------------------- # Test case 10-12,18 check message name, field type and field name @@ -164,14 +206,24 @@ def convert(name): if matchMessage is not None: # a new message or a new nested message noMessage += 1 - endOfLine = statement[matchMessage.end():] + endOfLine = statement[matchMessage.end() :] matchName = re.search(r"\b\w[\S]*\b", endOfLine) if matchName is not None: # Test case 10: Check name - no special char - # start with a capital letter - matchNameConv = re.search(r"\b[A-Z][a-zA-Z0-9]*\b",endOfLine[matchName.start():matchName.end()]) + matchNameConv = re.search( + r"\b[A-Z][a-zA-Z0-9]*\b", + endOfLine[matchName.start() : matchName.end()], + ) if matchNameConv is None: - print(file + " in line " + str(i) + ": message name wrong. '"+endOfLine[matchName.start():matchName.end()]+"'") + print( + file + + " in line " + + str(i) + + ": message name wrong. '" + + endOfLine[matchName.start() : matchName.end()] + + "'" + ) state = 1 elif re.search(r"\bextend\b", statement) is not None: # treat extend as message @@ -181,25 +233,58 @@ def convert(name): if noMessage > 0: matchName = re.search(r"\b\w[\S]*\b\s*=", statement) if matchName is not None: - checkName = statement[matchName.start():matchName.end()-1] + checkName = statement[ + matchName.start() : matchName.end() - 1 + ] # Test case 11: Check lowercase letters for field names if checkName != checkName.lower(): - print(file + " in line " + str(i) + ": field name wrong. '"+checkName+"' should use lower case") + print( + file + + " in line " + + str(i) + + ": field name wrong. '" + + checkName + + "' should use lower case" + ) state = 1 # Check field message type (remove field name) type = statement.replace(checkName, "") matchName = re.search(r"\b\w[\S\.]*\s*=", type) if matchName is not None: - checkType = " "+type[matchName.start():matchName.end()-1]+" " + checkType = ( + " " + + type[matchName.start() : matchName.end() - 1] + + " " + ) # Test case 12: Check nested message type - matchNameConv = re.search(r"[ ][a-zA-Z][a-zA-Z0-9]*([\.][A-Z][a-zA-Z0-9]*)*[ ]",checkType) + matchNameConv = re.search( + r"[ ][a-zA-Z][a-zA-Z0-9]*([\.][A-Z][a-zA-Z0-9]*)*[ ]", + checkType, + ) if matchNameConv is None: - print(file + " in line " + str(i) + ": field message type wrong. Check: '"+checkType+"'") + print( + file + + " in line " + + str(i) + + ": field message type wrong. Check: '" + + checkType + + "'" + ) state = 1 - - if re.search(r"\boptional\b",type) is None and re.search(r"\brepeated\b",type) is None: + + if ( + re.search(r"\boptional\b", type) is None + and re.search(r"\brepeated\b", type) is None + ): # Test 18 has every field the multiplicity "repeated" or "optional" - print(file + " in line " + str(i) + ": field multiplicity (\"optional\" or \"repeated\") is missing. Check: '"+statement+"'") + print( + file + + " in line " + + str(i) + + ': field multiplicity ("optional" or "repeated") is missing. Check: \'' + + statement + + "'" + ) state = 1 # Search for a closing brace. @@ -216,36 +301,74 @@ def convert(name): elif len(saveStatement) == 0: # Test case 13 is checking if comment is min. 2 lines if noComment == 1: - print(file + " in line " + str(i-1) + ": short comment - min. 2 lines.") + print( + file + + " in line " + + str(i - 1) + + ": short comment - min. 2 lines." + ) state = 1 - if re.search(r"\bmessage\b", statement) is not None or re.search(r"\bextend\b", statement) is not None: + if ( + re.search(r"\bmessage\b", statement) is not None + or re.search(r"\bextend\b", statement) is not None + ): if hasBrief == False: # Test case 14 each message and extend has a \brief comment - print(file + " in line " + str(i-1) + ": \\brief section in comment is missing for: '"+statement+"'") + print( + file + + " in line " + + str(i - 1) + + ": \\brief section in comment is missing for: '" + + statement + + "'" + ) state = 1 elif hasBrief == True: # Test case 15 only message and extend has a \brief comment - print(file + " in line " + str(i-1) + ": \\brief section in comment is not necessary for: '"+statement+"'") + print( + file + + " in line " + + str(i - 1) + + ": \\brief section in comment is not necessary for: '" + + statement + + "'" + ) state = 1 - - if re.search(r"\bmessage\b", statement) is not None or re.search(r"\bextend\b", statement) is not None or re.search(r"\benum\b", statement) is not None: + + if ( + re.search(r"\bmessage\b", statement) is not None + or re.search(r"\bextend\b", statement) is not None + or re.search(r"\benum\b", statement) is not None + ): if noComment == 0: # Test case 16 every message, extend or enum has a comment - print(file + " in line " + str(i) + ": comment is missing for: '"+statement+"'") + print( + file + + " in line " + + str(i) + + ": comment is missing for: '" + + statement + + "'" + ) state = 1 if noMessage > 0 or isEnum == True: if statement.find(";") != -1: if noComment == 0: # Test case 17 every statement has a comment - print(file + " in line " + str(i) + ": comment is missing for: '"+statement+"'") + print( + file + + " in line " + + str(i) + + ": comment is missing for: '" + + statement + + "'" + ) state = 1 - noComment = 0 hasBrief = False - # -------------------------------------------------------------- # Test case 20 is checking comment and html tags if matchComment is not None: @@ -255,48 +378,75 @@ def convert(name): matchHTMLOnly = re.search(r"\\htmlonly", comment) if matchHTMLOnly is not None: - htmlComment = comment[matchHTMLOnly.end():] - htmlFreeComment = comment[:matchHTMLOnly.start()] + htmlComment = comment[matchHTMLOnly.end() :] + htmlFreeComment = comment[: matchHTMLOnly.start()] htmlblock = True else: htmlComment = comment htmlFreeComment = "" - + if htmlblock is True: matchEndHTMLOnly = re.search(r"\\endhtmlonly", htmlComment) if matchEndHTMLOnly is not None: - htmlFreeComment = htmlFreeComment+htmlComment[matchEndHTMLOnly.end():] - htmlComment = htmlComment[:matchEndHTMLOnly.start()] + htmlFreeComment = ( + htmlFreeComment + htmlComment[matchEndHTMLOnly.end() :] + ) + htmlComment = htmlComment[: matchEndHTMLOnly.start()] htmlblock = False - - #if htmlFreeComment.find("<") != -1: - # Test case 20 html tags only in htmlonly sections --> no error - #print(file + " in line " + str(i) + ": doxygen comment html tag found (use htmlonly if possible): '"+htmlFreeComment+"'") - ##state = 1 + + # if htmlFreeComment.find("<") != -1: + # Test case 20 html tags only in htmlonly sections --> no error + # print(file + " in line " + str(i) + ": doxygen comment html tag found (use htmlonly if possible): '"+htmlFreeComment+"'") + ##state = 1 if htmlComment.find("\\") != -1: # Test case 23 html tags only in htmlonly sections - print(file + " in line " + str(i) + ": doxygen comment \\.. reference found: '"+htmlComment+"'") - #state = 1 + print( + file + + " in line " + + str(i) + + ": doxygen comment \\.. reference found: '" + + htmlComment + + "'" + ) + # state = 1 if htmlComment.find("#") != -1: # Test case 24 html tags only in htmlonly sections - print(file + " in line " + str(i) + ": doxygen comment #.. reference found: '"+htmlComment+"'") - #state = 1 - + print( + file + + " in line " + + str(i) + + ": doxygen comment #.. reference found: '" + + htmlComment + + "'" + ) + # state = 1 + elif htmlblock is True: # Test case 22 html tags only in htmlonly sections without end html only - print(file + " in line " + str(i-1) + ": doxygen comment html section without endhtmlonly") + print( + file + + " in line " + + str(i - 1) + + ": doxygen comment html section without endhtmlonly" + ) htmlblock = False - #state = 1 - + # state = 1 # -------------------------------------------------------------- # Test case 21 is checking comment and html tags if matchComment is not None: if comment.find("@") != -1: # Test case 21 html tags only in htmlonly sections - print(file + " in line " + str(i) + ": @ tag found (please replace with \\): '"+htmlFreeComment+"'") + print( + file + + " in line " + + str(i) + + ": @ tag found (please replace with \\): '" + + htmlFreeComment + + "'" + ) state = 1 - + # -------------------------------------------------------------- # Test case 25 is checking if each field has a rule and must be set if isEnum is False: @@ -309,52 +459,88 @@ def convert(name): lineruleCount = -1 foundruleCount = -1 - if re.search(r'\b(is_set)\b', comment): + if re.search(r"\b(is_set)\b", comment): isSet = True # foundruleCount += 1 # TODO There also needs to be a checking for inner rules for check_if and first_element - if not endRule and comment != '': + if not endRule and comment != "": foundRule = False for rulename, ruleregex in rules_dict.items(): if re.search(ruleregex, comment): foundRule = True foundruleCount += 1 - elif len(saveStatement) == 0: if noMessage > 0 or isEnum == True: if statement.find(";") != -1: statement = statement.strip() if not hasRule and not endRule: # Test case 17 every statement has a comment - print(file + " in line " + str(i) + ": rule is missing for: '"+statement+"'") + print( + file + + " in line " + + str(i) + + ": rule is missing for: '" + + statement + + "'" + ) state = 1 if hasRule and not isSet and endRule: - print(file + " in line " + str(i) + ": rule is_set is missing for: '"+statement+"'") + print( + file + + " in line " + + str(i) + + ": rule is_set is missing for: '" + + statement + + "'" + ) state = 1 - if hasRule and lineruleCount != foundruleCount and endRule and lineruleCount-foundruleCount-1>0: - print(file + " in line " + str(i) + ": "+str(lineruleCount-foundruleCount-1)+" defined rule(s) does not exists for: '"+statement+"'") + if ( + hasRule + and lineruleCount != foundruleCount + and endRule + and lineruleCount - foundruleCount - 1 > 0 + ): + print( + file + + " in line " + + str(i) + + ": " + + str(lineruleCount - foundruleCount - 1) + + " defined rule(s) does not exists for: '" + + statement + + "'" + ) state = 1 - if hasRule and lineruleCount > foundruleCount and not endRule: - print(file + " in line " + str(i) + ": endrules statement does not exists for: '"+statement+"'") + if ( + hasRule + and lineruleCount > foundruleCount + and not endRule + ): + print( + file + + " in line " + + str(i) + + ": endrules statement does not exists for: '" + + statement + + "'" + ) state = 1 - + foundRule = False hasRule = False endRule = False isSet = False - if hasRule and not endRule: lineruleCount += 1 # -------------------------------------------------------------- # Next Test 26 - - + # Test case 19 last line must end with a new line. if hasNewLine == False: print(file + " has no new line at the end of the file.") diff --git a/tests/test_trace.py b/tests/test_trace.py index 66bf4d7..ea4b9e9 100644 --- a/tests/test_trace.py +++ b/tests/test_trace.py @@ -9,17 +9,20 @@ class TestDataContainer(unittest.TestCase): def setUp(self): self.MESSAGE_LENGTH = 15 - + self.txt = OSITrace(buffer_size=1000000) self.osi = OSITrace(buffer_size=1000000) self.osi_nobuffer = OSITrace(buffer_size=0) - self.txt.from_file(path="data/small_test.txt.lzma", - type_name="SensorView", format_type='separated') - self.osi.from_file(path="data/small_test.osi.lzma", - type_name="SensorView") - self.osi_nobuffer.from_file(path="data/small_test.osi.lzma", - type_name="SensorView") + self.txt.from_file( + path="data/small_test.txt.lzma", + type_name="SensorView", + format_type="separated", + ) + self.osi.from_file(path="data/small_test.osi.lzma", type_name="SensorView") + self.osi_nobuffer.from_file( + path="data/small_test.osi.lzma", type_name="SensorView" + ) def tearDown(self): self.txt.trace_file.close() diff --git a/tests/test_validation_rules.py b/tests/test_validation_rules.py index c68ec9d..b2a99e0 100644 --- a/tests/test_validation_rules.py +++ b/tests/test_validation_rules.py @@ -1,7 +1,13 @@ """Module for test class of OSIValidationRules class""" import unittest -from osivalidator.osi_rules import Rule, TypeRulesContainer, ProtoMessagePath, OSIRules, OSIRuleNode +from osivalidator.osi_rules import ( + Rule, + TypeRulesContainer, + ProtoMessagePath, + OSIRules, + OSIRuleNode, +) class TestValidationRules(unittest.TestCase): @@ -11,21 +17,21 @@ class TestValidationRules(unittest.TestCase): def test_from_directory(self): """Test import from directory""" ovr = OSIRules() - ovr.from_yaml_directory('requirements-osi-3') - test_path = ProtoMessagePath(['LaneBoundary', 'BoundaryPoint']) + ovr.from_yaml_directory("requirements-osi-3") + test_path = ProtoMessagePath(["LaneBoundary", "BoundaryPoint"]) ovr_container = ovr.rules.get_type(test_path) self.assertIsInstance(ovr_container.path, ProtoMessagePath) self.assertEqual(test_path.path, ovr_container.path.path) def test_creation_node(self): """Test creation of a node in OSI rules""" - node = OSIRuleNode('foo') - self.assertEqual(node.path, 'foo') + node = OSIRuleNode("foo") + self.assertEqual(node.path, "foo") def test_add_type_from_path(self): """Test the adding of a Message type from a path in the rule tree""" container = TypeRulesContainer() - path = ProtoMessagePath(['foo', 'bar', 'type']) + path = ProtoMessagePath(["foo", "bar", "type"]) container.add_type_from_path(path) typecontainer = container.get_type(path) self.assertIsInstance(typecontainer.path, ProtoMessagePath) @@ -43,13 +49,15 @@ def test_parse_yaml(self): validation_rules = OSIRules() validation_rules.from_yaml(raw) rules = validation_rules.rules - field = rules['HostVehicleData'].get_field('location') - rule_check = Rule(verb='is_set', - field_name="location", - path=ProtoMessagePath(["HostVehicleData", "location", "is_set"])) + field = rules["HostVehicleData"].get_field("location") + rule_check = Rule( + verb="is_set", + field_name="location", + path=ProtoMessagePath(["HostVehicleData", "location", "is_set"]), + ) - self.assertEqual(field['is_set'], rule_check) + self.assertEqual(field["is_set"], rule_check) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/tests_rules/test_check_if.py b/tests/tests_rules/test_check_if.py index 9ac9481..b9a741c 100644 --- a/tests/tests_rules/test_check_if.py +++ b/tests/tests_rules/test_check_if.py @@ -41,36 +41,51 @@ def setUp(self): self.VECTOR3D = LinkedProtoField(pb_VECTOR3D, "Vector3D") def test_comply1(self): - rule = Rule(verb="check_if", field_name='x', - params=[{'is_equal_to': 3, 'target': 'this.y'}], - extra_params={'do_check': [{'is_equal_to': 2}]}) + rule = Rule( + verb="check_if", + field_name="x", + params=[{"is_equal_to": 3, "target": "this.y"}], + extra_params={"do_check": [{"is_equal_to": 2}]}, + ) compliance = self.FRC.check_if(self.VECTOR3D, rule) self.assertTrue(compliance) def test_comply2(self): - rule = Rule(verb="check_if", field_name='x', - params=[{'is_equal_to': 2, 'target': 'this.y'}], - extra_params={'do_check': [{'is_equal_to': 1}]}) + rule = Rule( + verb="check_if", + field_name="x", + params=[{"is_equal_to": 2, "target": "this.y"}], + extra_params={"do_check": [{"is_equal_to": 1}]}, + ) compliance = self.FRC.check_if(self.VECTOR3D, rule) self.assertTrue(compliance) def test_comply_is_set(self): - rule = Rule(verb="check_if", field_name='x', - params=[{'is_equal_to': 2, 'target': 'this.y'}], - extra_params={'do_check': [{'is_set': None}]}) + rule = Rule( + verb="check_if", + field_name="x", + params=[{"is_equal_to": 2, "target": "this.y"}], + extra_params={"do_check": [{"is_set": None}]}, + ) compliance = self.FRC.check_if(self.VECTOR3D, rule) self.assertTrue(compliance) def test_not_comply(self): - rule = Rule(verb="check_if", field_name='x', - params=[{'is_equal_to': 2, 'target': 'this.y'}], - extra_params={'do_check': [{'is_equal_to': 2}]}) + rule = Rule( + verb="check_if", + field_name="x", + params=[{"is_equal_to": 2, "target": "this.y"}], + extra_params={"do_check": [{"is_equal_to": 2}]}, + ) compliance = self.FRC.check_if(self.VECTOR3D, rule) self.assertFalse(compliance) def test_not_comply_is_set_if(self): - rule = Rule(verb="check_if", field_name='z', - params=[{'is_equal_to': 2, 'target': 'this.y'}], - extra_params={'do_check': [{'is_set': None}]}) + rule = Rule( + verb="check_if", + field_name="z", + params=[{"is_equal_to": 2, "target": "this.y"}], + extra_params={"do_check": [{"is_set": None}]}, + ) compliance = self.FRC.check_if(self.VECTOR3D, rule) self.assertFalse(compliance) diff --git a/tests/tests_rules/test_first_element_of.py b/tests/tests_rules/test_first_element_of.py index 1b9b170..d1c5b98 100644 --- a/tests/tests_rules/test_first_element_of.py +++ b/tests/tests_rules/test_first_element_of.py @@ -10,14 +10,15 @@ class TestFirstElement(unittest.TestCase): - def setUp(self): self.FRC = OSIRulesChecker() sv1 = SensorView() linked_sv1 = LinkedProtoField(sv1, name="SensorView") gt1 = sv1.global_ground_truth - linked_gt1 = LinkedProtoField(gt1, name="global_ground_truth", parent=linked_sv1) + linked_gt1 = LinkedProtoField( + gt1, name="global_ground_truth", parent=linked_sv1 + ) gtlb1 = gt1.lane_boundary.add() linked_gtlb1 = LinkedProtoField(gtlb1, name="lane_boundary", parent=linked_gt1) @@ -30,14 +31,16 @@ def setUp(self): bladd1.height = 0.0 self.lb1 = LinkedProtoField(bladd1, name="boundary_line", parent=linked_gtlb1) - self.lb1.path = 'SensorView.global_ground_truth.lane_boundary.boundary_line' + self.lb1.path = "SensorView.global_ground_truth.lane_boundary.boundary_line" # self.lb1.parent = sv2 = SensorView() linked_sv2 = LinkedProtoField(sv2, name="SensorView") gt2 = sv2.global_ground_truth - linked_gt2 = LinkedProtoField(gt2, name="global_ground_truth", parent=linked_sv2) + linked_gt2 = LinkedProtoField( + gt2, name="global_ground_truth", parent=linked_sv2 + ) gtlb2 = gt2.lane_boundary.add() linked_gtlb2 = LinkedProtoField(gtlb2, name="lane_boundary", parent=linked_gt2) @@ -49,7 +52,7 @@ def setUp(self): bladd2.width = 0.14 bladd2.height = 0.13 self.lb2 = LinkedProtoField(bladd2, name="boundary_line", parent=linked_gtlb2) - self.lb2.path = 'SensorView.global_ground_truth.lane_boundary.boundary_line' + self.lb2.path = "SensorView.global_ground_truth.lane_boundary.boundary_line" def tearDown(self): del self.FRC @@ -57,16 +60,17 @@ def tearDown(self): def test_comply_first_element(self): field_list = [self.lb1, self.lb2] container = TypeRulesContainer() - proto_path = ProtoMessagePath(['LaneBoundary', 'BoundaryPoint']) + proto_path = ProtoMessagePath(["LaneBoundary", "BoundaryPoint"]) container.add_type_from_path(proto_path) - container.add_type_from_path(ProtoMessagePath(['Vector3d'])) - - rule = Rule(verb="first_element", - params={'width': [{'is_equal_to': 0.13}], - 'height': [{'is_equal_to': 0.0}]}, - path=proto_path, - extra_params=dict(), - field_name='boundary_line') + container.add_type_from_path(ProtoMessagePath(["Vector3d"])) + + rule = Rule( + verb="first_element", + params={"width": [{"is_equal_to": 0.13}], "height": [{"is_equal_to": 0.0}]}, + path=proto_path, + extra_params=dict(), + field_name="boundary_line", + ) rule.root = container compliance = self.FRC.first_element(field_list, rule) self.assertTrue(compliance) @@ -74,16 +78,20 @@ def test_comply_first_element(self): def test_not_comply_first_element(self): field_list = [self.lb1, self.lb2] container = TypeRulesContainer() - proto_path = ProtoMessagePath(['LaneBoundary', 'BoundaryPoint']) + proto_path = ProtoMessagePath(["LaneBoundary", "BoundaryPoint"]) container.add_type_from_path(proto_path) - container.add_type_from_path(ProtoMessagePath(['Vector3d'])) - - rule = Rule(verb="first_element", - params={'width': [{'is_equal_to': 0.11}], - 'height': [{'is_equal_to': 0.13}]}, - path=proto_path, - extra_params=dict(), - field_name='boundary_line') + container.add_type_from_path(ProtoMessagePath(["Vector3d"])) + + rule = Rule( + verb="first_element", + params={ + "width": [{"is_equal_to": 0.11}], + "height": [{"is_equal_to": 0.13}], + }, + path=proto_path, + extra_params=dict(), + field_name="boundary_line", + ) rule.root = container compliance = self.FRC.first_element(field_list, rule) self.assertFalse(compliance) diff --git a/tests/tests_rules/test_is_globally_unique.py b/tests/tests_rules/test_is_globally_unique.py index 34d0d19..15bacb1 100644 --- a/tests/tests_rules/test_is_globally_unique.py +++ b/tests/tests_rules/test_is_globally_unique.py @@ -8,7 +8,6 @@ class TestIsGlobalUnique(unittest.TestCase): - def setUp(self): self.FRC = OSIRulesChecker() @@ -34,20 +33,18 @@ def test_comply_is_globally_unique(self): Test if the ID Manager has unique indices """ container = TypeRulesContainer() - proto_path = ProtoMessagePath(['SensorView', 'sensor_id', 'is_globally_unique']) + proto_path = ProtoMessagePath(["SensorView", "sensor_id", "is_globally_unique"]) container.add_type_from_path(proto_path) - rule = Rule(verb='is_globally_unique', - field_name='sensor_id', - extra_params=dict(), - path=proto_path) + rule = Rule( + verb="is_globally_unique", + field_name="sensor_id", + extra_params=dict(), + path=proto_path, + ) rule.root = container self.FRC.is_globally_unique(self.linked_sid, rule) self.FRC.is_globally_unique(self.linked_sid2, rule) self.FRC.is_globally_unique(self.linked_sid2, rule) index_dict = self.FRC.id_manager._index self.assertEqual(2, len(index_dict)) - - - - diff --git a/tests/tests_rules/test_is_greater_than.py b/tests/tests_rules/test_is_greater_than.py index 345a9f4..8b1031a 100644 --- a/tests/tests_rules/test_is_greater_than.py +++ b/tests/tests_rules/test_is_greater_than.py @@ -18,48 +18,54 @@ def tearDown(self): def test_comply_greater(self): - field_params_rule_params = [[2, 1], - [0, -1], - [1, 0], - [1, -1], - [-1, -2], - [-1.3, -1.5], - [0.9, -1.3]] + field_params_rule_params = [ + [2, 1], + [0, -1], + [1, 0], + [1, -1], + [-1, -2], + [-1.3, -1.5], + [0.9, -1.3], + ] for fr_param in field_params_rule_params: with self.subTest(fr_param=fr_param): self.assertTrue( - self.FRC.is_greater_than(LinkedProtoField(value=fr_param[0]), - Rule(verb="is_greater_than", params=fr_param[1])) + self.FRC.is_greater_than( + LinkedProtoField(value=fr_param[0]), + Rule(verb="is_greater_than", params=fr_param[1]), + ) ) def test_not_comply_greater(self): - field_params_rule_params = [[2, 1], - [0, -1], - [1, 0], - [1, -1], - [-1, -2], - [-1.3, -1.5], - [0.9, -1.3]] + field_params_rule_params = [ + [2, 1], + [0, -1], + [1, 0], + [1, -1], + [-1, -2], + [-1.3, -1.5], + [0.9, -1.3], + ] for fr_param in field_params_rule_params: with self.subTest(fr_param=fr_param): self.assertFalse( - self.FRC.is_greater_than(LinkedProtoField(value=fr_param[1]), - Rule(verb="is_greater_than", params=fr_param[0])) + self.FRC.is_greater_than( + LinkedProtoField(value=fr_param[1]), + Rule(verb="is_greater_than", params=fr_param[0]), + ) ) def test_not_comply_equal(self): - field_params_rule_params = [[3, 3], - [0, 0], - [-1, -1], - [-1.5, -1.5], - [2.3, 2.3]] + field_params_rule_params = [[3, 3], [0, 0], [-1, -1], [-1.5, -1.5], [2.3, 2.3]] for fr_param in field_params_rule_params: with self.subTest(fr_param=fr_param): self.assertFalse( - self.FRC.is_greater_than(LinkedProtoField(value=fr_param[1]), - Rule(verb="is_greater_than", params=fr_param[0])) + self.FRC.is_greater_than( + LinkedProtoField(value=fr_param[1]), + Rule(verb="is_greater_than", params=fr_param[0]), + ) ) diff --git a/tests/tests_rules/test_is_greater_than_or_equal_to.py b/tests/tests_rules/test_is_greater_than_or_equal_to.py index 4831b6f..b97d2ff 100644 --- a/tests/tests_rules/test_is_greater_than_or_equal_to.py +++ b/tests/tests_rules/test_is_greater_than_or_equal_to.py @@ -18,48 +18,54 @@ def tearDown(self): def test_comply_greater(self): - field_params_rule_params = [[2, 1], - [0, -1], - [1, 0], - [1, -1], - [-1, -2], - [-1.3, -1.5], - [0.9, -1.3]] + field_params_rule_params = [ + [2, 1], + [0, -1], + [1, 0], + [1, -1], + [-1, -2], + [-1.3, -1.5], + [0.9, -1.3], + ] for fr_param in field_params_rule_params: with self.subTest(fr_param=fr_param): self.assertTrue( - self.FRC.is_greater_than_or_equal_to(LinkedProtoField(value=fr_param[0]), - Rule(verb="is_greater_than_or_equal_to", params=fr_param[1])) + self.FRC.is_greater_than_or_equal_to( + LinkedProtoField(value=fr_param[0]), + Rule(verb="is_greater_than_or_equal_to", params=fr_param[1]), + ) ) def test_not_comply_greater(self): - field_params_rule_params = [[2, 1], - [0, -1], - [1, 0], - [1, -1], - [-1, -2], - [-1.3, -1.5], - [0.9, -1.3]] + field_params_rule_params = [ + [2, 1], + [0, -1], + [1, 0], + [1, -1], + [-1, -2], + [-1.3, -1.5], + [0.9, -1.3], + ] for fr_param in field_params_rule_params: with self.subTest(fr_param=fr_param): self.assertFalse( - self.FRC.is_greater_than_or_equal_to(LinkedProtoField(value=fr_param[1]), - Rule(verb="is_greater_than_or_equal_to", params=fr_param[0])) + self.FRC.is_greater_than_or_equal_to( + LinkedProtoField(value=fr_param[1]), + Rule(verb="is_greater_than_or_equal_to", params=fr_param[0]), + ) ) def test_comply_equal(self): - field_params_rule_params = [[3, 3], - [0, 0], - [-1, -1], - [-1.5, -1.5], - [2.3, 2.3]] + field_params_rule_params = [[3, 3], [0, 0], [-1, -1], [-1.5, -1.5], [2.3, 2.3]] for fr_param in field_params_rule_params: with self.subTest(fr_param=fr_param): self.assertTrue( - self.FRC.is_greater_than_or_equal_to(LinkedProtoField(value=fr_param[1]), - Rule(verb="is_greater_than_or_equal_to", params=fr_param[0])) + self.FRC.is_greater_than_or_equal_to( + LinkedProtoField(value=fr_param[1]), + Rule(verb="is_greater_than_or_equal_to", params=fr_param[0]), + ) ) diff --git a/tests/tests_rules/test_is_iso_country_code.py b/tests/tests_rules/test_is_iso_country_code.py index 9ffb298..c1f638f 100644 --- a/tests/tests_rules/test_is_iso_country_code.py +++ b/tests/tests_rules/test_is_iso_country_code.py @@ -5,7 +5,6 @@ class TestIsIsoCountryCode(unittest.TestCase): - def setUp(self): self.FRC = OSIRulesChecker() @@ -13,10 +12,7 @@ def tearDown(self): del self.FRC def test_comply_iso_country_code(self): - self.assertTrue(self.FRC.is_iso_country_code('DEU', None)) + self.assertTrue(self.FRC.is_iso_country_code("DEU", None)) def test_not_comply_iso_country_code(self): - self.assertFalse(self.FRC.is_iso_country_code('1234', None)) - - - + self.assertFalse(self.FRC.is_iso_country_code("1234", None)) diff --git a/tests/tests_rules/test_is_less_than.py b/tests/tests_rules/test_is_less_than.py index a312f32..cece4e0 100644 --- a/tests/tests_rules/test_is_less_than.py +++ b/tests/tests_rules/test_is_less_than.py @@ -18,48 +18,54 @@ def tearDown(self): def test_comply_less(self): - field_params_rule_params = [[2, 1], - [0, -1], - [1, 0], - [1, -1], - [-1, -2], - [-1.3, -1.5], - [0.9, -1.3]] + field_params_rule_params = [ + [2, 1], + [0, -1], + [1, 0], + [1, -1], + [-1, -2], + [-1.3, -1.5], + [0.9, -1.3], + ] for fr_param in field_params_rule_params: with self.subTest(fr_param=fr_param): self.assertTrue( - self.FRC.is_less_than(LinkedProtoField(value=fr_param[1]), - Rule(verb="is_less_than", params=fr_param[0])) + self.FRC.is_less_than( + LinkedProtoField(value=fr_param[1]), + Rule(verb="is_less_than", params=fr_param[0]), + ) ) def test_not_comply_less(self): - field_params_rule_params = [[2, 1], - [0, -1], - [1, 0], - [1, -1], - [-1, -2], - [-1.3, -1.5], - [0.9, -1.3]] + field_params_rule_params = [ + [2, 1], + [0, -1], + [1, 0], + [1, -1], + [-1, -2], + [-1.3, -1.5], + [0.9, -1.3], + ] for fr_param in field_params_rule_params: with self.subTest(fr_param=fr_param): self.assertFalse( - self.FRC.is_less_than(LinkedProtoField(value=fr_param[0]), - Rule(verb="is_less_than", params=fr_param[1])) + self.FRC.is_less_than( + LinkedProtoField(value=fr_param[0]), + Rule(verb="is_less_than", params=fr_param[1]), + ) ) def test_not_comply_equal(self): - field_params_rule_params = [[3, 3], - [0, 0], - [-1, -1], - [-1.5, -1.5], - [2.3, 2.3]] + field_params_rule_params = [[3, 3], [0, 0], [-1, -1], [-1.5, -1.5], [2.3, 2.3]] for fr_param in field_params_rule_params: with self.subTest(fr_param=fr_param): self.assertFalse( - self.FRC.is_less_than(LinkedProtoField(value=fr_param[1]), - Rule(verb="is_less_than", params=fr_param[0])) + self.FRC.is_less_than( + LinkedProtoField(value=fr_param[1]), + Rule(verb="is_less_than", params=fr_param[0]), + ) ) diff --git a/tests/tests_rules/test_is_less_than_or_equal_to.py b/tests/tests_rules/test_is_less_than_or_equal_to.py index d3c5220..eb683f4 100644 --- a/tests/tests_rules/test_is_less_than_or_equal_to.py +++ b/tests/tests_rules/test_is_less_than_or_equal_to.py @@ -18,48 +18,54 @@ def tearDown(self): def test_comply_less(self): - field_params_rule_params = [[2, 1], - [0, -1], - [1, 0], - [1, -1], - [-1, -2], - [-1.3, -1.5], - [0.9, -1.3]] + field_params_rule_params = [ + [2, 1], + [0, -1], + [1, 0], + [1, -1], + [-1, -2], + [-1.3, -1.5], + [0.9, -1.3], + ] for fr_param in field_params_rule_params: with self.subTest(fr_param=fr_param): self.assertTrue( - self.FRC.is_less_than_or_equal_to(LinkedProtoField(value=fr_param[1]), - Rule(verb="is_less_than_or_equal_to", params=fr_param[0])) + self.FRC.is_less_than_or_equal_to( + LinkedProtoField(value=fr_param[1]), + Rule(verb="is_less_than_or_equal_to", params=fr_param[0]), + ) ) def test_not_comply_less(self): - field_params_rule_params = [[2, 1], - [0, -1], - [1, 0], - [1, -1], - [-1, -2], - [-1.3, -1.5], - [0.9, -1.3]] + field_params_rule_params = [ + [2, 1], + [0, -1], + [1, 0], + [1, -1], + [-1, -2], + [-1.3, -1.5], + [0.9, -1.3], + ] for fr_param in field_params_rule_params: with self.subTest(fr_param=fr_param): self.assertFalse( - self.FRC.is_less_than_or_equal_to(LinkedProtoField(value=fr_param[0]), - Rule(verb="is_less_than_or_equal_to", params=fr_param[1])) + self.FRC.is_less_than_or_equal_to( + LinkedProtoField(value=fr_param[0]), + Rule(verb="is_less_than_or_equal_to", params=fr_param[1]), + ) ) def test_comply_equal(self): - field_params_rule_params = [[3, 3], - [0, 0], - [-1, -1], - [-1.5, -1.5], - [2.3, 2.3]] + field_params_rule_params = [[3, 3], [0, 0], [-1, -1], [-1.5, -1.5], [2.3, 2.3]] for fr_param in field_params_rule_params: with self.subTest(fr_param=fr_param): self.assertTrue( - self.FRC.is_less_than_or_equal_to(LinkedProtoField(value=fr_param[1]), - Rule(verb="is_less_than_or_equal_to", params=fr_param[0])) + self.FRC.is_less_than_or_equal_to( + LinkedProtoField(value=fr_param[1]), + Rule(verb="is_less_than_or_equal_to", params=fr_param[0]), + ) ) diff --git a/tests/tests_rules/test_is_optional.py b/tests/tests_rules/test_is_optional.py index e1d152d..1c0b36e 100644 --- a/tests/tests_rules/test_is_optional.py +++ b/tests/tests_rules/test_is_optional.py @@ -19,7 +19,7 @@ def tearDown(self): def test_comply_is_optional(self): self.assertTrue( - self.FRC.is_optional(LinkedProtoField(value=1), - Rule(verb="is_optional", params=None) - ) + self.FRC.is_optional( + LinkedProtoField(value=1), Rule(verb="is_optional", params=None) + ) ) diff --git a/tests/tests_rules/test_is_set.py b/tests/tests_rules/test_is_set.py index 993b9da..558c3b4 100644 --- a/tests/tests_rules/test_is_set.py +++ b/tests/tests_rules/test_is_set.py @@ -28,7 +28,6 @@ class TestIsSet(unittest.TestCase): - def setUp(self): self.FRC = OSIRulesChecker() diff --git a/tests/tests_rules/test_is_valid.py b/tests/tests_rules/test_is_valid.py index 7a8f45e..8fd872e 100644 --- a/tests/tests_rules/test_is_valid.py +++ b/tests/tests_rules/test_is_valid.py @@ -5,13 +5,18 @@ from osi3.osi_common_pb2 import Orientation3d from osivalidator.linked_proto_field import LinkedProtoField -from osivalidator.osi_rules import Rule, TypeRulesContainer, ProtoMessagePath, MessageTypeRules, FieldRules +from osivalidator.osi_rules import ( + Rule, + TypeRulesContainer, + ProtoMessagePath, + MessageTypeRules, + FieldRules, +) from osivalidator.osi_rules_checker import OSIRulesChecker class TestIsValid(unittest.TestCase): - def setUp(self): self.FRC = OSIRulesChecker() orient3d = Orientation3d() @@ -26,17 +31,31 @@ def tearDown(self): def test_comply_is_valid(self): container = TypeRulesContainer() - proto_path = ProtoMessagePath(['Orientation3d', 'roll']) + proto_path = ProtoMessagePath(["Orientation3d", "roll"]) container.add_type_from_path(proto_path) root_container = TypeRulesContainer() - root_path = ProtoMessagePath(['Orientation3d']) + root_path = ProtoMessagePath(["Orientation3d"]) root_container.add_type_from_path(root_path) - rule1 = Rule(verb="is_greater_than", field_name='roll', params=0, path=proto_path, extra_params=dict()) - rule2 = Rule(verb="is_less_than", field_name='roll', params=10, path=proto_path, extra_params=dict()) - - rules = FieldRules("roll", rules=[rule1, rule2], path=proto_path, root=root_container) + rule1 = Rule( + verb="is_greater_than", + field_name="roll", + params=0, + path=proto_path, + extra_params=dict(), + ) + rule2 = Rule( + verb="is_less_than", + field_name="roll", + params=10, + path=proto_path, + extra_params=dict(), + ) + + rules = FieldRules( + "roll", rules=[rule1, rule2], path=proto_path, root=root_container + ) rule = MessageTypeRules(name="Orientation3d") rule.add_field(rules) @@ -51,17 +70,31 @@ def test_comply_is_valid(self): def test_comply_is_not_valid(self): container = TypeRulesContainer() - proto_path = ProtoMessagePath(['Orientation3d', 'roll']) + proto_path = ProtoMessagePath(["Orientation3d", "roll"]) container.add_type_from_path(proto_path) root_container = TypeRulesContainer() - root_path = ProtoMessagePath(['Orientation3d']) + root_path = ProtoMessagePath(["Orientation3d"]) root_container.add_type_from_path(root_path) - rule1 = Rule(verb="is_greater_than", field_name='roll', params=6, path=proto_path, extra_params=dict()) - rule2 = Rule(verb="is_less_than", field_name='roll', params=10, path=proto_path, extra_params=dict()) - - rules = FieldRules("roll", rules=[rule1, rule2], path=proto_path, root=root_container) + rule1 = Rule( + verb="is_greater_than", + field_name="roll", + params=6, + path=proto_path, + extra_params=dict(), + ) + rule2 = Rule( + verb="is_less_than", + field_name="roll", + params=10, + path=proto_path, + extra_params=dict(), + ) + + rules = FieldRules( + "roll", rules=[rule1, rule2], path=proto_path, root=root_container + ) rule = MessageTypeRules(name="Orientation3d") rule.add_field(rules) diff --git a/tests/tests_rules/test_last_element_of.py b/tests/tests_rules/test_last_element_of.py index a092de4..4710622 100644 --- a/tests/tests_rules/test_last_element_of.py +++ b/tests/tests_rules/test_last_element_of.py @@ -10,14 +10,15 @@ class TestFirstElement(unittest.TestCase): - def setUp(self): self.FRC = OSIRulesChecker() sv1 = SensorView() linked_sv1 = LinkedProtoField(sv1, name="SensorView") gt1 = sv1.global_ground_truth - linked_gt1 = LinkedProtoField(gt1, name="global_ground_truth", parent=linked_sv1) + linked_gt1 = LinkedProtoField( + gt1, name="global_ground_truth", parent=linked_sv1 + ) gtlb1 = gt1.lane_boundary.add() linked_gtlb1 = LinkedProtoField(gtlb1, name="lane_boundary", parent=linked_gt1) @@ -30,14 +31,16 @@ def setUp(self): bladd1.height = 0.0 self.lb1 = LinkedProtoField(bladd1, name="boundary_line", parent=linked_gtlb1) - self.lb1.path = 'SensorView.global_ground_truth.lane_boundary.boundary_line' + self.lb1.path = "SensorView.global_ground_truth.lane_boundary.boundary_line" # self.lb1.parent = sv2 = SensorView() linked_sv2 = LinkedProtoField(sv2, name="SensorView") gt2 = sv2.global_ground_truth - linked_gt2 = LinkedProtoField(gt2, name="global_ground_truth", parent=linked_sv2) + linked_gt2 = LinkedProtoField( + gt2, name="global_ground_truth", parent=linked_sv2 + ) gtlb2 = gt2.lane_boundary.add() linked_gtlb2 = LinkedProtoField(gtlb2, name="lane_boundary", parent=linked_gt2) @@ -49,7 +52,7 @@ def setUp(self): bladd2.width = 0.14 bladd2.height = 0.13 self.lb2 = LinkedProtoField(bladd2, name="boundary_line", parent=linked_gtlb2) - self.lb2.path = 'SensorView.global_ground_truth.lane_boundary.boundary_line' + self.lb2.path = "SensorView.global_ground_truth.lane_boundary.boundary_line" def tearDown(self): del self.FRC @@ -57,16 +60,20 @@ def tearDown(self): def test_comply_last_element(self): field_list = [self.lb1, self.lb2] container = TypeRulesContainer() - proto_path = ProtoMessagePath(['LaneBoundary', 'BoundaryPoint']) + proto_path = ProtoMessagePath(["LaneBoundary", "BoundaryPoint"]) container.add_type_from_path(proto_path) - container.add_type_from_path(ProtoMessagePath(['Vector3d'])) - - rule = Rule(verb="last_element", - params={'width': [{'is_equal_to': 0.14}], - 'height': [{'is_equal_to': 0.13}]}, - path=proto_path, - extra_params=dict(), - field_name='boundary_line') + container.add_type_from_path(ProtoMessagePath(["Vector3d"])) + + rule = Rule( + verb="last_element", + params={ + "width": [{"is_equal_to": 0.14}], + "height": [{"is_equal_to": 0.13}], + }, + path=proto_path, + extra_params=dict(), + field_name="boundary_line", + ) rule.root = container compliance = self.FRC.last_element(field_list, rule) self.assertTrue(compliance) @@ -74,16 +81,20 @@ def test_comply_last_element(self): def test_not_comply_last_element(self): field_list = [self.lb1, self.lb2] container = TypeRulesContainer() - proto_path = ProtoMessagePath(['LaneBoundary', 'BoundaryPoint']) + proto_path = ProtoMessagePath(["LaneBoundary", "BoundaryPoint"]) container.add_type_from_path(proto_path) - container.add_type_from_path(ProtoMessagePath(['Vector3d'])) - - rule = Rule(verb="last_element", - params={'width': [{'is_equal_to': 0.11}], - 'height': [{'is_equal_to': 0.13}]}, - path=proto_path, - extra_params=dict(), - field_name='boundary_line') + container.add_type_from_path(ProtoMessagePath(["Vector3d"])) + + rule = Rule( + verb="last_element", + params={ + "width": [{"is_equal_to": 0.11}], + "height": [{"is_equal_to": 0.13}], + }, + path=proto_path, + extra_params=dict(), + field_name="boundary_line", + ) rule.root = container compliance = self.FRC.last_element(field_list, rule) self.assertFalse(compliance) diff --git a/tests/tests_rules/test_refers_to.py b/tests/tests_rules/test_refers_to.py index 50345e1..fa6bf28 100644 --- a/tests/tests_rules/test_refers_to.py +++ b/tests/tests_rules/test_refers_to.py @@ -10,7 +10,6 @@ class TestRefersTo(unittest.TestCase): - def setUp(self): self.FRC = OSIRulesChecker() @@ -18,21 +17,29 @@ def setUp(self): linked_sv1 = LinkedProtoField(sv1, name="SensorView") gt1 = sv1.global_ground_truth - linked_gt1 = LinkedProtoField(gt1, name="global_ground_truth", parent=linked_sv1) + linked_gt1 = LinkedProtoField( + gt1, name="global_ground_truth", parent=linked_sv1 + ) gt1.host_vehicle_id.value = 0 hvid1 = gt1.host_vehicle_id - self.linked_hvid1 = LinkedProtoField(hvid1, name="host_vehicle_id", parent=linked_gt1) + self.linked_hvid1 = LinkedProtoField( + hvid1, name="host_vehicle_id", parent=linked_gt1 + ) sv2 = SensorView() linked_sv2 = LinkedProtoField(sv2, name="SensorView") gt2 = sv2.global_ground_truth - linked_gt2 = LinkedProtoField(gt2, name="global_ground_truth", parent=linked_sv2) + linked_gt2 = LinkedProtoField( + gt2, name="global_ground_truth", parent=linked_sv2 + ) gt2.host_vehicle_id.value = 1 hvid1 = gt2.host_vehicle_id - self.linked_hvid2 = LinkedProtoField(hvid1, name="host_vehicle_id", parent=linked_gt2) + self.linked_hvid2 = LinkedProtoField( + hvid1, name="host_vehicle_id", parent=linked_gt2 + ) def tearDown(self): del self.FRC @@ -44,14 +51,16 @@ def test_comply_refers_to(self): Check if the message object is referenced correctly """ container = TypeRulesContainer() - proto_path = ProtoMessagePath(['GroundTruth', 'host_vehicle_id', 'refers_to']) + proto_path = ProtoMessagePath(["GroundTruth", "host_vehicle_id", "refers_to"]) container.add_type_from_path(proto_path) - rule = Rule(verb="refers_to", - params='MovingObject', - extra_params=dict(), - path=proto_path, - field_name='host_vehicle_id') + rule = Rule( + verb="refers_to", + params="MovingObject", + extra_params=dict(), + path=proto_path, + field_name="host_vehicle_id", + ) rule.root = container self.FRC.refers_to(self.linked_hvid1, rule) self.FRC.refers_to(self.linked_hvid2, rule) @@ -67,4 +76,4 @@ def test_comply_refers_to(self): # Check the id assignment of the reference to the object self.assertEqual(references_list[0][0].host_vehicle_id.value, 0) self.assertEqual(references_list[0][1], 0) - self.assertEqual(references_list[0][2], 'MovingObject') + self.assertEqual(references_list[0][2], "MovingObject") From 0f0b3fb5356b5e0cccc450e4a592e2057d92bade Mon Sep 17 00:00:00 2001 From: Viktor Kreschenski Date: Tue, 18 Feb 2020 20:43:03 +0100 Subject: [PATCH 6/7] Added vulture and removed unused variables --- .travis.yml | 7 +- osivalidator/osi_trace.py | 2 - test_cases.py | 549 -------------------------------------- 3 files changed, 4 insertions(+), 554 deletions(-) delete mode 100644 test_cases.py diff --git a/.travis.yml b/.travis.yml index 1ae0128..d81ce6a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,6 +38,10 @@ script: - pip install black - black --check --exclude "(open-simulation-interface|proto2cpp)" . + # Install vulture and search for dead code with 100% confidence + - pip install vulture + - vulture . --min-confidence 100 + # Install osi-validation - cd open-simulation-interface - pip install . @@ -47,9 +51,6 @@ script: # Generate parsed rules - python rules2yml.py -d rules - # Check if rule syntax in osi is correct - # - python test_cases.py - # Show validator usage - osivalidator -h diff --git a/osivalidator/osi_trace.py b/osivalidator/osi_trace.py index f4a285c..94a1108 100644 --- a/osivalidator/osi_trace.py +++ b/osivalidator/osi_trace.py @@ -132,9 +132,7 @@ def retrieve_message(self): message_offset += message_length + self._int_length self.message_offsets.append(message_offset) self.update_bar(progress_bar, message_offset) - before_tell = self.trace_file.tell() self.trace_file.seek(message_offset) - after_tell = self.trace_file.tell() eof = self.trace_file.tell() > self.buffer_size * (counter + 1) # Check if reached end of file diff --git a/test_cases.py b/test_cases.py deleted file mode 100644 index 1f1f9c5..0000000 --- a/test_cases.py +++ /dev/null @@ -1,549 +0,0 @@ -import sys -import unicodedata -import re -from glob import * - -state = 0 - -# Define grammar rules for rule definitions -rules_dict = { - "in_range": r"\b(in_range)\b: \[\d+(\.\d+)?, \d+(\.\d+)?\]", - "is_greater_than": r"\b(is_greater_than)\b: \d+(\.\d+)?", - "is_greater_than_or_equal_to": r"\b(is_greater_than_or_equal_to)\b: \d+(\.\d+)?", - "is_less_than_or_equal_to": r"\b(is_less_than_or_equal_to)\b: \d+(\.\d+)?", - "is_less_than": r"\b(is_less_than)\b: \d+(\.\d+)?", - "is_equal_to": r"\b(is_equal_to)\b: \d+(\.\d+)?", - "is_different_to": r"\b(is_different_to)\b: \d+(\.\d+)?", - "is_globally_unique": r"\b(is_globally_unique)\b", - "refers_to": r"\b(refers_to)\b", - "is_iso_country_code": r"\b(is_iso_country_code)\b", - "first_element": r"\b(first_element)\b: \{.*: \d+\.\d+\}", - "last_element": r"\b(last_element)\b: \{.*: \d+\.\d+\}", - "is_optional": r"\b(is_optional)\b", - "check_if": r"\b(check_if)\b: \[\{.*: \d+(\.\d+)?, target: .*}, \{do_check: \{.*: \d+(\.\d+)?}}]", - "is_set": r"\b(is_set)\b", -} - - -for file in glob("open-simulation-interface/*.proto"): - with open(file, "rt") as fin: - i = 0 - isEnum = False - enumName = "" - noMessage = 0 - noComment = 0 - hasBrief = False - hasNewLine = True - htmlblock = False - saveStatement = "" - - for line in fin: - i = i + 1 - hasNewLine = line.endswith("\n") - - # -------------------------------------------------------------- - # Test case 1 is checking if there are illegal tabulators in the code - if line.find("\t") != -1: - print(file + " in line " + str(i) + ": not permitted tab found") - state = 1 - - # -------------------------------------------------------------- - # Test case 2 is checking if there is an "Umlaut" etc. - if sys.version_info >= (3, 0): - if ( - line - != unicodedata.normalize("NFKD", line) - .encode("ASCII", "ignore") - .decode() - ): - print( - file + " in line " + str(i) + ": a none ASCII char is present" - ) - state = 1 - else: - if line != unicodedata.normalize( - "NFKD", unicode(line, "ISO-8859-1") - ).encode("ASCII", "ignore"): - print( - file + " in line " + str(i) + ": a none ASCII char is present" - ) - state = 1 - - if file.find(".proto") != -1: - # -------------------------------------------------------------- - # Test case 3 is checking if there are more than the two allowed '/' - if line.find("///") != -1: - print(file + " in line " + str(i) + ": not permitted use of '///' ") - state = 1 - - # -------------------------------------------------------------- - # Test case 4 is checking if there is an other type of comment - if line.find("/*") != -1: - print(file + " in line " + str(i) + ": not permitted use of '/*' ") - state = 1 - - # -------------------------------------------------------------- - # Test case 5 is checking if there is an other type of comment - if line.find("*/") != -1: - print(file + " in line " + str(i) + ": not permitted use of '*/' ") - state = 1 - - # -------------------------------------------------------------- - # Test case 9 is checking if there is '__' - if line.find("__") != -1: - print(file + " in line " + str(i) + ": not permitted use of '__' ") - state = 1 - - # -------------------------------------------------------------- - # Divide statement and comment. Concatenate multi line statements. - - # Search for comment ("//"). - matchComment = re.search("//", line) - if matchComment is not None: - statement = line[: matchComment.start()] - comment = line[matchComment.end() :] - else: - statement = line - comment = "" - - # Add part of the statement from last line. - statement = saveStatement + " " + statement - saveStatement = "" - - # New line is not necessary. Remove for a better output. - statement = statement.replace("\n", "") - comment = comment.replace("\n", "") - - # Is statement complete - matchSep = re.search(r"[{};]", statement) - if matchSep is None: - saveStatement = statement - statement = "" - else: - saveStatement = statement[matchSep.end() :] - statement = statement[: matchSep.end()] - - # -------------------------------------------------------------- - # Test case 6-8 camelcase for enums and check enum name? - - # . - if isEnum is True: - matchName = re.search(r"\b\w[\S:]+\b", statement) - if matchName is not None: - checkName = statement[matchName.start() : matchName.end()] - # Test case 6: Check correct name - if checkName.find(enumName) != 0: - print( - file - + " in line " - + str(i) - + ": enum type wrong. '" - + checkName - + "' should start with '" - + enumName - + "'" - ) - state = 1 - # Test case 7: Check upper case - elif checkName != checkName.upper(): - print( - file - + " in line " - + str(i) - + ": enum type wrong. '" - + checkName - + "' should use upper case" - ) - state = 1 - - # Search for "enum". - matchEnum = re.search(r"\benum\b", statement) - if matchEnum is not None: - isEnum = True - endOfLine = statement[matchEnum.end() :] - matchName = re.search(r"\b\w[\S]*\b", endOfLine) - if matchName is not None: - # Test case 8: Check name - no special char - matchNameConv = re.search( - r"\b[A-Z][a-zA-Z0-9]*\b", - endOfLine[matchName.start() : matchName.end()], - ) - if matchNameConv is None: - print( - file - + " in line " - + str(i) - + ": enum name wrong. '" - + endOfLine[matchName.start() : matchName.end()] - + "'" - ) - state = 1 - enumName = ( - convert(endOfLine[matchName.start() : matchName.end()]) - + "_" - ) - - # Search for a closing brace. - matchClosingBrace = re.search("}", statement) - if isEnum is True and matchClosingBrace is not None: - isEnum = False - enumName = "" - - def convert(name): - s1 = re.sub(r"(.)([A-Z][a-z]+)", r"\1_\2", name) - return re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", s1).upper() - - # -------------------------------------------------------------- - # Test case 10-12,18 check message name, field type and field name - # - # Check (nested) messages - - if isEnum is False: - # Check if not inside an enum. - - # Search for "message". - matchMessage = re.search(r"\bmessage\b", statement) - if matchMessage is not None: - # a new message or a new nested message - noMessage += 1 - endOfLine = statement[matchMessage.end() :] - matchName = re.search(r"\b\w[\S]*\b", endOfLine) - if matchName is not None: - # Test case 10: Check name - no special char - - # start with a capital letter - matchNameConv = re.search( - r"\b[A-Z][a-zA-Z0-9]*\b", - endOfLine[matchName.start() : matchName.end()], - ) - if matchNameConv is None: - print( - file - + " in line " - + str(i) - + ": message name wrong. '" - + endOfLine[matchName.start() : matchName.end()] - + "'" - ) - state = 1 - elif re.search(r"\bextend\b", statement) is not None: - # treat extend as message - noMessage += 1 - else: - # Check field names - if noMessage > 0: - matchName = re.search(r"\b\w[\S]*\b\s*=", statement) - if matchName is not None: - checkName = statement[ - matchName.start() : matchName.end() - 1 - ] - # Test case 11: Check lowercase letters for field names - if checkName != checkName.lower(): - print( - file - + " in line " - + str(i) - + ": field name wrong. '" - + checkName - + "' should use lower case" - ) - state = 1 - # Check field message type (remove field name) - type = statement.replace(checkName, "") - matchName = re.search(r"\b\w[\S\.]*\s*=", type) - if matchName is not None: - checkType = ( - " " - + type[matchName.start() : matchName.end() - 1] - + " " - ) - # Test case 12: Check nested message type - matchNameConv = re.search( - r"[ ][a-zA-Z][a-zA-Z0-9]*([\.][A-Z][a-zA-Z0-9]*)*[ ]", - checkType, - ) - if matchNameConv is None: - print( - file - + " in line " - + str(i) - + ": field message type wrong. Check: '" - + checkType - + "'" - ) - state = 1 - - if ( - re.search(r"\boptional\b", type) is None - and re.search(r"\brepeated\b", type) is None - ): - # Test 18 has every field the multiplicity "repeated" or "optional" - print( - file - + " in line " - + str(i) - + ': field multiplicity ("optional" or "repeated") is missing. Check: \'' - + statement - + "'" - ) - state = 1 - - # Search for a closing brace. - matchClosingBrace = re.search("}", statement) - if noMessage > 0 and matchClosingBrace is not None: - noMessage -= 1 - - # -------------------------------------------------------------- - # Test case 13-17 is checking comment - if matchComment is not None: - noComment += 1 - if comment.find("\\brief") != -1: - hasBrief = True - elif len(saveStatement) == 0: - # Test case 13 is checking if comment is min. 2 lines - if noComment == 1: - print( - file - + " in line " - + str(i - 1) - + ": short comment - min. 2 lines." - ) - state = 1 - if ( - re.search(r"\bmessage\b", statement) is not None - or re.search(r"\bextend\b", statement) is not None - ): - if hasBrief == False: - # Test case 14 each message and extend has a \brief comment - print( - file - + " in line " - + str(i - 1) - + ": \\brief section in comment is missing for: '" - + statement - + "'" - ) - state = 1 - elif hasBrief == True: - # Test case 15 only message and extend has a \brief comment - print( - file - + " in line " - + str(i - 1) - + ": \\brief section in comment is not necessary for: '" - + statement - + "'" - ) - state = 1 - - if ( - re.search(r"\bmessage\b", statement) is not None - or re.search(r"\bextend\b", statement) is not None - or re.search(r"\benum\b", statement) is not None - ): - if noComment == 0: - # Test case 16 every message, extend or enum has a comment - print( - file - + " in line " - + str(i) - + ": comment is missing for: '" - + statement - + "'" - ) - state = 1 - - if noMessage > 0 or isEnum == True: - if statement.find(";") != -1: - if noComment == 0: - # Test case 17 every statement has a comment - print( - file - + " in line " - + str(i) - + ": comment is missing for: '" - + statement - + "'" - ) - state = 1 - - noComment = 0 - hasBrief = False - - # -------------------------------------------------------------- - # Test case 20 is checking comment and html tags - if matchComment is not None: - htmlComment = "" - htmlFreeComment = comment - if htmlblock is False: - matchHTMLOnly = re.search(r"\\htmlonly", comment) - if matchHTMLOnly is not None: - - htmlComment = comment[matchHTMLOnly.end() :] - htmlFreeComment = comment[: matchHTMLOnly.start()] - htmlblock = True - else: - htmlComment = comment - htmlFreeComment = "" - - if htmlblock is True: - matchEndHTMLOnly = re.search(r"\\endhtmlonly", htmlComment) - if matchEndHTMLOnly is not None: - htmlFreeComment = ( - htmlFreeComment + htmlComment[matchEndHTMLOnly.end() :] - ) - htmlComment = htmlComment[: matchEndHTMLOnly.start()] - htmlblock = False - - # if htmlFreeComment.find("<") != -1: - # Test case 20 html tags only in htmlonly sections --> no error - # print(file + " in line " + str(i) + ": doxygen comment html tag found (use htmlonly if possible): '"+htmlFreeComment+"'") - ##state = 1 - if htmlComment.find("\\") != -1: - # Test case 23 html tags only in htmlonly sections - print( - file - + " in line " - + str(i) - + ": doxygen comment \\.. reference found: '" - + htmlComment - + "'" - ) - # state = 1 - if htmlComment.find("#") != -1: - # Test case 24 html tags only in htmlonly sections - print( - file - + " in line " - + str(i) - + ": doxygen comment #.. reference found: '" - + htmlComment - + "'" - ) - # state = 1 - - elif htmlblock is True: - # Test case 22 html tags only in htmlonly sections without end html only - print( - file - + " in line " - + str(i - 1) - + ": doxygen comment html section without endhtmlonly" - ) - htmlblock = False - # state = 1 - - # -------------------------------------------------------------- - # Test case 21 is checking comment and html tags - if matchComment is not None: - if comment.find("@") != -1: - # Test case 21 html tags only in htmlonly sections - print( - file - + " in line " - + str(i) - + ": @ tag found (please replace with \\): '" - + htmlFreeComment - + "'" - ) - state = 1 - - # -------------------------------------------------------------- - # Test case 25 is checking if each field has a rule and must be set - if isEnum is False: - if matchComment is not None: - if comment.find("\\endrules") != -1: - endRule = True - - if comment.find("\\rules") != -1: - hasRule = True - lineruleCount = -1 - foundruleCount = -1 - - if re.search(r"\b(is_set)\b", comment): - isSet = True - # foundruleCount += 1 - - # TODO There also needs to be a checking for inner rules for check_if and first_element - if not endRule and comment != "": - foundRule = False - for rulename, ruleregex in rules_dict.items(): - if re.search(ruleregex, comment): - foundRule = True - foundruleCount += 1 - - elif len(saveStatement) == 0: - if noMessage > 0 or isEnum == True: - if statement.find(";") != -1: - statement = statement.strip() - if not hasRule and not endRule: - # Test case 17 every statement has a comment - print( - file - + " in line " - + str(i) - + ": rule is missing for: '" - + statement - + "'" - ) - state = 1 - - if hasRule and not isSet and endRule: - print( - file - + " in line " - + str(i) - + ": rule is_set is missing for: '" - + statement - + "'" - ) - state = 1 - - if ( - hasRule - and lineruleCount != foundruleCount - and endRule - and lineruleCount - foundruleCount - 1 > 0 - ): - print( - file - + " in line " - + str(i) - + ": " - + str(lineruleCount - foundruleCount - 1) - + " defined rule(s) does not exists for: '" - + statement - + "'" - ) - state = 1 - - if ( - hasRule - and lineruleCount > foundruleCount - and not endRule - ): - print( - file - + " in line " - + str(i) - + ": endrules statement does not exists for: '" - + statement - + "'" - ) - state = 1 - - foundRule = False - hasRule = False - endRule = False - isSet = False - - if hasRule and not endRule: - lineruleCount += 1 - # -------------------------------------------------------------- - # Next Test 26 - - # Test case 19 last line must end with a new line. - if hasNewLine == False: - print(file + " has no new line at the end of the file.") - state = 1 - -sys.exit(state) From b6426823a07762701015bd60640a9ca1491f8195 Mon Sep 17 00:00:00 2001 From: Viktor Kreschenski Date: Tue, 18 Feb 2020 21:13:15 +0100 Subject: [PATCH 7/7] Removed redundant code and added explicit folders for vulture --- .travis.yml | 2 +- pyosi/README.md | 4 ---- pyosi/__init__.py | 0 pyosi/conversions.py | 3 --- pyosi/reading.py | 42 ----------------------------------------- pyosi/testing_script.py | 22 --------------------- rules2yml.py | 13 ------------- 7 files changed, 1 insertion(+), 85 deletions(-) delete mode 100644 pyosi/README.md delete mode 100644 pyosi/__init__.py delete mode 100644 pyosi/conversions.py delete mode 100644 pyosi/reading.py delete mode 100644 pyosi/testing_script.py diff --git a/.travis.yml b/.travis.yml index d81ce6a..18fd13a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,7 +40,7 @@ script: # Install vulture and search for dead code with 100% confidence - pip install vulture - - vulture . --min-confidence 100 + - vulture *.py tests/ osivalidator/ --min-confidence 100 # Install osi-validation - cd open-simulation-interface diff --git a/pyosi/README.md b/pyosi/README.md deleted file mode 100644 index e40d868..0000000 --- a/pyosi/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# PyOSI - -The intention of this module is to provide place to hold general utilities for -OSI messages in Python. diff --git a/pyosi/__init__.py b/pyosi/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/pyosi/conversions.py b/pyosi/conversions.py deleted file mode 100644 index 5baf6be..0000000 --- a/pyosi/conversions.py +++ /dev/null @@ -1,3 +0,0 @@ -def timestamp_string_value(timestamp): - """ String representation of osi timestamp""" - return "{:d}.{:09d}".format(timestamp.seconds, timestamp.nanos) diff --git a/pyosi/reading.py b/pyosi/reading.py deleted file mode 100644 index f4034ac..0000000 --- a/pyosi/reading.py +++ /dev/null @@ -1,42 +0,0 @@ -# Open and Read text file -def read_text_data(file_name): - """ Read data from file """ - with open(file_name, "rb") as f: - data = f.read() - return data - - -# Search for separators -def separate_all_sections(encoded): - """ Separate files based on """ - SEPARATOR = b"$$__$$" - start, end = 0, 0 - - is_finished = False - while not is_finished: - end = encoded.find(SEPARATOR, start) - - # Finishing coverage - if end == -1: - is_finished = True - end = len(encoded) - - if len(encoded[start:end]) == 0: - break - yield encoded[start:end] - start = end + len(SEPARATOR) - - -# Decode protobuff data -def decode_data(encoded_data, data_class): - """ Decoder osi binary data into provided OSI class - Input: - - encoded_data - byte type data - - osi class to be parsed - """ - decoded_data = [] - for row in encoded_data: - data_object = data_class() - data_object.ParseFromString(row) - decoded_data.append(data_object) - return decoded_data diff --git a/pyosi/testing_script.py b/pyosi/testing_script.py deleted file mode 100644 index 6132579..0000000 --- a/pyosi/testing_script.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# import osi3.osi_sensorview_pb2 -# import reading - - -# def main(): - -# example_data_file = '/home/cszyszka/projects/osi/osi-validation/Data/overtake_right_straight_SensorView.txt' - -# data_class = osi3.osi_sensorview_pb2.SensorView - -# binary_data = reading.read_text_data(example_data_file) -# generator_separator = reading.separate_all_sections(binary_data) -# list_of_data = reading.decode_data(generator_separator, data_class) - -# print(f'Data file {example_data_file} contained {len(list_of_data)} timestamps.') - - -# if __name__ == '__main__': -# main() diff --git a/rules2yml.py b/rules2yml.py index 3b24859..4f35731 100644 --- a/rules2yml.py +++ b/rules2yml.py @@ -1,5 +1,4 @@ import sys -import unicodedata import argparse import re from glob import * @@ -45,7 +44,6 @@ def gen_yml_rules(dir_name="rules"): with open(f"{dir_name}/{filename}.yml", "a") as yml_file: with open(file, "rt") as fin: isEnum = False - enumName = "" numMessage = 0 shiftCounter = False saveStatement = "" @@ -100,24 +98,13 @@ def gen_yml_rules(dir_name="rules"): r"\b[A-Z][a-zA-Z0-9]*\b", endOfLine[matchName.start() : matchName.end()], ) - enumName = ( - convert( - endOfLine[matchName.start() : matchName.end()] - ) - + "_" - ) # Search for a closing brace. matchClosingBrace = re.search("}", statement) if isEnum is True and matchClosingBrace is not None: isEnum = False - enumName = "" continue - def convert(name): - s1 = re.sub(r"(.)([A-Z][a-z]+)", r"\1_\2", name) - return re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", s1).upper() - # Check if not inside an enum. if isEnum is False: # Search for "message".