From 65ca9aef292f40ff4d90a3963f7d873862fa75ca Mon Sep 17 00:00:00 2001 From: Jaekwon Bang Date: Fri, 3 Feb 2023 10:18:50 +0900 Subject: [PATCH 1/3] Add handling for parsing yaml error --- src/fosslight_util/compare_yaml.py | 4 ++-- src/fosslight_util/parsing_yaml.py | 20 +++++++++++++++----- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/fosslight_util/compare_yaml.py b/src/fosslight_util/compare_yaml.py index 0e334d3..e0c7812 100644 --- a/src/fosslight_util/compare_yaml.py +++ b/src/fosslight_util/compare_yaml.py @@ -15,8 +15,8 @@ def compare_yaml(before_file, after_file): - before_oss_items, _ = parsing_yml(before_file, os.path.dirname(before_file)) - after_oss_items, _ = parsing_yml(after_file, os.path.dirname(after_file)) + before_oss_items, _, _ = parsing_yml(before_file, os.path.dirname(before_file)) + after_oss_items, _, _ = parsing_yml(after_file, os.path.dirname(after_file)) before_items = get_merged_item(before_oss_items) after_items = get_merged_item(after_oss_items) diff --git a/src/fosslight_util/parsing_yaml.py b/src/fosslight_util/parsing_yaml.py index 20d765e..b19fc7c 100644 --- a/src/fosslight_util/parsing_yaml.py +++ b/src/fosslight_util/parsing_yaml.py @@ -20,6 +20,7 @@ def parsing_yml(yaml_file, base_path, print_log=True): oss_list = [] license_list = [] idx = 1 + err_reason = "" OLD_YAML_ROOT_ELEMENT = ['Open Source Software Package', 'Open Source Package'] @@ -32,6 +33,11 @@ def parsing_yml(yaml_file, base_path, print_log=True): else: relative_path = "" doc = yaml.safe_load(codecs.open(yaml_file, "r", "utf-8")) + # If yaml file is empty, return immediately + if doc is None: + err_reason = "empty" + return oss_list, license_list, err_reason + is_old_format = any(x in doc for x in OLD_YAML_ROOT_ELEMENT) for root_element in doc: @@ -46,19 +52,21 @@ def parsing_yml(yaml_file, base_path, print_log=True): for key, value in oss.items(): if key: key = key.lower().strip() - set_value_switch(item, key, value) + set_value_switch(item, key, value, yaml_file) oss_list.append(item) license_list.extend(item.license) idx += 1 except AttributeError as ex: if print_log: - _logger.error(f"Not supported yaml file format {ex}") + _logger.warning(f"Not supported yaml file format: {yaml_file} {ex}") oss_list = [] + err_reason = "not_supported" except yaml.YAMLError: if print_log: - _logger.warning(f"Can't parse yaml - skip to parse yaml file: {yaml_file}") + _logger.warning(f"Error to parse yaml - skip to parse yaml file: {yaml_file}") oss_list = [] - return oss_list, set(license_list) + err_reason = "yaml_error" + return oss_list, set(license_list), err_reason def find_sbom_yaml_files(path_to_find): @@ -82,7 +90,7 @@ def find_sbom_yaml_files(path_to_find): return oss_pkg_files -def set_value_switch(oss, key, value): +def set_value_switch(oss, key, value, yaml_file): if key in ['oss name', 'name']: oss.name = value elif key in ['oss version', 'version']: @@ -105,3 +113,5 @@ def set_value_switch(oss, key, value): oss.yocto_package = value elif key == 'yocto_recipe': oss.yocto_recipe = value + else: + _logger.info(f"file:{yaml_file} - key:{key} cannot be parsed") From a42cd0e4a27a83f2493686071506737513439526 Mon Sep 17 00:00:00 2001 From: Jaekwon Bang Date: Wed, 8 Feb 2023 16:12:02 +0900 Subject: [PATCH 2/3] Change log priority at parsing yaml --- src/fosslight_util/parsing_yaml.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/fosslight_util/parsing_yaml.py b/src/fosslight_util/parsing_yaml.py index b19fc7c..18bd3ec 100644 --- a/src/fosslight_util/parsing_yaml.py +++ b/src/fosslight_util/parsing_yaml.py @@ -36,6 +36,7 @@ def parsing_yml(yaml_file, base_path, print_log=True): # If yaml file is empty, return immediately if doc is None: err_reason = "empty" + _logger.debug(f"The yaml file is empty file: {yaml_file}") return oss_list, license_list, err_reason is_old_format = any(x in doc for x in OLD_YAML_ROOT_ELEMENT) @@ -58,12 +59,12 @@ def parsing_yml(yaml_file, base_path, print_log=True): idx += 1 except AttributeError as ex: if print_log: - _logger.warning(f"Not supported yaml file format: {yaml_file} {ex}") + _logger.debug(f"Not supported yaml file format: {yaml_file} {ex}") oss_list = [] err_reason = "not_supported" except yaml.YAMLError: if print_log: - _logger.warning(f"Error to parse yaml - skip to parse yaml file: {yaml_file}") + _logger.debug(f"Error to parse yaml - skip to parse yaml file: {yaml_file}") oss_list = [] err_reason = "yaml_error" return oss_list, set(license_list), err_reason @@ -114,4 +115,4 @@ def set_value_switch(oss, key, value, yaml_file): elif key == 'yocto_recipe': oss.yocto_recipe = value else: - _logger.info(f"file:{yaml_file} - key:{key} cannot be parsed") + _logger.debug(f"file:{yaml_file} - key:{key} cannot be parsed") From 9e38b67edb8bbe300d9823cce6f9a29f9d745524 Mon Sep 17 00:00:00 2001 From: Jaekwon Bang Date: Thu, 9 Feb 2023 11:02:44 +0900 Subject: [PATCH 3/3] Modify to print log using print_log --- src/fosslight_util/parsing_yaml.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/fosslight_util/parsing_yaml.py b/src/fosslight_util/parsing_yaml.py index 18bd3ec..69fc49a 100644 --- a/src/fosslight_util/parsing_yaml.py +++ b/src/fosslight_util/parsing_yaml.py @@ -36,7 +36,8 @@ def parsing_yml(yaml_file, base_path, print_log=True): # If yaml file is empty, return immediately if doc is None: err_reason = "empty" - _logger.debug(f"The yaml file is empty file: {yaml_file}") + if print_log: + _logger.warning(f"The yaml file is empty file: {yaml_file}") return oss_list, license_list, err_reason is_old_format = any(x in doc for x in OLD_YAML_ROOT_ELEMENT) @@ -59,12 +60,12 @@ def parsing_yml(yaml_file, base_path, print_log=True): idx += 1 except AttributeError as ex: if print_log: - _logger.debug(f"Not supported yaml file format: {yaml_file} {ex}") + _logger.warning(f"Not supported yaml file format: {yaml_file} {ex}") oss_list = [] err_reason = "not_supported" except yaml.YAMLError: if print_log: - _logger.debug(f"Error to parse yaml - skip to parse yaml file: {yaml_file}") + _logger.warning(f"Error to parse yaml - skip to parse yaml file: {yaml_file}") oss_list = [] err_reason = "yaml_error" return oss_list, set(license_list), err_reason