From bd375890ae5f0617d643936466a690f299dfe972 Mon Sep 17 00:00:00 2001 From: Jiyeong Seok Date: Tue, 30 May 2023 17:38:09 +0900 Subject: [PATCH 1/4] Fix to check exclude for correcting oss info Signed-off-by: Jiyeong Seok --- src/fosslight_util/correct.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/fosslight_util/correct.py b/src/fosslight_util/correct.py index bb3d49a..11a5086 100644 --- a/src/fosslight_util/correct.py +++ b/src/fosslight_util/correct.py @@ -42,7 +42,7 @@ def correct_with_yaml(correct_filepath, path_to_scan, scanner_oss_list): if sheet_name not in constant.supported_sheet_and_scanner.keys(): continue correct_contents = copy.deepcopy(sheet_contents) - for oss_raw_item in sheet_contents: + for idx, oss_raw_item in enumerate(sheet_contents): if len(oss_raw_item) < 9: logger.warning(f"sheet list is too short ({len(oss_raw_item)}): {oss_raw_item}") continue @@ -65,19 +65,27 @@ def correct_with_yaml(correct_filepath, path_to_scan, scanner_oss_list): find_match = True matched_yi.append(yi) if len(matched_yi) > 0: - correct_contents.remove(oss_raw_item) for mi in matched_yi: - if ','.join(sorted(mi.license)).casefold() != ','.join(sorted(oss_item.license)).casefold(): - if len(oss_item.license) > 0 and oss_item.license != ['']: - mi.comment = f'scanner license: {",".join(oss_item.license)}' - matched_yaml_item = mi.get_print_array()[0] - matched_yaml_item[0] = oss_item.source_name_or_path[0] - correct_contents.append(matched_yaml_item) + matched_oss_item = copy.deepcopy(mi) + if oss_item.exclude: + matched_oss_item.exclude = oss_item.exclude + if matched_oss_item.comment: + matched_oss_item.comment += '/' + matched_oss_item.comment += 'Loaded from sbom-info.yaml' + matched_oss_item.source_name_or_path = [] + matched_oss_item.source_name_or_path = oss_item.source_name_or_path[0] + matched_oss_array = matched_oss_item.get_print_array()[0] + correct_contents.append(matched_oss_array) + oss_item.exclude = True + if oss_item.comment: + oss_item.comment += '/' + oss_item.comment += 'Excluded by sbom-info.yaml' + correct_contents[idx] = oss_item.get_print_array()[0] correct_list[sheet_name] = correct_contents if not find_match: success = False - err_msg = f'No match items in {SBOM_INFO_YAML}' + err_msg = 'No match items in sbom-info.yaml' return success, err_msg, yaml_oss_list return success, msg, correct_list From 56fcd3f19619aa1d4a45533680384398d2a1edcb Mon Sep 17 00:00:00 2001 From: Jiyeong Seok Date: Fri, 2 Jun 2023 08:08:36 +0900 Subject: [PATCH 2/4] Add result if path not found in scanner Signed-off-by: Jiyeong Seok --- src/fosslight_util/correct.py | 62 +++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/src/fosslight_util/correct.py b/src/fosslight_util/correct.py index 11a5086..a911ff0 100644 --- a/src/fosslight_util/correct.py +++ b/src/fosslight_util/correct.py @@ -22,7 +22,7 @@ def correct_with_yaml(correct_filepath, path_to_scan, scanner_oss_list): correct_list = {} correct_yaml = "" if correct_filepath == "": - correct_filepath = os.getcwd() + correct_filepath = path_to_scan for filename in os.listdir(correct_filepath): if re.search(SBOM_INFO_YAML, filename, re.IGNORECASE): @@ -34,10 +34,12 @@ def correct_with_yaml(correct_filepath, path_to_scan, scanner_oss_list): return success, msg, correct_list rel_path = os.path.relpath(path_to_scan, correct_filepath) + rel_correct_path = os.path.relpath(correct_filepath, path_to_scan) yaml_oss_list, _, err_msg = parsing_yml(correct_yaml, os.path.dirname(correct_yaml), print_log=True) find_match = False + matched_yaml = [[0]]*len(yaml_oss_list) for sheet_name, sheet_contents in scanner_oss_list.items(): if sheet_name not in constant.supported_sheet_and_scanner.keys(): continue @@ -50,30 +52,28 @@ def correct_with_yaml(correct_filepath, path_to_scan, scanner_oss_list): oss_item.set_sheet_item(oss_raw_item) matched_yi = [] - oss_rel_path = os.path.join(rel_path, oss_item.source_name_or_path[0]) - for yi in yaml_oss_list: - filtered_exact = next(filter(lambda ys: os.path.normpath(ys) == os.path.normpath(oss_rel_path), - yi.source_name_or_path), None) - if filtered_exact: - find_match = True - matched_yi.append(yi) + oss_rel_path = os.path.normpath(os.path.join(rel_path, oss_item.source_name_or_path[0])) + for y_idx, yi in enumerate(yaml_oss_list): + if not yi.source_name_or_path: continue - filtered_contain = next(filter(lambda ys: - os.path.normpath(oss_rel_path).startswith(os.path.normpath(ys.rstrip('*'))), - yi.source_name_or_path), None) - if filtered_contain: - find_match = True - matched_yi.append(yi) + for ys_idx, yi_path in enumerate(yi.source_name_or_path): + yi_item = copy.deepcopy(yi) + if ((os.path.normpath(yi_path) == os.path.normpath(oss_rel_path)) + or ((os.path.normpath(oss_rel_path).startswith(os.path.normpath(yi_path.rstrip('*')))))): + find_match = True + yi_item.source_name_or_path = [] + yi_item.source_name_or_path = oss_item.source_name_or_path[0] + matched_yi.append(yi_item) + matched_yaml[y_idx].append(ys_idx + 1) + matched_yaml[y_idx] = list(set(matched_yaml[y_idx])) if len(matched_yi) > 0: - for mi in matched_yi: - matched_oss_item = copy.deepcopy(mi) + for matched_yi_item in matched_yi: + matched_oss_item = copy.deepcopy(matched_yi_item) if oss_item.exclude: matched_oss_item.exclude = oss_item.exclude if matched_oss_item.comment: matched_oss_item.comment += '/' matched_oss_item.comment += 'Loaded from sbom-info.yaml' - matched_oss_item.source_name_or_path = [] - matched_oss_item.source_name_or_path = oss_item.source_name_or_path[0] matched_oss_array = matched_oss_item.get_print_array()[0] correct_contents.append(matched_oss_array) oss_item.exclude = True @@ -81,8 +81,34 @@ def correct_with_yaml(correct_filepath, path_to_scan, scanner_oss_list): oss_item.comment += '/' oss_item.comment += 'Excluded by sbom-info.yaml' correct_contents[idx] = oss_item.get_print_array()[0] + + if sheet_name == 'SRC_FL_Source': + for n_idx, ni in enumerate(matched_yaml): + y_item = copy.deepcopy(yaml_oss_list[n_idx]) + if sum(ni) != 0: + ni = list(set(ni)) + if len(ni[1:]) == len(y_item.source_name_or_path): + continue + id = [j-1 for j in ni if j] + for iid in sorted(id, reverse=True): + y_item.source_name_or_path.pop(iid) + if y_item.comment: + y_item.comment += '/' + y_item.comment += 'Added by sbom-info.yaml' + if not y_item.source_name_or_path: + correct_contents.append(y_item.get_print_array()[0]) + continue + for y_path in y_item.source_name_or_path: + y_item_i = copy.deepcopy(y_item) + if not os.path.exists(os.path.normpath(os.path.join(correct_filepath, y_path))): + y_item_i.exclude = True + y_item_i.source_name_or_path = [] + y_item_i.source_name_or_path = os.path.join(rel_correct_path, y_path) + correct_contents.append(y_item_i.get_print_array()[0]) + correct_list[sheet_name] = correct_contents + if not find_match: success = False err_msg = 'No match items in sbom-info.yaml' From 43e230a080a94977a4314ccc345f4d1a04a738a7 Mon Sep 17 00:00:00 2001 From: Jiyeong Seok Date: Fri, 2 Jun 2023 08:17:44 +0900 Subject: [PATCH 3/4] fix the tox error Signed-off-by: Jiyeong Seok --- src/fosslight_util/correct.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/fosslight_util/correct.py b/src/fosslight_util/correct.py index 780ebb6..00f5109 100644 --- a/src/fosslight_util/correct.py +++ b/src/fosslight_util/correct.py @@ -107,7 +107,6 @@ def correct_with_yaml(correct_filepath, path_to_scan, scanner_oss_list): correct_contents.append(y_item_i.get_print_array()[0]) correct_list[sheet_name] = correct_contents - if not find_match: success = False err_msg = 'No match items in sbom-info.yaml' From b95295ee4d2b7ac92e15a8a86ebefa1307884ccf Mon Sep 17 00:00:00 2001 From: Jiyeong Seok Date: Fri, 2 Jun 2023 10:20:41 +0900 Subject: [PATCH 4/4] Fix the error Signed-off-by: Jiyeong Seok --- src/fosslight_util/correct.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/fosslight_util/correct.py b/src/fosslight_util/correct.py index 00f5109..5769cf2 100644 --- a/src/fosslight_util/correct.py +++ b/src/fosslight_util/correct.py @@ -24,6 +24,8 @@ def correct_with_yaml(correct_filepath, path_to_scan, scanner_oss_list): if correct_filepath == "": correct_filepath = path_to_scan + path_to_scan = os.path.normpath(path_to_scan) + correct_filepath = os.path.normpath(correct_filepath) for filename in os.listdir(correct_filepath): if re.search(SBOM_INFO_YAML, filename, re.IGNORECASE): correct_yaml = os.path.join(correct_filepath, filename) @@ -39,7 +41,10 @@ def correct_with_yaml(correct_filepath, path_to_scan, scanner_oss_list): yaml_oss_list, _, err_msg = parsing_yml(correct_yaml, os.path.dirname(correct_yaml), print_log=True) find_match = False - matched_yaml = [[0]]*len(yaml_oss_list) + matched_yaml = [] + for yitem in yaml_oss_list: + matched_yaml.append([0]*len(yitem.source_name_or_path)) + for sheet_name, sheet_contents in scanner_oss_list.items(): if sheet_name not in constant.supported_sheet_and_scanner.keys(): continue @@ -64,8 +69,7 @@ def correct_with_yaml(correct_filepath, path_to_scan, scanner_oss_list): yi_item.source_name_or_path = [] yi_item.source_name_or_path = oss_item.source_name_or_path[0] matched_yi.append(yi_item) - matched_yaml[y_idx].append(ys_idx + 1) - matched_yaml[y_idx] = list(set(matched_yaml[y_idx])) + matched_yaml[y_idx][ys_idx] = 1 if len(matched_yi) > 0: for matched_yi_item in matched_yi: matched_oss_item = copy.deepcopy(matched_yi_item) @@ -86,12 +90,12 @@ def correct_with_yaml(correct_filepath, path_to_scan, scanner_oss_list): for n_idx, ni in enumerate(matched_yaml): y_item = copy.deepcopy(yaml_oss_list[n_idx]) if sum(ni) != 0: - ni = list(set(ni)) - if len(ni[1:]) == len(y_item.source_name_or_path): - continue - id = [j-1 for j in ni if j] - for iid in sorted(id, reverse=True): - y_item.source_name_or_path.pop(iid) + not_matched_path = [] + for idx, id in enumerate(ni): + if not id: + not_matched_path.append(y_item.source_name_or_path[idx]) + y_item.source_name_or_path = [] + y_item.source_name_or_path = not_matched_path if y_item.comment: y_item.comment += '/' y_item.comment += 'Added by sbom-info.yaml'