From 1944caeb43a32da0901aca127a07d8d35def8df5 Mon Sep 17 00:00:00 2001 From: Jiyeong Seok Date: Mon, 16 May 2022 14:43:34 +0900 Subject: [PATCH 1/7] Add to write yaml Signed-off-by: Jiyeong Seok --- src/fosslight_util/oss_item.py | 83 +++++++++++++++++---- src/fosslight_util/output_format.py | 5 +- src/fosslight_util/write_yaml.py | 74 +++++++++++++++++++ tests/test_yaml.py | 109 ++++++++++++++++++++++++++++ tox.ini | 4 + 5 files changed, 259 insertions(+), 16 deletions(-) create mode 100644 src/fosslight_util/write_yaml.py create mode 100644 tests/test_yaml.py diff --git a/src/fosslight_util/oss_item.py b/src/fosslight_util/oss_item.py index 2ffb968..77e4537 100644 --- a/src/fosslight_util/oss_item.py +++ b/src/fosslight_util/oss_item.py @@ -20,6 +20,9 @@ class OssItem: exclude = "" homepage = "" relative_path = "" + source_name_or_path = [] + download_location = "" + copyright_text = "" def __init__(self, value): self.name = "-" @@ -32,6 +35,9 @@ def __init__(self, value): self.exclude = "" self.homepage = "" self.relative_path = value + self.source_name_or_path = [] + self.download_location = "" + self.copyright_text = "" def __del__(self): pass @@ -51,15 +57,19 @@ def set_copyright(self, value): def set_exclude(self, value): if value: - self.exclude = "Exclude" + self.exclude = True else: - self.exclude = "" + self.exclude = False def set_name(self, value): - self.name = value + if value != "": + self.name = value def set_version(self, value): - self.version = str(value) + if value: + self.version = str(value) + else: + self.version = "" def set_licenses(self, value): if not isinstance(value, list): @@ -78,41 +88,84 @@ def set_files(self, value): def set_source(self, value): self.source = value + # new keys for updated yaml format + # replace 'file' key + def set_source_name_or_path(self, value): + if not isinstance(value, list): + value = value.split(",") + self.source_name_or_path.extend(value) + self.source_name_or_path = [item.strip() for item in self.source_name_or_path] + self.source_name_or_path = list(set(self.source_name_or_path)) + + # replace 'source' key + def set_download_location(self, value): + self.download_location = value + + # replace 'copyright' key + def set_copyright_text(self, value): + self.copyright_text = value + + def set_sheet_item(self, item): + switcher = { + 0: self.set_source_name_or_path, + 1: self.set_name, + 2: self.set_version, + 3: self.set_licenses, + 4: self.set_download_location, + 5: self.set_homepage, + 6: self.set_copyright_text, + 7: self.set_exclude, + 8: self.set_comment + } + + for i in range(0, len(switcher)): + func = switcher.get(i, "invalid") + func(item[i]) + def get_print_array(self): items = [] if len(self.files) == 0: self.files.append("") if len(self.licenses) == 0: self.licenses.append("") + + exclude = "" + if self.exclude: + exclude = "Exclude" + for file in self.files: lic = ",".join(self.licenses) if self.relative_path != "" and not str(self.relative_path).endswith("/"): self.relative_path += "/" items.append([self.relative_path + file, self.name, self.version, lic, self.source, self.homepage, - self.copyright, "", self.exclude, self.comment]) + self.copyright, "", exclude, self.comment]) return items def get_print_json(self): json_item = {} json_item["name"] = self.name - if self.version != "": - json_item["version"] = self.version + json_item["version"] = self.version + if len(self.files) > 0: + json_item["file"] = self.files + if len(self.source_name_or_path) > 0: + json_item["source_name_or_path"] = self.source_name_or_path + if len(self.licenses) > 0: + json_item["license"] = self.licenses if self.source != "": json_item["source"] = self.source + if self.download_location != "": + json_item["download_location"] = self.download_location + if self.homepage != "": + json_item["homepage"] = self.homepage if self.copyright != "": json_item["copyright"] = self.copyright - if self.exclude != "": + if self.copyright_text != "": + json_item["copyright_text"] = self.copyright_text + if self.exclude: json_item["exclude"] = self.exclude if self.comment != "": json_item["comment"] = self.comment - if self.homepage != "": - json_item["homepage"] = self.homepage - - if len(self.files) > 0: - json_item["file"] = self.files - if len(self.licenses) > 0: - json_item["license"] = self.licenses return json_item diff --git a/src/fosslight_util/output_format.py b/src/fosslight_util/output_format.py index d30f965..d780de1 100644 --- a/src/fosslight_util/output_format.py +++ b/src/fosslight_util/output_format.py @@ -5,8 +5,9 @@ import os from fosslight_util.write_excel import write_result_to_excel, write_excel_and_csv, write_result_to_csv from fosslight_util.write_opossum import write_opossum +from fosslight_util.write_yaml import write_yaml -SUPPORT_FORMAT = {'excel': '.xlsx', 'csv': '.csv', 'opossum': '.json'} +SUPPORT_FORMAT = {'excel': '.xlsx', 'csv': '.csv', 'opossum': '.json', 'yaml': '.yaml'} def check_output_format(output='', format='', customized_format={}): @@ -71,6 +72,8 @@ def write_output_file(output_file_without_ext, file_extension, sheet_list, exten success, msg, result_file = write_result_to_csv(result_file, sheet_list) elif file_extension == '.json': success, msg = write_opossum(result_file, sheet_list) + elif file_extension == '.yaml': + success, msg, result_file = write_yaml(result_file, sheet_list, False) else: success = False msg = f'Not supported file extension({file_extension})' diff --git a/src/fosslight_util/write_yaml.py b/src/fosslight_util/write_yaml.py new file mode 100644 index 0000000..5f73f7c --- /dev/null +++ b/src/fosslight_util/write_yaml.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2022 LG Electronics Inc. +# SPDX-License-Identifier: Apache-2.0 + +import yaml +import logging +import os +import copy +from pathlib import Path +from .constant import LOGGER_NAME +from .oss_item import OssItem +from fosslight_util.write_excel import remove_empty_sheet, _EMPTY_ITEM_MSG + +_logger = logging.getLogger(LOGGER_NAME) + + +def write_yaml(output_file, sheet_list_origin, separate_yaml=False): + success = True + error_msg = "" + output = "" + + try: + sheet_list = copy.deepcopy(sheet_list_origin) + is_not_null, sheet_list = remove_empty_sheet(sheet_list) + if is_not_null: + output_files = [] + output_dir = os.path.dirname(output_file) + + Path(output_dir).mkdir(parents=True, exist_ok=True) + if separate_yaml: + filename = os.path.splitext(os.path.basename(output_file))[0] + separate_output_file = os.path.join(output_dir, filename) + + merge_sheet = [] + for sheet_name, sheet_contents in sheet_list.items(): + if not separate_yaml: + merge_sheet.extend(sheet_contents) + if sheet_name == list(sheet_list.keys())[-1]: + sheet_contents = merge_sheet + else: + continue + else: + output_file = f'{separate_output_file}_{sheet_name}.yaml' + + sheet_contents = [list(t) for t in set(tuple(e) for e in sheet_contents)] + + yaml_dict = {} + for sheet_item in sheet_contents: + item = OssItem('') + item.set_sheet_item(sheet_item) + item_json = copy.deepcopy(item.get_print_json()) + item_name = item_json.pop("name") + if item_name not in yaml_dict.keys(): + yaml_dict[item_name] = [] + yaml_dict[item_name].append(item_json) + + with open(output_file, 'w') as f: + yaml.dump(yaml_dict, f, default_flow_style=False, sort_keys=False) + + output_files.append(output_file) + if output_files: + output = ", ".join(output_files) + else: + success = False + error_msg = _EMPTY_ITEM_MSG + except Exception as ex: + error_msg = str(ex) + success = False + + if not success: + error_msg = "[Error] Writing yaml:" + error_msg + + return success, error_msg, output diff --git a/tests/test_yaml.py b/tests/test_yaml.py new file mode 100644 index 0000000..6ada042 --- /dev/null +++ b/tests/test_yaml.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2022 LG Electronics Inc. +# SPDX-License-Identifier: Apache-2.0 +from fosslight_util.write_yaml import write_yaml +from fosslight_util.set_log import init_log + + +def main(): + logger, _result_log = init_log("test_result/excel/log_write_yaml.txt") + logger.warning("TESTING - Writing a yaml") + + sheet_list = {'SRC_FL_Source': [ + ['test/lib/babel-polyfill.js', '', '', 'bsd-3-clause,facebook-patent-rights-2', '', '', + 'Copyright (c) 2014, Facebook, Inc.', 'Exclude', ''], + ['test/lib/babel-polyfill.js', '', '', 'bsd-3-clause,facebook-patent-rights-2', '', '', + 'Copyright (c) 2014, Facebook, Inc.', 'Exclude', ''], + ['lib/babel-polyfill.js', '', '', 'bsd-3-clause', '', '', + 'Copyright (c) 2014, Facebook, Inc.', '', ''], + ['lib/babel-polyfill.js', '', '', 'facebook-patent-rights-2', '', '', + 'Copyright (c) 2014, Facebook, Inc.', '', ''], + ['requirements.txt', '', '', 'MIT', 'https://pypi.org/project/future/0.18.2', '', '', '', ''], + ['bower.json', '', '', 'mit', '', '', '', '', ''], + ['LICENSE', '', '', 'mit', '', '', 'Copyright (c) 2016-2021, The Cytoscape Consortium', '', ''], + ['license-update.js', '', '', 'mit', '', '', 'Copyright (c) 2016-$ year, The Cytoscape Consortium', '', ''], + ['package.json', '', '', 'mit', '', '', '', '', ''], ['README.md', '', '', 'mit', '', '', '', '', ''], + ['dist/cytoscape.cjs.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,Copyright (c) 2016-2021,c \ + The Cytoscape Consortium,copyright Koen Bok,Copyright (c) 2013-2014 Ralf S. Engelschall \ + (http://engelschall.com)', '', ''], + ['dist/cytoscape.esm.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,Copyright (c) 2016-2021,\ + The Cytoscape Consortium,copyright Koen Bok,Copyright (c) 2013-2014 Ralf S. Engelschall \ + (http://engelschall.com)', '', ''], + ['dist/cytoscape.esm.min.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,copyright Koen Bok, \ + Copyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''], + ['dist/cytoscape.min.js', '', '', 'mit', + '', '', 'Copyright Gaetan Renaudeau,Copyright (c) 2016-2021, The Cytoscape Consortium,copyright Koen Bok,Copyright \ + (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''], + ['dist/cytoscape.umd.js', '', '', 'mit', '', '', + 'Copyright Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors,Copyright jQuery Foundation \ + and other contributors ,Copyright (c) 2016-2021, The Cytoscape Consortium,copyright Koen\ + Bok,Copyright Gaetan Renaudeau,Copyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''], + ['documentation/css/highlight/monokai_sublime.css', '', '', 'mit', '', '', '', '', ''], + ['documentation/js/cytoscape.min.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,\ + Copyright (c) 2016-2021, The Cytoscape Consortium,copyright Koen Bok, \ + Copyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''], + ['documentation/md/links.md', '', '', 'mit', '', '', '', '', ''], + ['src/event.js', '', '', 'mit', '', '', '', '', '']], + 'BIN_FL_Binary': [ + ['askalono_macos', 'askalono', '', 'Apache-2.0', '', '', '', '', ''], + ['test/askalono_macos', 'askalono', '', 'Apache-2.0', '', '', '', 'Exclude', '']], + 'SRC_FL_Dependency': [ + ['requirements.txt', 'pypi:future', '0.18.2', 'MIT', 'https://pypi.org/project/future/0.18.2', + 'https://python-future.org', '', '', ''], + ['requirements.txt', 'pypi:numpy', '1.19.5', 'BSD-3-Clause-Open-MPI,GCC-exception-3.1,GPL-3.0', + 'https://pypi.org/project/numpy/1.19.5', 'https://www.numpy.org', '', '', ''], + ['requirements.txt', 'pypi:pandas', '1.1.5', 'BSD-3-Clause', 'https://pypi.org/project/pandas/1.1.5', + 'https://pandas.pydata.org', '', '', '']]} + + sheet_list2 = {'SRC_FL_Dependency': [ + ['requirements.txt', 'pypi:future', '0.18.2', 'MIT', 'https://pypi.org/project/future/0.18.2', + 'https://python-future.org', '', '', ''], + ['requirements.txt', 'pypi:numpy', '1.19.5', 'BSD-3-Clause-Open-MPI,GCC-exception-3.1,GPL-3.0', + 'https://pypi.org/project/numpy/1.19.5', 'https://www.numpy.org', '', '', ''], + ['requirements.txt', 'pypi:pandas', '1.1.5', 'BSD-3-Clause', 'https://pypi.org/project/pandas/1.1.5', + 'https://pandas.pydata.org', '', '', '']], + 'SRC_FL_Source': [ + ['test/lib/babel-polyfill.js', '', '', 'bsd-3-clause,facebook-patent-rights-2', '', '', + 'Copyright (c) 2014, Facebook, Inc.', 'Exclude', ''], + ['requirements.txt', '', '', 'MIT', 'https://pypi.org/project/future/0.18.2', '', '', '', ''], + ['bower.json', '', '', 'mit', '', '', '', '', ''], + ['LICENSE', '', '', 'mit', '', '', 'Copyright (c) 2016-2021, The Cytoscape Consortium', '', ''], + ['license-update.js', '', '', 'mit', '', '', 'Copyright (c) 2016-$ year, The Cytoscape Consortium', '', ''], + ['package.json', '', '', 'mit', '', '', '', '', ''], ['README.md', '', '', 'mit', '', '', '', '', ''], + ['dist/cytoscape.cjs.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,Copyright (c) 2016-2021,c \ + The Cytoscape Consortium,copyright Koen Bok,Copyright (c) 2013-2014 Ralf S. Engelschall \ + (http://engelschall.com)', '', ''], + ['dist/cytoscape.esm.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,Copyright (c) 2016-2021,\ + The Cytoscape Consortium,copyright Koen Bok,Copyright (c) 2013-2014 Ralf S. Engelschall \ + (http://engelschall.com)', '', ''], + ['dist/cytoscape.esm.min.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,copyright Koen Bok, \ + Copyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''], + ['dist/cytoscape.min.js', '', '', 'mit', + '', '', 'Copyright Gaetan Renaudeau,Copyright (c) 2016-2021, The Cytoscape Consortium,copyright Koen Bok,Copyright \ + (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''], + ['dist/cytoscape.umd.js', '', '', 'mit', '', '', + 'Copyright Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors,Copyright jQuery Foundation \ + and other contributors ,Copyright (c) 2016-2021, The Cytoscape Consortium,copyright Koen\ + Bok,Copyright Gaetan Renaudeau,Copyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''], + ['documentation/css/highlight/monokai_sublime.css', '', '', 'mit', '', '', '', '', ''], + ['documentation/js/cytoscape.min.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,\ + Copyright (c) 2016-2021, The Cytoscape Consortium,copyright Koen Bok, \ + Copyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''], + ['documentation/md/links.md', '', '', 'mit', '', '', '', '', ''], + ['src/event.js', '', '', 'mit', '', '', '', '', '']], + 'BIN_FL_Binary': [ + ['askalono_macos', 'askalono', '', 'Apache-2.0', '', '', '', '', ''], + ['test/askalono_macos', 'askalono', '', 'Apache-2.0', '', '', '', 'Exclude', '']]} + + success, msg, output = write_yaml( + 'test_result/yaml/FL-TEST_yaml.yaml', sheet_list) + logger.warning(f"Result: {str(success)}, error_msg: {msg}, Output_files: {output}") + + success, msg, output = write_yaml( + 'test_result/yaml/FL-TEST2_yaml.yaml', sheet_list2) + logger.warning(f"Result: {str(success)}, error_msg: {msg}, Output_files: {output}") + + +if __name__ == '__main__': + main() diff --git a/tox.ini b/tox.ini index b6a3ea6..a0357f1 100644 --- a/tox.ini +++ b/tox.ini @@ -78,6 +78,10 @@ commands = # Test - writing opossum python tests/test_opossum.py ls test_result/opossum + # Test - writing yaml + python tests/test_yaml.py + cat test_result/yaml/FL-TEST_yaml.yaml + cat test_result/yaml/FL-TEST2_yaml.yaml # Test - timer python tests/test_timer.py # Test - downloading source From 272925977412dc8938ab9c98876a7e777caa5bf0 Mon Sep 17 00:00:00 2001 From: Jiyeong Seok Date: Mon, 16 May 2022 15:08:55 +0900 Subject: [PATCH 2/7] Change the exclude default value to false Signed-off-by: Jiyeong Seok --- src/fosslight_util/oss_item.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fosslight_util/oss_item.py b/src/fosslight_util/oss_item.py index 77e4537..ea1b150 100644 --- a/src/fosslight_util/oss_item.py +++ b/src/fosslight_util/oss_item.py @@ -32,7 +32,7 @@ def __init__(self, value): self.files = [] self.copyright = "" self.comment = "" - self.exclude = "" + self.exclude = False self.homepage = "" self.relative_path = value self.source_name_or_path = [] From 33289fe19f1990fa1b31f68ec55b7347bcfdf12d Mon Sep 17 00:00:00 2001 From: Jiyeong Seok Date: Tue, 17 May 2022 11:28:13 +0900 Subject: [PATCH 3/7] Fix not to print matched_text sheet and change property Signed-off-by: Jiyeong Seok --- src/fosslight_util/constant.py | 9 ++ src/fosslight_util/oss_item.py | 165 +++++++++++++++------------- src/fosslight_util/write_opossum.py | 30 ++--- src/fosslight_util/write_yaml.py | 49 +++++---- tests/test_yaml.py | 2 +- 5 files changed, 136 insertions(+), 119 deletions(-) diff --git a/src/fosslight_util/constant.py b/src/fosslight_util/constant.py index 0fa530a..f0796b5 100755 --- a/src/fosslight_util/constant.py +++ b/src/fosslight_util/constant.py @@ -4,3 +4,12 @@ # SPDX-License-Identifier: Apache-2.0 LOGGER_NAME = "FOSSLight" + +FL_SOURCE = 'FL_Source' +FL_DEPENDENCY = 'FL_Dependency' +FL_BINARY = 'FL_Binary' +supported_sheet_and_scanner = {'SRC': FL_SOURCE, + 'BIN': FL_BINARY, + f'SRC_{FL_SOURCE}': FL_SOURCE, + f'SRC_{FL_DEPENDENCY}': FL_DEPENDENCY, + f'BIN_{FL_BINARY}': FL_BINARY} diff --git a/src/fosslight_util/oss_item.py b/src/fosslight_util/oss_item.py index ea1b150..77906ba 100644 --- a/src/fosslight_util/oss_item.py +++ b/src/fosslight_util/oss_item.py @@ -10,117 +10,128 @@ class OssItem: - name = "-" - version = "" - licenses = [] - source = "" - files = [] - copyright = "" - comment = "" - exclude = "" - homepage = "" - relative_path = "" - source_name_or_path = [] - download_location = "" - copyright_text = "" - def __init__(self, value): - self.name = "-" - self.version = "" - self.licenses = [] + self._name = "-" + self._version = "" + self._licenses = [] self.source = "" - self.files = [] - self.copyright = "" + self._files = [] + self._copyright = "" self.comment = "" - self.exclude = False + self._exclude = False self.homepage = "" self.relative_path = value - self.source_name_or_path = [] + self._source_name_or_path = [] self.download_location = "" - self.copyright_text = "" + self._copyright_text = "" def __del__(self): pass - def set_homepage(self, value): - self.homepage = value + @property + def copyright(self): + return self._copyright - def set_comment(self, value): - self.comment = value + @copyright.setter + def copyright(self, value): + if value != "": + if isinstance(value, list): + value = "\n".join(value) + value = value.strip() + self._copyright = value + + @property + def copyright_text(self): + return self._copyright_text - def set_copyright(self, value): + @copyright_text.setter + def copyright_text(self, value): if value != "": if isinstance(value, list): value = "\n".join(value) value = value.strip() - self.copyright = value + self._copyright_text = value + + @property + def exclude(self): + return self._exclude - def set_exclude(self, value): + @exclude.setter + def exclude(self, value): if value: - self.exclude = True + self._exclude = True else: - self.exclude = False + self._exclude = False - def set_name(self, value): + @property + def name(self): + return self._name + + @name.setter + def name(self, value): if value != "": - self.name = value + self._name = value + + @property + def version(self): + return self._version - def set_version(self, value): + @version.setter + def version(self, value): if value: - self.version = str(value) + self._version = str(value) else: - self.version = "" + self._version = "" - def set_licenses(self, value): + @property + def licenses(self): + return self._licenses + + @licenses.setter + def licenses(self, value): if not isinstance(value, list): value = value.split(",") - self.licenses.extend(value) - self.licenses = [item.strip() for item in self.licenses] - self.licenses = list(set(self.licenses)) + self._licenses.extend(value) + self._licenses = [item.strip() for item in self._licenses] + self._licenses = list(set(self._licenses)) + + @property + def files(self): + return self._files - def set_files(self, value): + @files.setter + def files(self, value): if isinstance(value, list): - self.files.extend(value) + self._files.extend(value) else: - self.files.append(value) - self.files = list(set(self.files)) + self._files.append(value) + self._files = list(set(self._files)) - def set_source(self, value): - self.source = value + @property + def source_name_or_path(self): + return self._source_name_or_path - # new keys for updated yaml format - # replace 'file' key - def set_source_name_or_path(self, value): + @source_name_or_path.setter + def source_name_or_path(self, value): if not isinstance(value, list): value = value.split(",") - self.source_name_or_path.extend(value) - self.source_name_or_path = [item.strip() for item in self.source_name_or_path] - self.source_name_or_path = list(set(self.source_name_or_path)) - - # replace 'source' key - def set_download_location(self, value): - self.download_location = value - - # replace 'copyright' key - def set_copyright_text(self, value): - self.copyright_text = value + self._source_name_or_path.extend(value) + self._source_name_or_path = [item.strip() for item in self._source_name_or_path] + self._source_name_or_path = list(set(self._source_name_or_path)) def set_sheet_item(self, item): - switcher = { - 0: self.set_source_name_or_path, - 1: self.set_name, - 2: self.set_version, - 3: self.set_licenses, - 4: self.set_download_location, - 5: self.set_homepage, - 6: self.set_copyright_text, - 7: self.set_exclude, - 8: self.set_comment - } - - for i in range(0, len(switcher)): - func = switcher.get(i, "invalid") - func(item[i]) + if len(item) < 9: + _logger.warning(f"sheet list is too short ({len(item)}): {item}") + return + self.source_name_or_path = item[0] + self.name = item[1] + self.version = item[2] + self.licenses = item[3] + self.download_location = item[4] + self.homepage = item[5] + self.copyright_text = item[6] + self.exclude = item[7] + self.comment = item[8] def get_print_array(self): items = [] @@ -129,9 +140,7 @@ def get_print_array(self): if len(self.licenses) == 0: self.licenses.append("") - exclude = "" - if self.exclude: - exclude = "Exclude" + exclude = "Exclude" if self.exclude else "" for file in self.files: lic = ",".join(self.licenses) diff --git a/src/fosslight_util/write_opossum.py b/src/fosslight_util/write_opossum.py index 63eb004..7bf9ede 100644 --- a/src/fosslight_util/write_opossum.py +++ b/src/fosslight_util/write_opossum.py @@ -16,13 +16,8 @@ import fosslight_util.constant as constant from fosslight_util.write_excel import remove_empty_sheet -FL_SOURCE = 'FL_Source' -FL_DEPENDENCY = 'FL_Dependency' -FL_BINARY = 'FL_Binary' -supported_sheet_and_scanner = {'SRC': FL_SOURCE, 'BIN': FL_BINARY} -supported_sheet_name = ['SRC_' + FL_SOURCE, 'SRC_' + FL_DEPENDENCY, 'BIN_' + FL_BINARY] -PACKAE = { +PACKAGE = { 'requirements.txt': 'pypi', 'package.json': 'npm', 'pom.xml': 'maven', @@ -30,7 +25,8 @@ 'pubspec.yaml': 'pub', 'Podfile.lock': 'cocoapods', 'Package.resolved': 'swift', - 'Cartfile.resolved': 'carthage' + 'Cartfile.resolved': 'carthage', + 'go.mod': 'Go' } _attributionConfidence = 80 @@ -55,7 +51,7 @@ def __init__(self, self.excludeFromNotice = False self.source_name = source_name - if source_name == FL_DEPENDENCY: + if source_name == constant.FL_DEPENDENCY: self.preSelected = True else: self.preSelected = False @@ -117,12 +113,12 @@ def get_externalAttribution_dict(self): dict[licenseName] = self.licenseName dict[preSelected] = self.preSelected - if self.source_name == FL_SOURCE or FL_BINARY: + if self.source_name == constant.FL_SOURCE or constant.FL_BINARY: dict[copyright] = self.copyright dict[packageName] = self.packageName dict[packageVersion] = self.packageVersion dict[url] = self.url - elif self.source_name == FL_DEPENDENCY: + elif self.source_name == constant.FL_DEPENDENCY: dict[copyright] = self.copyright dict[packageName] = self.packageName dict[packageVersion] = self.packageVersion @@ -196,10 +192,8 @@ def write_opossum(filename, sheet_list): attributionBreakpoints_list = [] try: for sheet_name, sheet_contents in sheet_list.items(): - if sheet_name in supported_sheet_and_scanner: - scanner = supported_sheet_and_scanner.get(sheet_name) - elif sheet_name in supported_sheet_name: - scanner = '_'.join(sheet_name.split('_')[1:]) + if sheet_name in constant.supported_sheet_and_scanner.keys(): + scanner = constant.supported_sheet_and_scanner.get(sheet_name) else: logger.warning("Not supported scanner(sheet_name):" + sheet_name) continue @@ -263,15 +257,15 @@ def make_resources_and_attributions(sheet_items, scanner, resources, fc_list): items = items[0:9] path, oss_name, oss_version, license, url, homepage, copyright, exclude, comment = items - if scanner == FL_SOURCE: + if scanner == constant.FL_SOURCE: if (os.path.join(os.sep, path) + os.sep) not in fc_list: resources = make_resources(path, resources) attribution = Attribution(scanner, license, exclude, copyright, oss_name, oss_version, url) - elif scanner == FL_BINARY: + elif scanner == constant.FL_BINARY: resources = make_resources(path, resources) attribution = Attribution(scanner, license, exclude, copyright, oss_name, oss_version, url) - elif scanner == FL_DEPENDENCY: - packageType = PACKAE[path] + elif scanner == constant.FL_DEPENDENCY: + packageType = PACKAGE[path] if (os.path.join(os.sep, path) + os.sep) not in fc_list: fc_list.append(os.path.join(os.sep, path) + os.sep) ab_list.append(os.path.join(os.sep, path, packageType) + os.sep) diff --git a/src/fosslight_util/write_yaml.py b/src/fosslight_util/write_yaml.py index 5f73f7c..c77e66c 100644 --- a/src/fosslight_util/write_yaml.py +++ b/src/fosslight_util/write_yaml.py @@ -8,11 +8,11 @@ import os import copy from pathlib import Path -from .constant import LOGGER_NAME -from .oss_item import OssItem +import fosslight_util.constant as constant +from fosslight_util.oss_item import OssItem from fosslight_util.write_excel import remove_empty_sheet, _EMPTY_ITEM_MSG -_logger = logging.getLogger(LOGGER_NAME) +_logger = logging.getLogger(constant.LOGGER_NAME) def write_yaml(output_file, sheet_list_origin, separate_yaml=False): @@ -34,31 +34,19 @@ def write_yaml(output_file, sheet_list_origin, separate_yaml=False): merge_sheet = [] for sheet_name, sheet_contents in sheet_list.items(): + if sheet_name not in constant.supported_sheet_and_scanner.keys(): + continue if not separate_yaml: merge_sheet.extend(sheet_contents) - if sheet_name == list(sheet_list.keys())[-1]: - sheet_contents = merge_sheet - else: - continue else: output_file = f'{separate_output_file}_{sheet_name}.yaml' + convert_sheet_to_yaml(sheet_contents, output_file) + output_files.append(output_file) - sheet_contents = [list(t) for t in set(tuple(e) for e in sheet_contents)] - - yaml_dict = {} - for sheet_item in sheet_contents: - item = OssItem('') - item.set_sheet_item(sheet_item) - item_json = copy.deepcopy(item.get_print_json()) - item_name = item_json.pop("name") - if item_name not in yaml_dict.keys(): - yaml_dict[item_name] = [] - yaml_dict[item_name].append(item_json) - - with open(output_file, 'w') as f: - yaml.dump(yaml_dict, f, default_flow_style=False, sort_keys=False) - + if not separate_yaml: + convert_sheet_to_yaml(merge_sheet, output_file) output_files.append(output_file) + if output_files: output = ", ".join(output_files) else: @@ -72,3 +60,20 @@ def write_yaml(output_file, sheet_list_origin, separate_yaml=False): error_msg = "[Error] Writing yaml:" + error_msg return success, error_msg, output + + +def convert_sheet_to_yaml(sheet_contents, output_file): + sheet_contents = [list(t) for t in set(tuple(e) for e in sheet_contents)] + + yaml_dict = {} + for sheet_item in sheet_contents: + item = OssItem('') + item.set_sheet_item(sheet_item) + item_json = copy.deepcopy(item.get_print_json()) + item_name = item_json.pop("name") + if item_name not in yaml_dict.keys(): + yaml_dict[item_name] = [] + yaml_dict[item_name].append(item_json) + + with open(output_file, 'w') as f: + yaml.dump(yaml_dict, f, default_flow_style=False, sort_keys=False) diff --git a/tests/test_yaml.py b/tests/test_yaml.py index 6ada042..17eb09b 100644 --- a/tests/test_yaml.py +++ b/tests/test_yaml.py @@ -7,7 +7,7 @@ def main(): - logger, _result_log = init_log("test_result/excel/log_write_yaml.txt") + logger, _result_log = init_log("test_result/yaml/log_write_yaml.txt") logger.warning("TESTING - Writing a yaml") sheet_list = {'SRC_FL_Source': [ From d79e4fbd16ef03a4616101ae673aec9a87f64318 Mon Sep 17 00:00:00 2001 From: Jiyeong Seok Date: Wed, 18 May 2022 10:53:52 +0900 Subject: [PATCH 4/7] Delete the duplicated files, sources Signed-off-by: Jiyeong Seok --- src/fosslight_util/oss_item.py | 55 +++++++--------------------------- 1 file changed, 11 insertions(+), 44 deletions(-) diff --git a/src/fosslight_util/oss_item.py b/src/fosslight_util/oss_item.py index 77906ba..22b55be 100644 --- a/src/fosslight_util/oss_item.py +++ b/src/fosslight_util/oss_item.py @@ -14,8 +14,6 @@ def __init__(self, value): self._name = "-" self._version = "" self._licenses = [] - self.source = "" - self._files = [] self._copyright = "" self.comment = "" self._exclude = False @@ -23,7 +21,6 @@ def __init__(self, value): self.relative_path = value self._source_name_or_path = [] self.download_location = "" - self._copyright_text = "" def __del__(self): pass @@ -40,18 +37,6 @@ def copyright(self, value): value = value.strip() self._copyright = value - @property - def copyright_text(self): - return self._copyright_text - - @copyright_text.setter - def copyright_text(self, value): - if value != "": - if isinstance(value, list): - value = "\n".join(value) - value = value.strip() - self._copyright_text = value - @property def exclude(self): return self._exclude @@ -95,28 +80,16 @@ def licenses(self, value): self._licenses = [item.strip() for item in self._licenses] self._licenses = list(set(self._licenses)) - @property - def files(self): - return self._files - - @files.setter - def files(self, value): - if isinstance(value, list): - self._files.extend(value) - else: - self._files.append(value) - self._files = list(set(self._files)) - @property def source_name_or_path(self): return self._source_name_or_path @source_name_or_path.setter def source_name_or_path(self, value): - if not isinstance(value, list): - value = value.split(",") - self._source_name_or_path.extend(value) - self._source_name_or_path = [item.strip() for item in self._source_name_or_path] + if isinstance(value, list): + self._source_name_or_path.extend(value) + else: + self._source_name_or_path.append(value) self._source_name_or_path = list(set(self._source_name_or_path)) def set_sheet_item(self, item): @@ -129,25 +102,25 @@ def set_sheet_item(self, item): self.licenses = item[3] self.download_location = item[4] self.homepage = item[5] - self.copyright_text = item[6] + self.copyright = item[6] self.exclude = item[7] self.comment = item[8] def get_print_array(self): items = [] - if len(self.files) == 0: - self.files.append("") + if len(self.source_name_or_path) == 0: + self.source_name_or_path.append("") if len(self.licenses) == 0: self.licenses.append("") exclude = "Exclude" if self.exclude else "" - for file in self.files: + for source_name_or_path in self.source_name_or_path: lic = ",".join(self.licenses) if self.relative_path != "" and not str(self.relative_path).endswith("/"): self.relative_path += "/" - items.append([self.relative_path + file, self.name, self.version, lic, self.source, self.homepage, - self.copyright, "", exclude, self.comment]) + items.append([self.relative_path + source_name_or_path, self.name, self.version, lic, + self.download_location, self.homepage, self.copyright, "", exclude, self.comment]) return items def get_print_json(self): @@ -155,22 +128,16 @@ def get_print_json(self): json_item["name"] = self.name json_item["version"] = self.version - if len(self.files) > 0: - json_item["file"] = self.files if len(self.source_name_or_path) > 0: json_item["source_name_or_path"] = self.source_name_or_path if len(self.licenses) > 0: json_item["license"] = self.licenses - if self.source != "": - json_item["source"] = self.source if self.download_location != "": json_item["download_location"] = self.download_location if self.homepage != "": json_item["homepage"] = self.homepage if self.copyright != "": - json_item["copyright"] = self.copyright - if self.copyright_text != "": - json_item["copyright_text"] = self.copyright_text + json_item["copyright_text"] = self.copyright if self.exclude: json_item["exclude"] = self.exclude if self.comment != "": From a0d81abcc0a622ee6941165e8f702466a1a4a1ec Mon Sep 17 00:00:00 2001 From: Jiyeong Seok Date: Thu, 19 May 2022 10:17:02 +0900 Subject: [PATCH 5/7] Fix the key string and merge duplicated sheet Signed-off-by: Jiyeong Seok --- src/fosslight_util/oss_item.py | 23 +++++++++++------------ src/fosslight_util/write_yaml.py | 19 ++++++++++++++++++- tests/test_yaml.py | 2 ++ 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/src/fosslight_util/oss_item.py b/src/fosslight_util/oss_item.py index 22b55be..858c410 100644 --- a/src/fosslight_util/oss_item.py +++ b/src/fosslight_util/oss_item.py @@ -19,7 +19,7 @@ def __init__(self, value): self._exclude = False self.homepage = "" self.relative_path = value - self._source_name_or_path = [] + self._source_name_or_path = "" self.download_location = "" def __del__(self): @@ -87,10 +87,10 @@ def source_name_or_path(self): @source_name_or_path.setter def source_name_or_path(self, value): if isinstance(value, list): - self._source_name_or_path.extend(value) + self._source_name_or_path = list(set(self._source_name_or_path)) + self._source_name_or_path = ", ".join(self._source_name_or_path) else: - self._source_name_or_path.append(value) - self._source_name_or_path = list(set(self._source_name_or_path)) + self._source_name_or_path = value def set_sheet_item(self, item): if len(item) < 9: @@ -108,19 +108,18 @@ def set_sheet_item(self, item): def get_print_array(self): items = [] - if len(self.source_name_or_path) == 0: - self.source_name_or_path.append("") + if len(self.licenses) == 0: self.licenses.append("") exclude = "Exclude" if self.exclude else "" - for source_name_or_path in self.source_name_or_path: + for source_name_or_path in self.source_name_or_path.split(","): lic = ",".join(self.licenses) if self.relative_path != "" and not str(self.relative_path).endswith("/"): self.relative_path += "/" items.append([self.relative_path + source_name_or_path, self.name, self.version, lic, - self.download_location, self.homepage, self.copyright, "", exclude, self.comment]) + self.download_location, self.homepage, self.copyright, exclude, self.comment]) return items def get_print_json(self): @@ -128,16 +127,16 @@ def get_print_json(self): json_item["name"] = self.name json_item["version"] = self.version - if len(self.source_name_or_path) > 0: - json_item["source_name_or_path"] = self.source_name_or_path + if self.source_name_or_path != "": + json_item["source name or path"] = self.source_name_or_path if len(self.licenses) > 0: json_item["license"] = self.licenses if self.download_location != "": - json_item["download_location"] = self.download_location + json_item["download location"] = self.download_location if self.homepage != "": json_item["homepage"] = self.homepage if self.copyright != "": - json_item["copyright_text"] = self.copyright + json_item["copyright text"] = self.copyright if self.exclude: json_item["exclude"] = self.exclude if self.comment != "": diff --git a/src/fosslight_util/write_yaml.py b/src/fosslight_util/write_yaml.py index c77e66c..b6253f7 100644 --- a/src/fosslight_util/write_yaml.py +++ b/src/fosslight_util/write_yaml.py @@ -63,7 +63,24 @@ def write_yaml(output_file, sheet_list_origin, separate_yaml=False): def convert_sheet_to_yaml(sheet_contents, output_file): - sheet_contents = [list(t) for t in set(tuple(e) for e in sheet_contents)] + sheet_contents = [list(t) for t in set(tuple(e) for e in sorted(sheet_contents))] + try: + find_val = [list(t) for t in set(tuple(e[1:8]) for e in sheet_contents)] + find_idx = [] + for find_val_i in find_val: + idx = 0 + find = False + for e in sheet_contents: + if find_val_i == e[1:8]: + if find: + find_idx.append(idx) + find = True + idx += 1 + + for index in sorted(find_idx, reverse=True): + del sheet_contents[index] + except Exception as e: + _logger.debug(f"Fail to merge duplicated sheet: {e}") yaml_dict = {} for sheet_item in sheet_contents: diff --git a/tests/test_yaml.py b/tests/test_yaml.py index 17eb09b..372922f 100644 --- a/tests/test_yaml.py +++ b/tests/test_yaml.py @@ -13,6 +13,8 @@ def main(): sheet_list = {'SRC_FL_Source': [ ['test/lib/babel-polyfill.js', '', '', 'bsd-3-clause,facebook-patent-rights-2', '', '', 'Copyright (c) 2014, Facebook, Inc.', 'Exclude', ''], + ['test/lib/babel-polyfill2.js', '', '', 'bsd-3-clause,facebook-patent-rights-2', '', '', + 'Copyright (c) 2014, Facebook, Inc.', 'Exclude', ''], ['test/lib/babel-polyfill.js', '', '', 'bsd-3-clause,facebook-patent-rights-2', '', '', 'Copyright (c) 2014, Facebook, Inc.', 'Exclude', ''], ['lib/babel-polyfill.js', '', '', 'bsd-3-clause', '', '', From 523b1a92f91e58016af3e1c7dbaee60f036fd8b2 Mon Sep 17 00:00:00 2001 From: Jiyeong Seok Date: Thu, 19 May 2022 18:12:58 +0900 Subject: [PATCH 6/7] Fix the merge routine Signed-off-by: Jiyeong Seok --- src/fosslight_util/oss_item.py | 45 +++++++++++++++--------------- src/fosslight_util/write_yaml.py | 47 +++++++++++++++++--------------- tests/test_yaml.py | 2 ++ 3 files changed, 50 insertions(+), 44 deletions(-) diff --git a/src/fosslight_util/oss_item.py b/src/fosslight_util/oss_item.py index 858c410..f4006c6 100644 --- a/src/fosslight_util/oss_item.py +++ b/src/fosslight_util/oss_item.py @@ -13,13 +13,13 @@ class OssItem: def __init__(self, value): self._name = "-" self._version = "" - self._licenses = [] + self._license = [] self._copyright = "" self.comment = "" self._exclude = False self.homepage = "" self.relative_path = value - self._source_name_or_path = "" + self._source_name_or_path = [] self.download_location = "" def __del__(self): @@ -69,16 +69,16 @@ def version(self, value): self._version = "" @property - def licenses(self): - return self._licenses + def license(self): + return self._license - @licenses.setter - def licenses(self, value): + @license.setter + def license(self, value): if not isinstance(value, list): value = value.split(",") - self._licenses.extend(value) - self._licenses = [item.strip() for item in self._licenses] - self._licenses = list(set(self._licenses)) + self._license.extend(value) + self._license = [item.strip() for item in self._license] + self._license = list(set(self._license)) @property def source_name_or_path(self): @@ -86,11 +86,11 @@ def source_name_or_path(self): @source_name_or_path.setter def source_name_or_path(self, value): - if isinstance(value, list): - self._source_name_or_path = list(set(self._source_name_or_path)) - self._source_name_or_path = ", ".join(self._source_name_or_path) - else: - self._source_name_or_path = value + if not isinstance(value, list): + value = value.split(",") + self._source_name_or_path.extend(value) + self._source_name_or_path = [item.strip() for item in self._source_name_or_path] + self._source_name_or_path = list(set(self._source_name_or_path)) def set_sheet_item(self, item): if len(item) < 9: @@ -99,7 +99,7 @@ def set_sheet_item(self, item): self.source_name_or_path = item[0] self.name = item[1] self.version = item[2] - self.licenses = item[3] + self.license = item[3] self.download_location = item[4] self.homepage = item[5] self.copyright = item[6] @@ -108,14 +108,15 @@ def set_sheet_item(self, item): def get_print_array(self): items = [] - - if len(self.licenses) == 0: - self.licenses.append("") + if len(self.source_name_or_path) == 0: + self.source_name_or_path.append("") + if len(self.license) == 0: + self.license.append("") exclude = "Exclude" if self.exclude else "" for source_name_or_path in self.source_name_or_path.split(","): - lic = ",".join(self.licenses) + lic = ",".join(self.license) if self.relative_path != "" and not str(self.relative_path).endswith("/"): self.relative_path += "/" items.append([self.relative_path + source_name_or_path, self.name, self.version, lic, @@ -127,10 +128,10 @@ def get_print_json(self): json_item["name"] = self.name json_item["version"] = self.version - if self.source_name_or_path != "": + if len(self.source_name_or_path) > 0: json_item["source name or path"] = self.source_name_or_path - if len(self.licenses) > 0: - json_item["license"] = self.licenses + if len(self.license) > 0: + json_item["license"] = self.license if self.download_location != "": json_item["download location"] = self.download_location if self.homepage != "": diff --git a/src/fosslight_util/write_yaml.py b/src/fosslight_util/write_yaml.py index b6253f7..055d76a 100644 --- a/src/fosslight_util/write_yaml.py +++ b/src/fosslight_util/write_yaml.py @@ -64,33 +64,36 @@ def write_yaml(output_file, sheet_list_origin, separate_yaml=False): def convert_sheet_to_yaml(sheet_contents, output_file): sheet_contents = [list(t) for t in set(tuple(e) for e in sorted(sheet_contents))] - try: - find_val = [list(t) for t in set(tuple(e[1:8]) for e in sheet_contents)] - find_idx = [] - for find_val_i in find_val: - idx = 0 - find = False - for e in sheet_contents: - if find_val_i == e[1:8]: - if find: - find_idx.append(idx) - find = True - idx += 1 - - for index in sorted(find_idx, reverse=True): - del sheet_contents[index] - except Exception as e: - _logger.debug(f"Fail to merge duplicated sheet: {e}") yaml_dict = {} for sheet_item in sheet_contents: item = OssItem('') item.set_sheet_item(sheet_item) - item_json = copy.deepcopy(item.get_print_json()) - item_name = item_json.pop("name") - if item_name not in yaml_dict.keys(): - yaml_dict[item_name] = [] - yaml_dict[item_name].append(item_json) + create_yaml_with_ossitem(item, yaml_dict) with open(output_file, 'w') as f: yaml.dump(yaml_dict, f, default_flow_style=False, sort_keys=False) + + +def create_yaml_with_ossitem(item, yaml_dict): + item_json = item.get_print_json() + + item_name = item_json.pop("name") + if item_name not in yaml_dict.keys(): + yaml_dict[item_name] = [] + merged = False + for oss_info in yaml_dict[item_name]: + if oss_info.get('version', '') == item.version and \ + oss_info.get('license', []) == item.license and \ + oss_info.get('copyright text', '') == item.copyright and \ + oss_info.get('homepage', '') == item.homepage and \ + oss_info.get('download location', '') == item.download_location and \ + oss_info.get('exclude', False) == item.exclude: + oss_info.get('source name or path', []).extend(item.source_name_or_path) + merged = True + break + + if not merged: + yaml_dict[item_name].append(item_json) + + return yaml_dict diff --git a/tests/test_yaml.py b/tests/test_yaml.py index 372922f..7602913 100644 --- a/tests/test_yaml.py +++ b/tests/test_yaml.py @@ -11,6 +11,8 @@ def main(): logger.warning("TESTING - Writing a yaml") sheet_list = {'SRC_FL_Source': [ + ['test/lib/not_license.js', '', '', '', '', '', + 'Copyright (c) 2014, Facebook, Inc.', 'Exclude', ''], ['test/lib/babel-polyfill.js', '', '', 'bsd-3-clause,facebook-patent-rights-2', '', '', 'Copyright (c) 2014, Facebook, Inc.', 'Exclude', ''], ['test/lib/babel-polyfill2.js', '', '', 'bsd-3-clause,facebook-patent-rights-2', '', '', From ad545d4f8fae1a396ecc341fd808cec3a7bfd975 Mon Sep 17 00:00:00 2001 From: Jiyeong Seok Date: Fri, 20 May 2022 09:54:10 +0900 Subject: [PATCH 7/7] Remove comment if duplicated item Signed-off-by: Jiyeong Seok --- src/fosslight_util/write_yaml.py | 1 + tests/test_yaml.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/fosslight_util/write_yaml.py b/src/fosslight_util/write_yaml.py index 055d76a..328015b 100644 --- a/src/fosslight_util/write_yaml.py +++ b/src/fosslight_util/write_yaml.py @@ -90,6 +90,7 @@ def create_yaml_with_ossitem(item, yaml_dict): oss_info.get('download location', '') == item.download_location and \ oss_info.get('exclude', False) == item.exclude: oss_info.get('source name or path', []).extend(item.source_name_or_path) + oss_info.pop('comment', None) merged = True break diff --git a/tests/test_yaml.py b/tests/test_yaml.py index 7602913..b173055 100644 --- a/tests/test_yaml.py +++ b/tests/test_yaml.py @@ -16,7 +16,7 @@ def main(): ['test/lib/babel-polyfill.js', '', '', 'bsd-3-clause,facebook-patent-rights-2', '', '', 'Copyright (c) 2014, Facebook, Inc.', 'Exclude', ''], ['test/lib/babel-polyfill2.js', '', '', 'bsd-3-clause,facebook-patent-rights-2', '', '', - 'Copyright (c) 2014, Facebook, Inc.', 'Exclude', ''], + 'Copyright (c) 2014, Facebook, Inc.', 'Exclude', 'test_commend'], ['test/lib/babel-polyfill.js', '', '', 'bsd-3-clause,facebook-patent-rights-2', '', '', 'Copyright (c) 2014, Facebook, Inc.', 'Exclude', ''], ['lib/babel-polyfill.js', '', '', 'bsd-3-clause', '', '',