diff --git a/src/fosslight_util/constant.py b/src/fosslight_util/constant.py index 0fa530a..f0796b5 100755 --- a/src/fosslight_util/constant.py +++ b/src/fosslight_util/constant.py @@ -4,3 +4,12 @@ # SPDX-License-Identifier: Apache-2.0 LOGGER_NAME = "FOSSLight" + +FL_SOURCE = 'FL_Source' +FL_DEPENDENCY = 'FL_Dependency' +FL_BINARY = 'FL_Binary' +supported_sheet_and_scanner = {'SRC': FL_SOURCE, + 'BIN': FL_BINARY, + f'SRC_{FL_SOURCE}': FL_SOURCE, + f'SRC_{FL_DEPENDENCY}': FL_DEPENDENCY, + f'BIN_{FL_BINARY}': FL_BINARY} diff --git a/src/fosslight_util/oss_item.py b/src/fosslight_util/oss_item.py index 2ffb968..f4006c6 100644 --- a/src/fosslight_util/oss_item.py +++ b/src/fosslight_util/oss_item.py @@ -10,109 +10,138 @@ class OssItem: - name = "-" - version = "" - licenses = [] - source = "" - files = [] - copyright = "" - comment = "" - exclude = "" - homepage = "" - relative_path = "" - def __init__(self, value): - self.name = "-" - self.version = "" - self.licenses = [] - self.source = "" - self.files = [] - self.copyright = "" + self._name = "-" + self._version = "" + self._license = [] + self._copyright = "" self.comment = "" - self.exclude = "" + self._exclude = False self.homepage = "" self.relative_path = value + self._source_name_or_path = [] + self.download_location = "" def __del__(self): pass - def set_homepage(self, value): - self.homepage = value - - def set_comment(self, value): - self.comment = value + @property + def copyright(self): + return self._copyright - def set_copyright(self, value): + @copyright.setter + def copyright(self, value): if value != "": if isinstance(value, list): value = "\n".join(value) value = value.strip() - self.copyright = value + self._copyright = value + + @property + def exclude(self): + return self._exclude - def set_exclude(self, value): + @exclude.setter + def exclude(self, value): if value: - self.exclude = "Exclude" + self._exclude = True else: - self.exclude = "" + self._exclude = False + + @property + def name(self): + return self._name + + @name.setter + def name(self, value): + if value != "": + self._name = value - def set_name(self, value): - self.name = value + @property + def version(self): + return self._version - def set_version(self, value): - self.version = str(value) + @version.setter + def version(self, value): + if value: + self._version = str(value) + else: + self._version = "" - def set_licenses(self, value): + @property + def license(self): + return self._license + + @license.setter + def license(self, value): if not isinstance(value, list): value = value.split(",") - self.licenses.extend(value) - self.licenses = [item.strip() for item in self.licenses] - self.licenses = list(set(self.licenses)) + self._license.extend(value) + self._license = [item.strip() for item in self._license] + self._license = list(set(self._license)) - def set_files(self, value): - if isinstance(value, list): - self.files.extend(value) - else: - self.files.append(value) - self.files = list(set(self.files)) + @property + def source_name_or_path(self): + return self._source_name_or_path - def set_source(self, value): - self.source = value + @source_name_or_path.setter + def source_name_or_path(self, value): + if not isinstance(value, list): + value = value.split(",") + self._source_name_or_path.extend(value) + self._source_name_or_path = [item.strip() for item in self._source_name_or_path] + self._source_name_or_path = list(set(self._source_name_or_path)) + + def set_sheet_item(self, item): + if len(item) < 9: + _logger.warning(f"sheet list is too short ({len(item)}): {item}") + return + self.source_name_or_path = item[0] + self.name = item[1] + self.version = item[2] + self.license = item[3] + self.download_location = item[4] + self.homepage = item[5] + self.copyright = item[6] + self.exclude = item[7] + self.comment = item[8] def get_print_array(self): items = [] - if len(self.files) == 0: - self.files.append("") - if len(self.licenses) == 0: - self.licenses.append("") - for file in self.files: - lic = ",".join(self.licenses) + if len(self.source_name_or_path) == 0: + self.source_name_or_path.append("") + if len(self.license) == 0: + self.license.append("") + + exclude = "Exclude" if self.exclude else "" + + for source_name_or_path in self.source_name_or_path.split(","): + lic = ",".join(self.license) if self.relative_path != "" and not str(self.relative_path).endswith("/"): self.relative_path += "/" - items.append([self.relative_path + file, self.name, self.version, lic, self.source, self.homepage, - self.copyright, "", self.exclude, self.comment]) + items.append([self.relative_path + source_name_or_path, self.name, self.version, lic, + self.download_location, self.homepage, self.copyright, exclude, self.comment]) return items def get_print_json(self): json_item = {} json_item["name"] = self.name - if self.version != "": - json_item["version"] = self.version - if self.source != "": - json_item["source"] = self.source + json_item["version"] = self.version + if len(self.source_name_or_path) > 0: + json_item["source name or path"] = self.source_name_or_path + if len(self.license) > 0: + json_item["license"] = self.license + if self.download_location != "": + json_item["download location"] = self.download_location + if self.homepage != "": + json_item["homepage"] = self.homepage if self.copyright != "": - json_item["copyright"] = self.copyright - if self.exclude != "": + json_item["copyright text"] = self.copyright + if self.exclude: json_item["exclude"] = self.exclude if self.comment != "": json_item["comment"] = self.comment - if self.homepage != "": - json_item["homepage"] = self.homepage - - if len(self.files) > 0: - json_item["file"] = self.files - if len(self.licenses) > 0: - json_item["license"] = self.licenses return json_item diff --git a/src/fosslight_util/output_format.py b/src/fosslight_util/output_format.py index d30f965..d780de1 100644 --- a/src/fosslight_util/output_format.py +++ b/src/fosslight_util/output_format.py @@ -5,8 +5,9 @@ import os from fosslight_util.write_excel import write_result_to_excel, write_excel_and_csv, write_result_to_csv from fosslight_util.write_opossum import write_opossum +from fosslight_util.write_yaml import write_yaml -SUPPORT_FORMAT = {'excel': '.xlsx', 'csv': '.csv', 'opossum': '.json'} +SUPPORT_FORMAT = {'excel': '.xlsx', 'csv': '.csv', 'opossum': '.json', 'yaml': '.yaml'} def check_output_format(output='', format='', customized_format={}): @@ -71,6 +72,8 @@ def write_output_file(output_file_without_ext, file_extension, sheet_list, exten success, msg, result_file = write_result_to_csv(result_file, sheet_list) elif file_extension == '.json': success, msg = write_opossum(result_file, sheet_list) + elif file_extension == '.yaml': + success, msg, result_file = write_yaml(result_file, sheet_list, False) else: success = False msg = f'Not supported file extension({file_extension})' diff --git a/src/fosslight_util/write_opossum.py b/src/fosslight_util/write_opossum.py index 63eb004..7bf9ede 100644 --- a/src/fosslight_util/write_opossum.py +++ b/src/fosslight_util/write_opossum.py @@ -16,13 +16,8 @@ import fosslight_util.constant as constant from fosslight_util.write_excel import remove_empty_sheet -FL_SOURCE = 'FL_Source' -FL_DEPENDENCY = 'FL_Dependency' -FL_BINARY = 'FL_Binary' -supported_sheet_and_scanner = {'SRC': FL_SOURCE, 'BIN': FL_BINARY} -supported_sheet_name = ['SRC_' + FL_SOURCE, 'SRC_' + FL_DEPENDENCY, 'BIN_' + FL_BINARY] -PACKAE = { +PACKAGE = { 'requirements.txt': 'pypi', 'package.json': 'npm', 'pom.xml': 'maven', @@ -30,7 +25,8 @@ 'pubspec.yaml': 'pub', 'Podfile.lock': 'cocoapods', 'Package.resolved': 'swift', - 'Cartfile.resolved': 'carthage' + 'Cartfile.resolved': 'carthage', + 'go.mod': 'Go' } _attributionConfidence = 80 @@ -55,7 +51,7 @@ def __init__(self, self.excludeFromNotice = False self.source_name = source_name - if source_name == FL_DEPENDENCY: + if source_name == constant.FL_DEPENDENCY: self.preSelected = True else: self.preSelected = False @@ -117,12 +113,12 @@ def get_externalAttribution_dict(self): dict[licenseName] = self.licenseName dict[preSelected] = self.preSelected - if self.source_name == FL_SOURCE or FL_BINARY: + if self.source_name == constant.FL_SOURCE or constant.FL_BINARY: dict[copyright] = self.copyright dict[packageName] = self.packageName dict[packageVersion] = self.packageVersion dict[url] = self.url - elif self.source_name == FL_DEPENDENCY: + elif self.source_name == constant.FL_DEPENDENCY: dict[copyright] = self.copyright dict[packageName] = self.packageName dict[packageVersion] = self.packageVersion @@ -196,10 +192,8 @@ def write_opossum(filename, sheet_list): attributionBreakpoints_list = [] try: for sheet_name, sheet_contents in sheet_list.items(): - if sheet_name in supported_sheet_and_scanner: - scanner = supported_sheet_and_scanner.get(sheet_name) - elif sheet_name in supported_sheet_name: - scanner = '_'.join(sheet_name.split('_')[1:]) + if sheet_name in constant.supported_sheet_and_scanner.keys(): + scanner = constant.supported_sheet_and_scanner.get(sheet_name) else: logger.warning("Not supported scanner(sheet_name):" + sheet_name) continue @@ -263,15 +257,15 @@ def make_resources_and_attributions(sheet_items, scanner, resources, fc_list): items = items[0:9] path, oss_name, oss_version, license, url, homepage, copyright, exclude, comment = items - if scanner == FL_SOURCE: + if scanner == constant.FL_SOURCE: if (os.path.join(os.sep, path) + os.sep) not in fc_list: resources = make_resources(path, resources) attribution = Attribution(scanner, license, exclude, copyright, oss_name, oss_version, url) - elif scanner == FL_BINARY: + elif scanner == constant.FL_BINARY: resources = make_resources(path, resources) attribution = Attribution(scanner, license, exclude, copyright, oss_name, oss_version, url) - elif scanner == FL_DEPENDENCY: - packageType = PACKAE[path] + elif scanner == constant.FL_DEPENDENCY: + packageType = PACKAGE[path] if (os.path.join(os.sep, path) + os.sep) not in fc_list: fc_list.append(os.path.join(os.sep, path) + os.sep) ab_list.append(os.path.join(os.sep, path, packageType) + os.sep) diff --git a/src/fosslight_util/write_yaml.py b/src/fosslight_util/write_yaml.py new file mode 100644 index 0000000..328015b --- /dev/null +++ b/src/fosslight_util/write_yaml.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2022 LG Electronics Inc. +# SPDX-License-Identifier: Apache-2.0 + +import yaml +import logging +import os +import copy +from pathlib import Path +import fosslight_util.constant as constant +from fosslight_util.oss_item import OssItem +from fosslight_util.write_excel import remove_empty_sheet, _EMPTY_ITEM_MSG + +_logger = logging.getLogger(constant.LOGGER_NAME) + + +def write_yaml(output_file, sheet_list_origin, separate_yaml=False): + success = True + error_msg = "" + output = "" + + try: + sheet_list = copy.deepcopy(sheet_list_origin) + is_not_null, sheet_list = remove_empty_sheet(sheet_list) + if is_not_null: + output_files = [] + output_dir = os.path.dirname(output_file) + + Path(output_dir).mkdir(parents=True, exist_ok=True) + if separate_yaml: + filename = os.path.splitext(os.path.basename(output_file))[0] + separate_output_file = os.path.join(output_dir, filename) + + merge_sheet = [] + for sheet_name, sheet_contents in sheet_list.items(): + if sheet_name not in constant.supported_sheet_and_scanner.keys(): + continue + if not separate_yaml: + merge_sheet.extend(sheet_contents) + else: + output_file = f'{separate_output_file}_{sheet_name}.yaml' + convert_sheet_to_yaml(sheet_contents, output_file) + output_files.append(output_file) + + if not separate_yaml: + convert_sheet_to_yaml(merge_sheet, output_file) + output_files.append(output_file) + + if output_files: + output = ", ".join(output_files) + else: + success = False + error_msg = _EMPTY_ITEM_MSG + except Exception as ex: + error_msg = str(ex) + success = False + + if not success: + error_msg = "[Error] Writing yaml:" + error_msg + + return success, error_msg, output + + +def convert_sheet_to_yaml(sheet_contents, output_file): + sheet_contents = [list(t) for t in set(tuple(e) for e in sorted(sheet_contents))] + + yaml_dict = {} + for sheet_item in sheet_contents: + item = OssItem('') + item.set_sheet_item(sheet_item) + create_yaml_with_ossitem(item, yaml_dict) + + with open(output_file, 'w') as f: + yaml.dump(yaml_dict, f, default_flow_style=False, sort_keys=False) + + +def create_yaml_with_ossitem(item, yaml_dict): + item_json = item.get_print_json() + + item_name = item_json.pop("name") + if item_name not in yaml_dict.keys(): + yaml_dict[item_name] = [] + merged = False + for oss_info in yaml_dict[item_name]: + if oss_info.get('version', '') == item.version and \ + oss_info.get('license', []) == item.license and \ + oss_info.get('copyright text', '') == item.copyright and \ + oss_info.get('homepage', '') == item.homepage and \ + oss_info.get('download location', '') == item.download_location and \ + oss_info.get('exclude', False) == item.exclude: + oss_info.get('source name or path', []).extend(item.source_name_or_path) + oss_info.pop('comment', None) + merged = True + break + + if not merged: + yaml_dict[item_name].append(item_json) + + return yaml_dict diff --git a/tests/test_yaml.py b/tests/test_yaml.py new file mode 100644 index 0000000..b173055 --- /dev/null +++ b/tests/test_yaml.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2022 LG Electronics Inc. +# SPDX-License-Identifier: Apache-2.0 +from fosslight_util.write_yaml import write_yaml +from fosslight_util.set_log import init_log + + +def main(): + logger, _result_log = init_log("test_result/yaml/log_write_yaml.txt") + logger.warning("TESTING - Writing a yaml") + + sheet_list = {'SRC_FL_Source': [ + ['test/lib/not_license.js', '', '', '', '', '', + 'Copyright (c) 2014, Facebook, Inc.', 'Exclude', ''], + ['test/lib/babel-polyfill.js', '', '', 'bsd-3-clause,facebook-patent-rights-2', '', '', + 'Copyright (c) 2014, Facebook, Inc.', 'Exclude', ''], + ['test/lib/babel-polyfill2.js', '', '', 'bsd-3-clause,facebook-patent-rights-2', '', '', + 'Copyright (c) 2014, Facebook, Inc.', 'Exclude', 'test_commend'], + ['test/lib/babel-polyfill.js', '', '', 'bsd-3-clause,facebook-patent-rights-2', '', '', + 'Copyright (c) 2014, Facebook, Inc.', 'Exclude', ''], + ['lib/babel-polyfill.js', '', '', 'bsd-3-clause', '', '', + 'Copyright (c) 2014, Facebook, Inc.', '', ''], + ['lib/babel-polyfill.js', '', '', 'facebook-patent-rights-2', '', '', + 'Copyright (c) 2014, Facebook, Inc.', '', ''], + ['requirements.txt', '', '', 'MIT', 'https://pypi.org/project/future/0.18.2', '', '', '', ''], + ['bower.json', '', '', 'mit', '', '', '', '', ''], + ['LICENSE', '', '', 'mit', '', '', 'Copyright (c) 2016-2021, The Cytoscape Consortium', '', ''], + ['license-update.js', '', '', 'mit', '', '', 'Copyright (c) 2016-$ year, The Cytoscape Consortium', '', ''], + ['package.json', '', '', 'mit', '', '', '', '', ''], ['README.md', '', '', 'mit', '', '', '', '', ''], + ['dist/cytoscape.cjs.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,Copyright (c) 2016-2021,c \ + The Cytoscape Consortium,copyright Koen Bok,Copyright (c) 2013-2014 Ralf S. Engelschall \ + (http://engelschall.com)', '', ''], + ['dist/cytoscape.esm.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,Copyright (c) 2016-2021,\ + The Cytoscape Consortium,copyright Koen Bok,Copyright (c) 2013-2014 Ralf S. Engelschall \ + (http://engelschall.com)', '', ''], + ['dist/cytoscape.esm.min.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,copyright Koen Bok, \ + Copyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''], + ['dist/cytoscape.min.js', '', '', 'mit', + '', '', 'Copyright Gaetan Renaudeau,Copyright (c) 2016-2021, The Cytoscape Consortium,copyright Koen Bok,Copyright \ + (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''], + ['dist/cytoscape.umd.js', '', '', 'mit', '', '', + 'Copyright Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors,Copyright jQuery Foundation \ + and other contributors ,Copyright (c) 2016-2021, The Cytoscape Consortium,copyright Koen\ + Bok,Copyright Gaetan Renaudeau,Copyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''], + ['documentation/css/highlight/monokai_sublime.css', '', '', 'mit', '', '', '', '', ''], + ['documentation/js/cytoscape.min.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,\ + Copyright (c) 2016-2021, The Cytoscape Consortium,copyright Koen Bok, \ + Copyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''], + ['documentation/md/links.md', '', '', 'mit', '', '', '', '', ''], + ['src/event.js', '', '', 'mit', '', '', '', '', '']], + 'BIN_FL_Binary': [ + ['askalono_macos', 'askalono', '', 'Apache-2.0', '', '', '', '', ''], + ['test/askalono_macos', 'askalono', '', 'Apache-2.0', '', '', '', 'Exclude', '']], + 'SRC_FL_Dependency': [ + ['requirements.txt', 'pypi:future', '0.18.2', 'MIT', 'https://pypi.org/project/future/0.18.2', + 'https://python-future.org', '', '', ''], + ['requirements.txt', 'pypi:numpy', '1.19.5', 'BSD-3-Clause-Open-MPI,GCC-exception-3.1,GPL-3.0', + 'https://pypi.org/project/numpy/1.19.5', 'https://www.numpy.org', '', '', ''], + ['requirements.txt', 'pypi:pandas', '1.1.5', 'BSD-3-Clause', 'https://pypi.org/project/pandas/1.1.5', + 'https://pandas.pydata.org', '', '', '']]} + + sheet_list2 = {'SRC_FL_Dependency': [ + ['requirements.txt', 'pypi:future', '0.18.2', 'MIT', 'https://pypi.org/project/future/0.18.2', + 'https://python-future.org', '', '', ''], + ['requirements.txt', 'pypi:numpy', '1.19.5', 'BSD-3-Clause-Open-MPI,GCC-exception-3.1,GPL-3.0', + 'https://pypi.org/project/numpy/1.19.5', 'https://www.numpy.org', '', '', ''], + ['requirements.txt', 'pypi:pandas', '1.1.5', 'BSD-3-Clause', 'https://pypi.org/project/pandas/1.1.5', + 'https://pandas.pydata.org', '', '', '']], + 'SRC_FL_Source': [ + ['test/lib/babel-polyfill.js', '', '', 'bsd-3-clause,facebook-patent-rights-2', '', '', + 'Copyright (c) 2014, Facebook, Inc.', 'Exclude', ''], + ['requirements.txt', '', '', 'MIT', 'https://pypi.org/project/future/0.18.2', '', '', '', ''], + ['bower.json', '', '', 'mit', '', '', '', '', ''], + ['LICENSE', '', '', 'mit', '', '', 'Copyright (c) 2016-2021, The Cytoscape Consortium', '', ''], + ['license-update.js', '', '', 'mit', '', '', 'Copyright (c) 2016-$ year, The Cytoscape Consortium', '', ''], + ['package.json', '', '', 'mit', '', '', '', '', ''], ['README.md', '', '', 'mit', '', '', '', '', ''], + ['dist/cytoscape.cjs.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,Copyright (c) 2016-2021,c \ + The Cytoscape Consortium,copyright Koen Bok,Copyright (c) 2013-2014 Ralf S. Engelschall \ + (http://engelschall.com)', '', ''], + ['dist/cytoscape.esm.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,Copyright (c) 2016-2021,\ + The Cytoscape Consortium,copyright Koen Bok,Copyright (c) 2013-2014 Ralf S. Engelschall \ + (http://engelschall.com)', '', ''], + ['dist/cytoscape.esm.min.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,copyright Koen Bok, \ + Copyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''], + ['dist/cytoscape.min.js', '', '', 'mit', + '', '', 'Copyright Gaetan Renaudeau,Copyright (c) 2016-2021, The Cytoscape Consortium,copyright Koen Bok,Copyright \ + (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''], + ['dist/cytoscape.umd.js', '', '', 'mit', '', '', + 'Copyright Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors,Copyright jQuery Foundation \ + and other contributors ,Copyright (c) 2016-2021, The Cytoscape Consortium,copyright Koen\ + Bok,Copyright Gaetan Renaudeau,Copyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''], + ['documentation/css/highlight/monokai_sublime.css', '', '', 'mit', '', '', '', '', ''], + ['documentation/js/cytoscape.min.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,\ + Copyright (c) 2016-2021, The Cytoscape Consortium,copyright Koen Bok, \ + Copyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''], + ['documentation/md/links.md', '', '', 'mit', '', '', '', '', ''], + ['src/event.js', '', '', 'mit', '', '', '', '', '']], + 'BIN_FL_Binary': [ + ['askalono_macos', 'askalono', '', 'Apache-2.0', '', '', '', '', ''], + ['test/askalono_macos', 'askalono', '', 'Apache-2.0', '', '', '', 'Exclude', '']]} + + success, msg, output = write_yaml( + 'test_result/yaml/FL-TEST_yaml.yaml', sheet_list) + logger.warning(f"Result: {str(success)}, error_msg: {msg}, Output_files: {output}") + + success, msg, output = write_yaml( + 'test_result/yaml/FL-TEST2_yaml.yaml', sheet_list2) + logger.warning(f"Result: {str(success)}, error_msg: {msg}, Output_files: {output}") + + +if __name__ == '__main__': + main() diff --git a/tox.ini b/tox.ini index b6a3ea6..a0357f1 100644 --- a/tox.ini +++ b/tox.ini @@ -78,6 +78,10 @@ commands = # Test - writing opossum python tests/test_opossum.py ls test_result/opossum + # Test - writing yaml + python tests/test_yaml.py + cat test_result/yaml/FL-TEST_yaml.yaml + cat test_result/yaml/FL-TEST2_yaml.yaml # Test - timer python tests/test_timer.py # Test - downloading source