diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 313993e..d576348 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -4,7 +4,8 @@ name: Pull requests fosslight_source_scanner on: pull_request: - branches: [ main ] + branches: + - '*' jobs: build: diff --git a/requirements-dev.txt b/requirements-dev.txt index 306ef44..921d5d5 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,3 +2,4 @@ tox pytest pytest-cov pytest-flake8 +flake8==3.9.2 \ No newline at end of file diff --git a/src/fosslight_source/_help.py b/src/fosslight_source/_help.py index fd43ab0..6e8d5c0 100644 --- a/src/fosslight_source/_help.py +++ b/src/fosslight_source/_help.py @@ -21,7 +21,9 @@ -h\t\t\t\t Print help message -j\t\t\t\t Generate additional result of executing ScanCode in json format -m\t\t\t\t Print the Matched text for each license on a separate sheet - -o \t\t Output file name""" + -o \t\t Output path + \t\t\t\t (If you want to generate the specific file name, add the output path with file name.) + -f \t\t\t Output file format (excel, csv, opossum)""" _HELP_MESSAGE_CONVERT = """ Usage: fosslight_convert [option1] [option2] ... @@ -35,7 +37,9 @@ Optional -h\t\t\t\t Print help message -m\t\t\t\t Print the Matched text for each license on a separate sheet - -o \t\t Output file name""" + -o \t\t Output path + \t\t\t\t (If you want to generate the specific file name, add the output path with file name.) + -f \t\t\t Output file format (excel, csv, opossum)""" def print_help_msg_source(): diff --git a/src/fosslight_source/convert_scancode.py b/src/fosslight_source/convert_scancode.py index da45380..92c5b2c 100755 --- a/src/fosslight_source/convert_scancode.py +++ b/src/fosslight_source/convert_scancode.py @@ -13,7 +13,7 @@ from fosslight_util.set_log import init_log import yaml from ._parsing_scancode_file_item import parsing_file_item, get_error_from_header -from fosslight_util.write_excel import write_excel_and_csv +from fosslight_util.output_format import check_output_format, write_output_file from ._help import print_help_msg_convert from ._license_matched import get_license_list_to_print @@ -21,13 +21,38 @@ _PKG_NAME = "fosslight_source" -def convert_json_to_excel(scancode_json, excel_name, result_log, need_license=False): +def convert_json_to_output_report(scancode_json, output_file_name, need_license=False, format=""): + global logger + sheet_license_prefix = "matched_text" - sheet_SRC_prefix = "SRC" + sheet_SRC_prefix = "SRC_FL_Source" file_list = [] lic_list = {} msg = "" success = True + start_time = datetime.now().strftime('%Y%m%d_%H%M%S') + _json_ext = ".json" + + success, msg, output_path, output_file, output_extension = check_output_format(output_file_name, format) + if success: + if output_path == "": + output_path = os.getcwd() + else: + output_path = os.path.abspath(output_path) + + if output_file == "": + if output_extension == _json_ext: + output_file = "Opossum_input_" + start_time + else: + output_file = "FOSSLight-Report_" + start_time + else: + output_path = os.getcwd() + + logger, result_log = init_log(os.path.join(output_path, "fosslight_src_log_" + start_time + ".txt"), + True, logging.INFO, logging.DEBUG, _PKG_NAME) + if not success: + logger.error("Fail to convert scancode: " + msg) + return [] try: sheet_list = {} @@ -60,10 +85,11 @@ def convert_json_to_excel(scancode_json, excel_name, result_log, need_license=Fa except Exception as ex: logger.warning("Error parsing "+file+":" + str(ex)) - success_to_write, writing_msg = write_excel_and_csv(excel_name, sheet_list) - logger.info("Writing excel :" + str(success_to_write) + " " + writing_msg) + output_file_without_ext = os.path.join(output_path, output_file) + success_to_write, writing_msg = write_output_file(output_file_without_ext, output_extension, sheet_list) + logger.info("Writing Output file(" + output_file + output_extension + "):" + str(success_to_write) + " " + writing_msg) if success_to_write: - result_log["FOSSLight Report"] = excel_name + ".xlsx" + result_log["Output file"] = output_file_without_ext + output_extension except Exception as ex: success = False @@ -100,16 +126,14 @@ def get_detected_licenses_from_scancode(scancode_json_file, need_license): def main(): - global logger - argv = sys.argv[1:] path_to_find_json = "" - start_time = datetime.now().strftime('%Y%m%d_%H%M%S') output_file_name = "" print_matched_text = False + format = "" try: - opts, args = getopt.getopt(argv, 'hmp:o:') + opts, args = getopt.getopt(argv, 'hmp:o:f:') for opt, arg in opts: if opt == "-h": print_help_msg_convert() @@ -119,22 +143,15 @@ def main(): output_file_name = arg elif opt == "-m": print_matched_text = True + elif opt == "-f": + format = arg except Exception: print_help_msg_convert() if path_to_find_json == "": print_help_msg_convert() - if output_file_name == "": - output_dir = os.getcwd() - oss_report_name = "FOSSLight-Report_" + start_time - else: - oss_report_name = output_file_name - output_dir = os.path.dirname(os.path.abspath(output_file_name)) - - logger, _result_log = init_log(os.path.join(output_dir, "fosslight_src_log_" + start_time + ".txt"), - True, logging.INFO, logging.DEBUG, _PKG_NAME) - convert_json_to_excel(path_to_find_json, oss_report_name, _result_log, print_matched_text) + convert_json_to_output_report(path_to_find_json, output_file_name, print_matched_text, format) if __name__ == '__main__': diff --git a/src/fosslight_source/run_scancode.py b/src/fosslight_source/run_scancode.py index cf7361c..82af0bb 100755 --- a/src/fosslight_source/run_scancode.py +++ b/src/fosslight_source/run_scancode.py @@ -18,9 +18,9 @@ from fosslight_util.timer_thread import TimerThread from ._parsing_scancode_file_item import parsing_file_item from ._parsing_scancode_file_item import get_error_from_header -from fosslight_util.write_excel import write_excel_and_csv from ._help import print_help_msg_source from ._license_matched import get_license_list_to_print +from fosslight_util.output_format import check_output_format, write_output_file logger = logging.getLogger(constant.LOGGER_NAME) warnings.filterwarnings("ignore", category=FutureWarning) @@ -33,9 +33,10 @@ def main(): write_json_file = False output_file = "" print_matched_text = False + format = "" try: - opts, args = getopt.getopt(argv, 'hmjp:o:') + opts, args = getopt.getopt(argv, 'hmjp:o:f:') for opt, arg in opts: if opt == "-h": print_help_msg_source() @@ -47,17 +48,20 @@ def main(): output_file = arg elif opt == "-m": print_matched_text = True + elif opt == "-f": + format = arg except Exception: print_help_msg_source() timer = TimerThread() timer.setDaemon(True) timer.start() - run_scan(path_to_scan, output_file, write_json_file, -1, False, print_matched_text) + + run_scan(path_to_scan, output_file, write_json_file, -1, False, print_matched_text, format) def run_scan(path_to_scan, output_file_name="", - _write_json_file=False, num_cores=-1, return_results=False, need_license=False): + _write_json_file=False, num_cores=-1, return_results=False, need_license=False, format=""): global logger success = True @@ -65,90 +69,103 @@ def run_scan(path_to_scan, output_file_name="", _str_final_result_log = "" _result_log = {} result_list = [] + _json_ext = ".json" _windows = platform.system() == "Windows" start_time = datetime.now().strftime('%Y%m%d_%H%M%S') - if output_file_name == "": - output_file = "FOSSLight-Report_" + start_time - output_json_file = "scancode_" + start_time - output_dir = os.getcwd() - else: - output_file = output_file_name - output_json_file = output_file_name - output_dir = os.path.dirname(os.path.abspath(output_file_name)) - - logger, _result_log = init_log(os.path.join(output_dir, "fosslight_src_log_"+start_time+".txt"), - True, logging.INFO, logging.DEBUG, _PKG_NAME, path_to_scan) - - if path_to_scan == "": - if _windows: - path_to_scan = os.getcwd() + success, msg, output_path, output_file, output_extension = check_output_format(output_file_name, format) + if success: + if output_path == "": + output_path = os.getcwd() else: - print_help_msg_source() + output_path = os.path.abspath(output_path) - num_cores = multiprocessing.cpu_count() - 1 if num_cores < 0 else num_cores + if output_file == "": + if output_extension == _json_ext: + output_file = "Opossum_input_" + start_time + else: + output_file = "FOSSLight-Report_" + start_time - if os.path.isdir(path_to_scan): - try: - output_json_file = output_json_file+".json" if _write_json_file\ - else "" + if _write_json_file: + output_json_file = os.path.join(output_path, "scancode_raw_result.json") + else: + output_json_file = "" - rc, results = cli.run_scan(path_to_scan, max_depth=100, - strip_root=True, license=True, - copyright=True, return_results=True, - processes=num_cores, - output_json_pp=output_json_file, - only_findings=True, license_text=True) + logger, _result_log = init_log(os.path.join(output_path, "fosslight_src_log_"+start_time+".txt"), + True, logging.INFO, logging.DEBUG, _PKG_NAME, path_to_scan) - if not rc: - msg = "Source code analysis failed." - success = False + if path_to_scan == "": + if _windows: + path_to_scan = os.getcwd() + else: + print_help_msg_source() - if results: - sheet_list = {} - has_error = False - if "headers" in results: - has_error, error_msg = get_error_from_header(results["headers"]) - if has_error: - _result_log["Error_files"] = error_msg - msg = "Failed to analyze :" + error_msg - if "files" in results: - rc, result_list, parsing_msg, license_list = parsing_file_item(results["files"], has_error, need_license) - _result_log["Parsing Log"] = parsing_msg - if rc: - if not success: - success = True - result_list = sorted( - result_list, key=lambda row: (''.join(row.licenses))) - sheet_list["SRC"] = [scan_item.get_row_to_print() for scan_item in result_list] - if need_license: - sheet_list["matched_text"] = get_license_list_to_print(license_list) - - success_to_write, writing_msg = write_excel_and_csv( - output_file, sheet_list) - logger.info("Writing excel :" + str(success_to_write) + " " + writing_msg) - if success_to_write: - _result_log["FOSSLight Report"] = output_file + ".xlsx" - except Exception as ex: + num_cores = multiprocessing.cpu_count() - 1 if num_cores < 0 else num_cores + + if os.path.isdir(path_to_scan): + try: + rc, results = cli.run_scan(path_to_scan, max_depth=100, + strip_root=True, license=True, + copyright=True, return_results=True, + processes=num_cores, + output_json_pp=output_json_file, + only_findings=True, license_text=True) + + if not rc: + msg = "Source code analysis failed." + success = False + + if results: + sheet_list = {} + has_error = False + if "headers" in results: + has_error, error_msg = get_error_from_header(results["headers"]) + if has_error: + _result_log["Error_files"] = error_msg + msg = "Failed to analyze :" + error_msg + if "files" in results: + rc, result_list, parsing_msg, license_list = parsing_file_item(results["files"], has_error, need_license) + _result_log["Parsing Log"] = parsing_msg + if rc: + if not success: + success = True + result_list = sorted( + result_list, key=lambda row: (''.join(row.licenses))) + sheet_list["SRC_FL_Source"] = [scan_item.get_row_to_print() for scan_item in result_list] + if need_license: + sheet_list["matched_text"] = get_license_list_to_print(license_list) + + output_file_without_ext = os.path.join(output_path, output_file) + success_to_write, writing_msg = write_output_file(output_file_without_ext, output_extension, + sheet_list) + logger.info("Writing Output file(" + output_file + output_extension + "):" + str(success_to_write) + + " " + writing_msg) + if success_to_write: + _result_log["Output file"] = output_file_without_ext + output_extension + + except Exception as ex: + success = False + msg = str(ex) + logger.error("Analyze " + path_to_scan + ":" + msg) + else: success = False - msg = str(ex) - logger.error("Analyze " + path_to_scan + ":" + msg) - else: - success = False - msg = "Check the path to scan. :" + path_to_scan + msg = "Check the path to scan. :" + path_to_scan - if not return_results: - result_list = [] + if not return_results: + result_list = [] - scan_result_msg = str(success) if msg == "" else str(success) + "," + msg + scan_result_msg = str(success) if msg == "" else str(success) + ", " + msg _result_log["Scan Result"] = scan_result_msg - _result_log["Output Directory"] = output_dir + _result_log["Output Directory"] = output_path try: _str_final_result_log = yaml.safe_dump(_result_log, allow_unicode=True, sort_keys=True) logger.info(_str_final_result_log) except Exception as ex: logger.warning("Failed to print result log. " + str(ex)) + + if not success: + logger.error("Failed to run:" + str(scan_result_msg)) return success, _result_log["Scan Result"], result_list diff --git a/tox.ini b/tox.ini index 32f79de..aa77e82 100644 --- a/tox.ini +++ b/tox.ini @@ -20,11 +20,12 @@ filterwarnings = ignore::DeprecationWarning [testenv:test_run] commands = - fosslight_source -p tests/test_files -j -o test_scan/scan_result -m - cat test_scan/scan_result_SRC.csv + fosslight_source -p tests/test_files -j -o test_scan/scan_result.csv -m + cat test_scan/scan_result.csv + fosslight_source -p tests/test_files -o test_scan/scan_result2 -f opossum -m fosslight_convert -p tests/json_result/scan_has_error.json -o test_convert/convert_result2 - fosslight_convert -p test_scan/scan_result.json -o test_convert/convert_result -m - cat test_convert/convert_result_SRC.csv + fosslight_convert -p test_scan/scancode_raw_result.json -o test_convert/convert_result.csv -f csv -m + cat test_convert/convert_result.csv python tests/cli_test.py [testenv:release] @@ -34,10 +35,11 @@ deps = commands = fosslight_source -h fosslight_convert -h - fosslight_source -p tests/test_files -j -o test_scan/scan_result -m - cat test_scan/scan_result_SRC.csv + fosslight_source -p tests/test_files -j -o test_scan/scan_result.csv -m + cat test_scan/scan_result.csv + fosslight_source -p tests/test_files -o test_scan/scan_result2 -f opossum -m fosslight_convert -p tests/json_result/scan_has_error.json -o test_convert/convert_result2 - fosslight_convert -p test_scan/scan_result.json -o test_convert/convert_result -m - cat test_convert/convert_result_SRC.csv + fosslight_convert -p test_scan/scancode_raw_result.json -o test_convert/convert_result.csv -f csv -m + cat test_convert/convert_result.csv python tests/cli_test.py pytest -v --flake8