diff --git a/requirements.txt b/requirements.txt index e7732e2..8cf9bc3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ typecode-libmagic fosslight_util>=1.4.28 PyYAML wheel>=0.38.1 +fosslight_binary diff --git a/src/fosslight_source/_scan_item.py b/src/fosslight_source/_scan_item.py index baa5ae5..7c803d8 100644 --- a/src/fosslight_source/_scan_item.py +++ b/src/fosslight_source/_scan_item.py @@ -14,6 +14,7 @@ "aclocal.m4", "configure", "configure.ac", "depcomp", "compile", "missing", "libtool.m4", "makefile"] +_exclude_extension = [".m4"] _exclude_directory = ["test", "tests", "doc", "docs"] _exclude_directory = [os.path.sep + dir_name + os.path.sep for dir_name in _exclude_directory] @@ -104,6 +105,8 @@ def is_exclude_dir(dir_path): def is_exclude_file(file_path, prev_dir=None, prev_dir_exclude_value=None): file_path = file_path.lower() filename = os.path.basename(file_path) + if os.path.splitext(filename)[1] in _exclude_extension: + return True if filename in _exclude_filename: return True diff --git a/src/fosslight_source/cli.py b/src/fosslight_source/cli.py index 8489961..aa47f48 100755 --- a/src/fosslight_source/cli.py +++ b/src/fosslight_source/cli.py @@ -39,7 +39,6 @@ def main(): global logger - success = True _result_log = {} path_to_scan = os.getcwd() @@ -50,8 +49,6 @@ def main(): selected_scanner = "" correct_mode = True - license_list = [] - scanoss_result = [] time_out = 120 core = -1 @@ -101,39 +98,11 @@ def main(): timer.setDaemon(True) timer.start() - _start_time = datetime.now().strftime('%y%m%d_%H%M') - success, msg, output_path, output_file, output_extension = check_output_format(output_file_name, format) - if output_extension != '.xlsx' and output_extension != "" and print_matched_text: - logger.warning("-m option is only available for excel.") - print_matched_text = False - if not success: - logger.error(f"Format error. {msg}") - sys.exit(1) - logger, _result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{_start_time}.txt"), - True, logging.INFO, logging.DEBUG, _PKG_NAME, path_to_scan) - if os.path.isdir(path_to_scan): - scancode_result = [] - scanoss_result = [] - merged_result = [] - spdx_downloads = {} - success = True - - if selected_scanner == 'scancode' or selected_scanner == 'all' or selected_scanner == '': - success, _result_log["Scan Result"], scancode_result, license_list = run_scan(path_to_scan, output_file_name, - write_json_file, core, True, - print_matched_text, format, True, - time_out, correct_mode, - correct_filepath) - if selected_scanner == 'scanoss' or selected_scanner == 'all' or selected_scanner == '': - scanoss_result = run_scanoss_py(path_to_scan, output_file_name, format, True, write_json_file) - if selected_scanner not in SCANNER_TYPE: - print_help_msg_source_scanner() - sys.exit(1) - spdx_downloads = get_spdx_downloads(path_to_scan) - merged_result = merge_results(scancode_result, scanoss_result, spdx_downloads) - create_report_file(_start_time, merged_result, license_list, scanoss_result, selected_scanner, print_matched_text, - output_path, output_file, output_extension, correct_mode, correct_filepath, path_to_scan) + result = [] + result = run_scanners(path_to_scan, output_file_name, write_json_file, core, True, + print_matched_text, format, time_out, correct_mode, correct_filepath, selected_scanner) + _result_log["Scan Result"] = result[1] try: logger.info(yaml.safe_dump(_result_log, allow_unicode=True, sort_keys=True)) @@ -249,5 +218,61 @@ def merge_results(scancode_result=[], scanoss_result=[], spdx_downloads={}): return scancode_result +def run_scanners(path_to_scan, output_file_name="", _write_json_file=False, num_cores=-1, called_by_cli=True, + print_matched_text=False, format="", time_out=120, correct_mode=True, correct_filepath="", + selected_scanner='all'): + """ + Run Scancode and scanoss.py for the given path. + + :param path_to_scan: path of sourcecode to scan. + :param output_file_name: path or file name (with path) for the output. + :param _write_json_file: if requested, keep the raw files. + :param num_cores: number of cores used for scancode scanning. + :param called_by_cli: if not called by cli, initialize logger. + :param print_matched_text: if requested, output matched text (only for scancode). + :param format: output format (excel, csv, opossum). + :return success: success or failure of scancode. + :return _result_log["Scan Result"]: + :return merged_result: merged scan result of scancode and scanoss. + :return license_list: matched text.(only for scancode) + """ + global logger + + _start_time = datetime.now().strftime('%y%m%d_%H%M') + scancode_result = [] + scanoss_result = [] + merged_result = [] + spdx_downloads = {} + + success, msg, output_path, output_file, output_extension = check_output_format(output_file_name, format) + if output_extension != '.xlsx' and output_extension != "" and print_matched_text: + logger.warning("-m option is only available for excel.") + print_matched_text = False + if not success: + logger.error(f"Format error. {msg}") + sys.exit(1) + logger, _result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{_start_time}.txt"), + True, logging.INFO, logging.DEBUG, _PKG_NAME, path_to_scan) + + if selected_scanner == 'scancode' or selected_scanner == 'all' or selected_scanner == '': + success, _result_log["Scan Result"], scancode_result, license_list = run_scan(path_to_scan, output_file_name, + _write_json_file, num_cores, True, + print_matched_text, format, called_by_cli, + time_out, correct_mode, correct_filepath) + if selected_scanner == 'scanoss' or selected_scanner == 'all' or selected_scanner == '': + scanoss_result = run_scanoss_py(path_to_scan, output_file_name, format, True, _write_json_file) + if selected_scanner not in SCANNER_TYPE: + print_help_msg_source_scanner() + sys.exit(1) + + spdx_downloads = get_spdx_downloads(path_to_scan) + merged_result = merge_results(scancode_result, scanoss_result, spdx_downloads) + + create_report_file(_start_time, merged_result, license_list, scanoss_result, selected_scanner, print_matched_text, + output_path, output_file, output_extension, correct_mode, correct_filepath, path_to_scan) + + return success, _result_log["Scan Result"], merged_result, license_list, scanoss_result + + if __name__ == '__main__': main() diff --git a/src/fosslight_source/run_scancode.py b/src/fosslight_source/run_scancode.py index 53be2cb..b3d4a8b 100755 --- a/src/fosslight_source/run_scancode.py +++ b/src/fosslight_source/run_scancode.py @@ -15,8 +15,8 @@ from ._parsing_scancode_file_item import parsing_file_item from ._parsing_scancode_file_item import get_error_from_header from ._license_matched import get_license_list_to_print -from fosslight_util.output_format import check_output_format, write_output_file -from fosslight_util.correct import correct_with_yaml +from fosslight_util.output_format import check_output_format +from fosslight_binary.binary_analysis import check_binary logger = logging.getLogger(constant.LOGGER_NAME) warnings.filterwarnings("ignore", category=FutureWarning) @@ -99,26 +99,15 @@ def run_scan(path_to_scan, output_file_name="", success = True result_list = sorted( result_list, key=lambda row: (''.join(row.licenses))) + + for scan_item in result_list: + if check_binary(os.path.join(path_to_scan, scan_item.file)): + scan_item.exclude = True + sheet_list["SRC_FL_Source"] = [scan_item.get_row_to_print() for scan_item in result_list] if need_license: sheet_list["matched_text"] = get_license_list_to_print(license_list) - output_file_without_ext = os.path.join(output_path, output_file) - if not called_by_cli: - if correct_mode: - success, msg_correct, correct_list = correct_with_yaml(correct_filepath, - path_to_scan, sheet_list) - if not success: - logger.info(f"No correction with yaml: {msg_correct}") - else: - sheet_list = correct_list - logger.info("Success to correct with yaml.") - success_to_write, writing_msg, result_file = write_output_file(output_file_without_ext, - output_extension, sheet_list) - if success_to_write: - logger.info(f"Writing Output file({result_file}, success:{success_to_write}") - else: - logger.error(f"Fail to generate result file. msg:({writing_msg})") except Exception as ex: success = False msg = str(ex)