From dc0455a139136cf9d05a68bb1f7959581e111c29 Mon Sep 17 00:00:00 2001 From: Soim Date: Wed, 27 Sep 2023 10:02:51 +0900 Subject: [PATCH 1/4] Modify run_scanners to return --- src/fosslight_source/cli.py | 56 ++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/src/fosslight_source/cli.py b/src/fosslight_source/cli.py index aa47f48c..82ab29a2 100755 --- a/src/fosslight_source/cli.py +++ b/src/fosslight_source/cli.py @@ -35,6 +35,7 @@ logger = logging.getLogger(constant.LOGGER_NAME) warnings.filterwarnings("ignore", category=FutureWarning) _PKG_NAME = "fosslight_source" +RESULT_KEY = "Scan Result" def main(): @@ -218,7 +219,7 @@ def merge_results(scancode_result=[], scanoss_result=[], spdx_downloads={}): return scancode_result -def run_scanners(path_to_scan, output_file_name="", _write_json_file=False, num_cores=-1, called_by_cli=True, +def run_scanners(path_to_scan, output_file_name="", write_json_file=False, num_cores=-1, called_by_cli=True, print_matched_text=False, format="", time_out=120, correct_mode=True, correct_filepath="", selected_scanner='all'): """ @@ -226,52 +227,55 @@ def run_scanners(path_to_scan, output_file_name="", _write_json_file=False, num_ :param path_to_scan: path of sourcecode to scan. :param output_file_name: path or file name (with path) for the output. - :param _write_json_file: if requested, keep the raw files. + :param write_json_file: if requested, keep the raw files. :param num_cores: number of cores used for scancode scanning. :param called_by_cli: if not called by cli, initialize logger. :param print_matched_text: if requested, output matched text (only for scancode). :param format: output format (excel, csv, opossum). :return success: success or failure of scancode. - :return _result_log["Scan Result"]: + :return result_log["Scan Result"]: :return merged_result: merged scan result of scancode and scanoss. :return license_list: matched text.(only for scancode) """ global logger - _start_time = datetime.now().strftime('%y%m%d_%H%M') + start_time = datetime.now().strftime('%y%m%d_%H%M') scancode_result = [] scanoss_result = [] merged_result = [] spdx_downloads = {} + result_log = {} success, msg, output_path, output_file, output_extension = check_output_format(output_file_name, format) - if output_extension != '.xlsx' and output_extension != "" and print_matched_text: + logger, result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{start_time}.txt"), + True, logging.INFO, logging.DEBUG, _PKG_NAME, path_to_scan) + if output_extension != '.xlsx' and output_extension and print_matched_text: logger.warning("-m option is only available for excel.") print_matched_text = False - if not success: - logger.error(f"Format error. {msg}") - sys.exit(1) - logger, _result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{_start_time}.txt"), - True, logging.INFO, logging.DEBUG, _PKG_NAME, path_to_scan) - - if selected_scanner == 'scancode' or selected_scanner == 'all' or selected_scanner == '': - success, _result_log["Scan Result"], scancode_result, license_list = run_scan(path_to_scan, output_file_name, - _write_json_file, num_cores, True, + if success: + if selected_scanner == 'scancode' or selected_scanner == 'all' or selected_scanner == '': + success, result_log[RESULT_KEY], scancode_result, license_list = run_scan(path_to_scan, output_file_name, + write_json_file, num_cores, True, print_matched_text, format, called_by_cli, time_out, correct_mode, correct_filepath) - if selected_scanner == 'scanoss' or selected_scanner == 'all' or selected_scanner == '': - scanoss_result = run_scanoss_py(path_to_scan, output_file_name, format, True, _write_json_file) - if selected_scanner not in SCANNER_TYPE: - print_help_msg_source_scanner() - sys.exit(1) - - spdx_downloads = get_spdx_downloads(path_to_scan) - merged_result = merge_results(scancode_result, scanoss_result, spdx_downloads) - - create_report_file(_start_time, merged_result, license_list, scanoss_result, selected_scanner, print_matched_text, - output_path, output_file, output_extension, correct_mode, correct_filepath, path_to_scan) + if selected_scanner == 'scanoss' or selected_scanner == 'all' or selected_scanner == '': + scanoss_result = run_scanoss_py(path_to_scan, output_file_name, format, True, write_json_file) + if selected_scanner in SCANNER_TYPE: + spdx_downloads = get_spdx_downloads(path_to_scan) + merged_result = merge_results(scancode_result, scanoss_result, spdx_downloads) + + create_report_file(start_time, merged_result, license_list, scanoss_result, selected_scanner, print_matched_text, + output_path, output_file, output_extension, correct_mode, correct_filepath, path_to_scan) + else: + print_help_msg_source_scanner() + result_log[RESULT_KEY] = "Unsupported scanner" + success = False + else: + result_log[RESULT_KEY] = f"Format error. {msg}" + logger.error(f"Format error. {msg}") + success = False - return success, _result_log["Scan Result"], merged_result, license_list, scanoss_result + return success, result_log.get(RESULT_KEY, ""), merged_result, license_list, scanoss_result if __name__ == '__main__': From 116236d8f2d41bfdc1684be80dde15c2eac28fa2 Mon Sep 17 00:00:00 2001 From: Soim Date: Wed, 27 Sep 2023 10:07:12 +0900 Subject: [PATCH 2/4] Update python version in github actions --- .github/workflows/publish-release.yml | 2 +- .github/workflows/pull-request.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml index 21c0504c..25cc7018 100644 --- a/.github/workflows/publish-release.yml +++ b/.github/workflows/publish-release.yml @@ -60,7 +60,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: '3.7' + python-version: '3.8' - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 9e37a606..018f37ee 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8] + python-version: [3.8, 3.11] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} From 181486ed22ddc19530a0b95cf71f6ea707c20598 Mon Sep 17 00:00:00 2001 From: Soim Date: Wed, 27 Sep 2023 11:37:28 +0900 Subject: [PATCH 3/4] Fix the visual indent warning --- src/fosslight_source/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fosslight_source/cli.py b/src/fosslight_source/cli.py index 82ab29a2..84090493 100755 --- a/src/fosslight_source/cli.py +++ b/src/fosslight_source/cli.py @@ -265,7 +265,7 @@ def run_scanners(path_to_scan, output_file_name="", write_json_file=False, num_c merged_result = merge_results(scancode_result, scanoss_result, spdx_downloads) create_report_file(start_time, merged_result, license_list, scanoss_result, selected_scanner, print_matched_text, - output_path, output_file, output_extension, correct_mode, correct_filepath, path_to_scan) + output_path, output_file, output_extension, correct_mode, correct_filepath, path_to_scan) else: print_help_msg_source_scanner() result_log[RESULT_KEY] = "Unsupported scanner" From ef56d170e3b4401477ca0ec8f1a178dc387016e1 Mon Sep 17 00:00:00 2001 From: Soim Date: Wed, 27 Sep 2023 11:48:28 +0900 Subject: [PATCH 4/4] Fix a bug related assignment --- src/fosslight_source/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fosslight_source/cli.py b/src/fosslight_source/cli.py index 84090493..36085bd9 100755 --- a/src/fosslight_source/cli.py +++ b/src/fosslight_source/cli.py @@ -243,6 +243,7 @@ def run_scanners(path_to_scan, output_file_name="", write_json_file=False, num_c scancode_result = [] scanoss_result = [] merged_result = [] + license_list = [] spdx_downloads = {} result_log = {} @@ -272,7 +273,6 @@ def run_scanners(path_to_scan, output_file_name="", write_json_file=False, num_c success = False else: result_log[RESULT_KEY] = f"Format error. {msg}" - logger.error(f"Format error. {msg}") success = False return success, result_log.get(RESULT_KEY, ""), merged_result, license_list, scanoss_result