Merge pull request #37 from fosslight/develop

Add format(-f) option and modify the function to write output

Author: soimkim

.github/workflows/pull-request.yml

@@ -4,7 +4,8 @@ name: Pull requests fosslight_source_scanner
 
 on:
   pull_request:
-    branches: [ main ]
+    branches:
+      - '*'
 
 jobs:
   build:

requirements-dev.txt

@@ -2,3 +2,4 @@ tox
 pytest
 pytest-cov
 pytest-flake8
+flake8==3.9.2

src/fosslight_source/_help.py

@@ -21,7 +21,9 @@
             -h\t\t\t\t   Print help message
             -j\t\t\t\t   Generate additional result of executing ScanCode in json format
             -m\t\t\t\t   Print the Matched text for each license on a separate sheet
-            -o <file_name>\t\t   Output file name"""
+            -o <output_path>\t\t   Output path
+            \t\t\t\t    (If you want to generate the specific file name, add the output path with file name.)
+            -f <format>\t\t\t   Output file format (excel, csv, opossum)"""
 
 _HELP_MESSAGE_CONVERT = """
     Usage: fosslight_convert [option1] <arg1> [option2] <arg2>...
@@ -35,7 +37,9 @@
         Optional
             -h\t\t\t\t   Print help message
             -m\t\t\t\t   Print the Matched text for each license on a separate sheet
-            -o <file_name>\t\t   Output file name"""
+            -o <output_path>\t\t   Output path
+            \t\t\t\t    (If you want to generate the specific file name, add the output path with file name.)
+            -f <format>\t\t\t   Output file format (excel, csv, opossum)"""
 
 
 def print_help_msg_source():

src/fosslight_source/convert_scancode.py

@@ -13,21 +13,46 @@
 from fosslight_util.set_log import init_log
 import yaml
 from ._parsing_scancode_file_item import parsing_file_item, get_error_from_header
-from fosslight_util.write_excel import write_excel_and_csv
+from fosslight_util.output_format import check_output_format, write_output_file
 from ._help import print_help_msg_convert
 from ._license_matched import get_license_list_to_print
 
 logger = logging.getLogger(constant.LOGGER_NAME)
 _PKG_NAME = "fosslight_source"
 
 
-def convert_json_to_excel(scancode_json, excel_name, result_log, need_license=False):
+def convert_json_to_output_report(scancode_json, output_file_name, need_license=False, format=""):
+    global logger
+
     sheet_license_prefix = "matched_text"
-    sheet_SRC_prefix = "SRC"
+    sheet_SRC_prefix = "SRC_FL_Source"
     file_list = []
     lic_list = {}
     msg = ""
     success = True
+    start_time = datetime.now().strftime('%Y%m%d_%H%M%S')
+    _json_ext = ".json"
+
+    success, msg, output_path, output_file, output_extension = check_output_format(output_file_name, format)
+    if success:
+        if output_path == "":
+            output_path = os.getcwd()
+        else:
+            output_path = os.path.abspath(output_path)
+
+        if output_file == "":
+            if output_extension == _json_ext:
+                output_file = "Opossum_input_" + start_time
+            else:
+                output_file = "FOSSLight-Report_" + start_time
+    else:
+        output_path = os.getcwd()
+
+    logger, result_log = init_log(os.path.join(output_path, "fosslight_src_log_" + start_time + ".txt"),
+                                  True, logging.INFO, logging.DEBUG, _PKG_NAME)
+    if not success:
+        logger.error("Fail to convert scancode: " + msg)
+        return []
 
     try:
         sheet_list = {}
@@ -60,10 +85,11 @@ def convert_json_to_excel(scancode_json, excel_name, result_log, need_license=Fa
                         except Exception as ex:
                             logger.warning("Error parsing "+file+":" + str(ex))
 
-        success_to_write, writing_msg = write_excel_and_csv(excel_name, sheet_list)
-        logger.info("Writing excel :" + str(success_to_write) + " " + writing_msg)
+        output_file_without_ext = os.path.join(output_path, output_file)
+        success_to_write, writing_msg = write_output_file(output_file_without_ext, output_extension, sheet_list)
+        logger.info("Writing Output file(" + output_file + output_extension + "):" + str(success_to_write) + " " + writing_msg)
         if success_to_write:
-            result_log["FOSSLight Report"] = excel_name + ".xlsx"
+            result_log["Output file"] = output_file_without_ext + output_extension
 
     except Exception as ex:
         success = False
@@ -100,16 +126,14 @@ def get_detected_licenses_from_scancode(scancode_json_file, need_license):
 
 
 def main():
-    global logger
-
     argv = sys.argv[1:]
     path_to_find_json = ""
-    start_time = datetime.now().strftime('%Y%m%d_%H%M%S')
     output_file_name = ""
     print_matched_text = False
+    format = ""
 
     try:
-        opts, args = getopt.getopt(argv, 'hmp:o:')
+        opts, args = getopt.getopt(argv, 'hmp:o:f:')
         for opt, arg in opts:
             if opt == "-h":
                 print_help_msg_convert()
@@ -119,22 +143,15 @@ def main():
                 output_file_name = arg
             elif opt == "-m":
                 print_matched_text = True
+            elif opt == "-f":
+                format = arg
     except Exception:
         print_help_msg_convert()
 
     if path_to_find_json == "":
         print_help_msg_convert()
 
-    if output_file_name == "":
-        output_dir = os.getcwd()
-        oss_report_name = "FOSSLight-Report_" + start_time
-    else:
-        oss_report_name = output_file_name
-        output_dir = os.path.dirname(os.path.abspath(output_file_name))
-
-    logger, _result_log = init_log(os.path.join(output_dir, "fosslight_src_log_" + start_time + ".txt"),
-                                   True, logging.INFO, logging.DEBUG, _PKG_NAME)
-    convert_json_to_excel(path_to_find_json, oss_report_name, _result_log, print_matched_text)
+    convert_json_to_output_report(path_to_find_json, output_file_name, print_matched_text, format)
 
 
 if __name__ == '__main__':

src/fosslight_source/run_scancode.py

@@ -18,9 +18,9 @@
 from fosslight_util.timer_thread import TimerThread
 from ._parsing_scancode_file_item import parsing_file_item
 from ._parsing_scancode_file_item import get_error_from_header
-from fosslight_util.write_excel import write_excel_and_csv
 from ._help import print_help_msg_source
 from ._license_matched import get_license_list_to_print
+from fosslight_util.output_format import check_output_format, write_output_file
 
 logger = logging.getLogger(constant.LOGGER_NAME)
 warnings.filterwarnings("ignore", category=FutureWarning)
@@ -33,9 +33,10 @@ def main():
     write_json_file = False
     output_file = ""
     print_matched_text = False
+    format = ""
 
     try:
-        opts, args = getopt.getopt(argv, 'hmjp:o:')
+        opts, args = getopt.getopt(argv, 'hmjp:o:f:')
         for opt, arg in opts:
             if opt == "-h":
                 print_help_msg_source()
@@ -47,108 +48,124 @@ def main():
                 output_file = arg
             elif opt == "-m":
                 print_matched_text = True
+            elif opt == "-f":
+                format = arg
     except Exception:
         print_help_msg_source()
 
     timer = TimerThread()
     timer.setDaemon(True)
     timer.start()
-    run_scan(path_to_scan, output_file, write_json_file, -1, False, print_matched_text)
+
+    run_scan(path_to_scan, output_file, write_json_file, -1, False, print_matched_text, format)
 
 
 def run_scan(path_to_scan, output_file_name="",
-             _write_json_file=False, num_cores=-1, return_results=False, need_license=False):
+             _write_json_file=False, num_cores=-1, return_results=False, need_license=False, format=""):
     global logger
 
     success = True
     msg = ""
     _str_final_result_log = ""
     _result_log = {}
     result_list = []
+    _json_ext = ".json"
 
     _windows = platform.system() == "Windows"
     start_time = datetime.now().strftime('%Y%m%d_%H%M%S')
 
-    if output_file_name == "":
-        output_file = "FOSSLight-Report_" + start_time
-        output_json_file = "scancode_" + start_time
-        output_dir = os.getcwd()
-    else:
-        output_file = output_file_name
-        output_json_file = output_file_name
-        output_dir = os.path.dirname(os.path.abspath(output_file_name))
-
-    logger, _result_log = init_log(os.path.join(output_dir, "fosslight_src_log_"+start_time+".txt"),
-                                   True, logging.INFO, logging.DEBUG, _PKG_NAME, path_to_scan)
-
-    if path_to_scan == "":
-        if _windows:
-            path_to_scan = os.getcwd()
+    success, msg, output_path, output_file, output_extension = check_output_format(output_file_name, format)
+    if success:
+        if output_path == "":
+            output_path = os.getcwd()
         else:
-            print_help_msg_source()
+            output_path = os.path.abspath(output_path)
 
-    num_cores = multiprocessing.cpu_count() - 1 if num_cores < 0 else num_cores
+        if output_file == "":
+            if output_extension == _json_ext:
+                output_file = "Opossum_input_" + start_time
+            else:
+                output_file = "FOSSLight-Report_" + start_time
 
-    if os.path.isdir(path_to_scan):
-        try:
-            output_json_file = output_json_file+".json" if _write_json_file\
-                else ""
+        if _write_json_file:
+            output_json_file = os.path.join(output_path, "scancode_raw_result.json")
+        else:
+            output_json_file = ""
 
-            rc, results = cli.run_scan(path_to_scan, max_depth=100,
-                                       strip_root=True, license=True,
-                                       copyright=True, return_results=True,
-                                       processes=num_cores,
-                                       output_json_pp=output_json_file,
-                                       only_findings=True, license_text=True)
+        logger, _result_log = init_log(os.path.join(output_path, "fosslight_src_log_"+start_time+".txt"),
+                                       True, logging.INFO, logging.DEBUG, _PKG_NAME, path_to_scan)
 
-            if not rc:
-                msg = "Source code analysis failed."
-                success = False
+        if path_to_scan == "":
+            if _windows:
+                path_to_scan = os.getcwd()
+            else:
+                print_help_msg_source()
 
-            if results:
-                sheet_list = {}
-                has_error = False
-                if "headers" in results:
-                    has_error, error_msg = get_error_from_header(results["headers"])
-                    if has_error:
-                        _result_log["Error_files"] = error_msg
-                        msg = "Failed to analyze :" + error_msg
-                if "files" in results:
-                    rc, result_list, parsing_msg, license_list = parsing_file_item(results["files"], has_error, need_license)
-                    _result_log["Parsing Log"] = parsing_msg
-                    if rc:
-                        if not success:
-                            success = True
-                        result_list = sorted(
-                            result_list, key=lambda row: (''.join(row.licenses)))
-                        sheet_list["SRC"] = [scan_item.get_row_to_print() for scan_item in result_list]
-                        if need_license:
-                            sheet_list["matched_text"] = get_license_list_to_print(license_list)
-
-                        success_to_write, writing_msg = write_excel_and_csv(
-                            output_file, sheet_list)
-                        logger.info("Writing excel :" + str(success_to_write) + " " + writing_msg)
-                        if success_to_write:
-                            _result_log["FOSSLight Report"] = output_file + ".xlsx"
-        except Exception as ex:
+        num_cores = multiprocessing.cpu_count() - 1 if num_cores < 0 else num_cores
+
+        if os.path.isdir(path_to_scan):
+            try:
+                rc, results = cli.run_scan(path_to_scan, max_depth=100,
+                                           strip_root=True, license=True,
+                                           copyright=True, return_results=True,
+                                           processes=num_cores,
+                                           output_json_pp=output_json_file,
+                                           only_findings=True, license_text=True)
+
+                if not rc:
+                    msg = "Source code analysis failed."
+                    success = False
+
+                if results:
+                    sheet_list = {}
+                    has_error = False
+                    if "headers" in results:
+                        has_error, error_msg = get_error_from_header(results["headers"])
+                        if has_error:
+                            _result_log["Error_files"] = error_msg
+                            msg = "Failed to analyze :" + error_msg
+                    if "files" in results:
+                        rc, result_list, parsing_msg, license_list = parsing_file_item(results["files"], has_error, need_license)
+                        _result_log["Parsing Log"] = parsing_msg
+                        if rc:
+                            if not success:
+                                success = True
+                            result_list = sorted(
+                                result_list, key=lambda row: (''.join(row.licenses)))
+                            sheet_list["SRC_FL_Source"] = [scan_item.get_row_to_print() for scan_item in result_list]
+                            if need_license:
+                                sheet_list["matched_text"] = get_license_list_to_print(license_list)
+
+                            output_file_without_ext = os.path.join(output_path, output_file)
+                            success_to_write, writing_msg = write_output_file(output_file_without_ext, output_extension,
+                                                                              sheet_list)
+                            logger.info("Writing Output file(" + output_file + output_extension + "):" + str(success_to_write)
+                                        + " " + writing_msg)
+                            if success_to_write:
+                                _result_log["Output file"] = output_file_without_ext + output_extension
+
+            except Exception as ex:
+                success = False
+                msg = str(ex)
+                logger.error("Analyze " + path_to_scan + ":" + msg)
+        else:
             success = False
-            msg = str(ex)
-            logger.error("Analyze " + path_to_scan + ":" + msg)
-    else:
-        success = False
-        msg = "Check the path to scan. :" + path_to_scan
+            msg = "Check the path to scan. :" + path_to_scan
 
-    if not return_results:
-        result_list = []
+        if not return_results:
+            result_list = []
 
-    scan_result_msg = str(success) if msg == "" else str(success) + "," + msg
+    scan_result_msg = str(success) if msg == "" else str(success) + ", " + msg
     _result_log["Scan Result"] = scan_result_msg
-    _result_log["Output Directory"] = output_dir
+    _result_log["Output Directory"] = output_path
     try:
         _str_final_result_log = yaml.safe_dump(_result_log, allow_unicode=True, sort_keys=True)
         logger.info(_str_final_result_log)
     except Exception as ex:
         logger.warning("Failed to print result log. " + str(ex))
+
+    if not success:
+        logger.error("Failed to run:" + str(scan_result_msg))
     return success, _result_log["Scan Result"], result_list
 
 

tox.ini

@@ -20,11 +20,12 @@ filterwarnings = ignore::DeprecationWarning
 
 [testenv:test_run]
 commands =
-  fosslight_source -p tests/test_files -j -o test_scan/scan_result -m
-  cat test_scan/scan_result_SRC.csv
+  fosslight_source -p tests/test_files -j -o test_scan/scan_result.csv -m
+  cat test_scan/scan_result.csv
+  fosslight_source -p tests/test_files -o test_scan/scan_result2 -f opossum -m
   fosslight_convert -p tests/json_result/scan_has_error.json -o test_convert/convert_result2
-  fosslight_convert -p test_scan/scan_result.json -o test_convert/convert_result -m
-  cat test_convert/convert_result_SRC.csv
+  fosslight_convert -p test_scan/scancode_raw_result.json -o test_convert/convert_result.csv -f csv -m
+  cat test_convert/convert_result.csv
   python tests/cli_test.py
 
 [testenv:release]
@@ -34,10 +35,11 @@ deps =
 commands =
   fosslight_source -h
   fosslight_convert -h
-  fosslight_source -p tests/test_files -j -o test_scan/scan_result -m
-  cat test_scan/scan_result_SRC.csv
+  fosslight_source -p tests/test_files -j -o test_scan/scan_result.csv -m
+  cat test_scan/scan_result.csv
+  fosslight_source -p tests/test_files -o test_scan/scan_result2 -f opossum -m
   fosslight_convert -p tests/json_result/scan_has_error.json -o test_convert/convert_result2
-  fosslight_convert -p test_scan/scan_result.json -o test_convert/convert_result -m
-  cat test_convert/convert_result_SRC.csv
+  fosslight_convert -p test_scan/scancode_raw_result.json -o test_convert/convert_result.csv -f csv -m
+  cat test_convert/convert_result.csv
   python tests/cli_test.py
   pytest -v --flake8