From e38935f29f54c458f574d9ad0a35287947c6093c Mon Sep 17 00:00:00 2001 From: Jaekwon Bang Date: Thu, 17 Feb 2022 16:12:09 +0900 Subject: [PATCH 1/2] Add error handle for dependency-check --- src/fosslight_binary/_jar_analysis.py | 155 ++++++++++++------------ src/fosslight_binary/binary_analysis.py | 8 +- 2 files changed, 84 insertions(+), 79 deletions(-) diff --git a/src/fosslight_binary/_jar_analysis.py b/src/fosslight_binary/_jar_analysis.py index 7b30211..83b6e85 100644 --- a/src/fosslight_binary/_jar_analysis.py +++ b/src/fosslight_binary/_jar_analysis.py @@ -107,6 +107,7 @@ def analyze_jar_file(path_to_find_bin): owasp_items = {} remove_vulnerability_items = [] vulnerability_items = {} + success = True try: command = ['dependency-check', '--scan', f'{path_to_find_bin}', '--out', f'{path_to_find_bin}', @@ -115,86 +116,88 @@ def analyze_jar_file(path_to_find_bin): '--disableOssIndex', '--disableBundleAudit', '--cveValidForHours', '24', '-f', 'ALL'] run_analysis(command, dependency_check_run) - json_file = os.path.join(path_to_find_bin, 'dependency-check-report.json') - try: + json_file = os.path.join(path_to_find_bin, 'dependency-check-report.json') with open(json_file, 'r') as f: jar_contents = json.load(f) - - dependencies = jar_contents.get("dependencies") - for val in dependencies: - bin_with_path = "" - oss_name = "" - oss_ver = "" - oss_artifactid = "" - oss_groupid = "" - oss_dl_url = "" - oss_license = get_oss_lic_in_jar(val) - get_oss_info = False - - all_evidence = val.get("evidenceCollected") - vulnerability = val.get("vulnerabilityIds") - vendor_evidences = all_evidence.get('vendorEvidence') - product_evidences = all_evidence.get('productEvidence') - version_evidences = all_evidence.get('versionEvidence') - - # Check if the file is .jar file - # Even if the oss info is from pom.xml in jar file, the file name will be .jar file. - # But the oss info from pom.xml could be different from .jar file. - bin_with_path = val.get("filePath") - if not bin_with_path.endswith('.jar'): - bin_with_path = bin_with_path.split('.jar')[0] + '.jar' - - file_with_path = os.path.relpath(bin_with_path, path_to_find_bin) - # Get Version info from versionEvidence - for version_info in version_evidences: - oss_ver = get_oss_ver(version_info) - - # Get Artifact ID, Group ID, OSS Name from vendorEvidence - for vendor_info in vendor_evidences: - # Get OSS Info from POM - if vendor_info['source'] == 'pom': - if vendor_info['name'] == 'artifactid': - oss_artifactid = vendor_info['value'] - if vendor_info['name'] == 'groupid': - oss_groupid = vendor_info['value'] - if vendor_info['name'] == 'url': - oss_dl_url = vendor_info['value'] - if oss_artifactid != "" and oss_groupid != "": - oss_name = f"{oss_groupid}:{oss_artifactid}" - - # Check if get oss_name and version from pom - if oss_name != "" and oss_ver != "": - get_oss_info = True - - # If there is no pom.mxl in .jar file, get oss info from MANIFEST.MF file - if get_oss_info is False: - for product_info in product_evidences: - if product_info['source'] == 'Manifest': - if oss_name == "" and (product_info['name'] == 'Implementation-Title' or product_info['name'] == 'specification-title'): - oss_name = product_info['value'] - if oss_ver == "" and (product_info['name'] == 'Implementation-Version' or product_info['name'] == 'Bundle-Version'): - oss_ver = product_info['value'] - - # Get Vulnerability Info. - try: - vulnerability_items = get_vulnerability_info(file_with_path, vulnerability, vulnerability_items, remove_vulnerability_items) - except Exception as ex: - logger.info(f"Error to get vulnerability Info. : {ex}") - - if oss_name != "" or oss_ver != "" or oss_license != "" or oss_dl_url != "": - oss = OssItem(oss_name, oss_ver, oss_license, oss_dl_url) - oss.set_comment("OWASP Result. ") - - remove_owasp_item = owasp_items.get(file_with_path) - if remove_owasp_item: - remove_owasp_item.append(oss) - else: - owasp_items[file_with_path] = [oss] - except Exception as ex: - logger.warning(f"Error to read json file : {ex}") + logger.warning(f"Error to read dependency-check-report.json file : {ex}") + success = False + return + + dependencies = jar_contents.get("dependencies") + for val in dependencies: + bin_with_path = "" + oss_name = "" + oss_ver = "" + oss_artifactid = "" + oss_groupid = "" + oss_dl_url = "" + oss_license = get_oss_lic_in_jar(val) + get_oss_info = False + + all_evidence = val.get("evidenceCollected") + vulnerability = val.get("vulnerabilityIds") + vendor_evidences = all_evidence.get('vendorEvidence') + product_evidences = all_evidence.get('productEvidence') + version_evidences = all_evidence.get('versionEvidence') + + # Check if the file is .jar file + # Even if the oss info is from pom.xml in jar file, the file name will be .jar file. + # But the oss info from pom.xml could be different from .jar file. + bin_with_path = val.get("filePath") + if not bin_with_path.endswith('.jar'): + bin_with_path = bin_with_path.split('.jar')[0] + '.jar' + + file_with_path = os.path.relpath(bin_with_path, path_to_find_bin) + # Get Version info from versionEvidence + for version_info in version_evidences: + oss_ver = get_oss_ver(version_info) + + # Get Artifact ID, Group ID, OSS Name from vendorEvidence + for vendor_info in vendor_evidences: + # Get OSS Info from POM + if vendor_info['source'] == 'pom': + if vendor_info['name'] == 'artifactid': + oss_artifactid = vendor_info['value'] + if vendor_info['name'] == 'groupid': + oss_groupid = vendor_info['value'] + if vendor_info['name'] == 'url': + oss_dl_url = vendor_info['value'] + if oss_artifactid != "" and oss_groupid != "": + oss_name = f"{oss_groupid}:{oss_artifactid}" + + # Check if get oss_name and version from pom + if oss_name != "" and oss_ver != "": + get_oss_info = True + + # If there is no pom.mxl in .jar file, get oss info from MANIFEST.MF file + if get_oss_info is False: + for product_info in product_evidences: + if product_info['source'] == 'Manifest': + if oss_name == "" and (product_info['name'] == 'Implementation-Title' or product_info['name'] == 'specification-title'): + oss_name = product_info['value'] + if oss_ver == "" and (product_info['name'] == 'Implementation-Version' or product_info['name'] == 'Bundle-Version'): + oss_ver = product_info['value'] + + # Get Vulnerability Info. + try: + vulnerability_items = get_vulnerability_info(file_with_path, vulnerability, vulnerability_items, remove_vulnerability_items) + except Exception as ex: + logger.info(f"Error to get vulnerability Info. : {ex}") + success = False + + if oss_name != "" or oss_ver != "" or oss_license != "" or oss_dl_url != "": + oss = OssItem(oss_name, oss_ver, oss_license, oss_dl_url) + oss.set_comment("OWASP Result. ") + + remove_owasp_item = owasp_items.get(file_with_path) + if remove_owasp_item: + remove_owasp_item.append(oss) + else: + owasp_items[file_with_path] = [oss] except Exception as ex: logger.warning(f"Error to use dependency-check : {ex}") + success = False - return owasp_items, vulnerability_items + return owasp_items, vulnerability_items, success diff --git a/src/fosslight_binary/binary_analysis.py b/src/fosslight_binary/binary_analysis.py index 07ea233..c01f1f2 100755 --- a/src/fosslight_binary/binary_analysis.py +++ b/src/fosslight_binary/binary_analysis.py @@ -154,10 +154,12 @@ def find_binaries(path_to_find_bin, output_dir, format, dburl=""): # Run OWASP Dependency-check if found_jar: logger.info("Run OWASP Dependency-check to analyze .jar file") - owasp_items, vulnerability_items = analyze_jar_file(path_to_find_bin) - if owasp_items: + owasp_items, vulnerability_items, success = analyze_jar_file(path_to_find_bin) + if success: return_list = merge_binary_list(owasp_items, vulnerability_items, return_list) - extended_header = JAR_VUL_HEADER + extended_header = JAR_VUL_HEADER + else: + logger.warning("Fail to analyze jar file") return_list, db_loaded_cnt = get_oss_info_from_db(return_list, dburl) return_list = sorted(return_list, key=lambda row: (row.bin_name)) From 6de92c67aafeb6d30f51006a49ee6ac8380b6202 Mon Sep 17 00:00:00 2001 From: Jaekwon Bang Date: Fri, 18 Feb 2022 11:24:22 +0900 Subject: [PATCH 2/2] Remove double try-except code --- src/fosslight_binary/_jar_analysis.py | 38 +++++++++++-------------- src/fosslight_binary/binary_analysis.py | 2 +- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/src/fosslight_binary/_jar_analysis.py b/src/fosslight_binary/_jar_analysis.py index 83b6e85..05333f9 100644 --- a/src/fosslight_binary/_jar_analysis.py +++ b/src/fosslight_binary/_jar_analysis.py @@ -109,23 +109,23 @@ def analyze_jar_file(path_to_find_bin): vulnerability_items = {} success = True - try: - command = ['dependency-check', '--scan', f'{path_to_find_bin}', '--out', f'{path_to_find_bin}', - '--disableArchive', '--disableAssembly', '--disableRetireJS', '--disableNodeJS', - '--disableNodeAudit', '--disableNugetconf', '--disableNuspec', '--disableOpenSSL', - '--disableOssIndex', '--disableBundleAudit', '--cveValidForHours', '24', '-f', 'ALL'] - run_analysis(command, dependency_check_run) + command = ['dependency-check', '--scan', f'{path_to_find_bin}', '--out', f'{path_to_find_bin}', + '--disableArchive', '--disableAssembly', '--disableRetireJS', '--disableNodeJS', + '--disableNodeAudit', '--disableNugetconf', '--disableNuspec', '--disableOpenSSL', + '--disableOssIndex', '--disableBundleAudit', '--cveValidForHours', '24', '-f', 'ALL'] + run_analysis(command, dependency_check_run) - try: - json_file = os.path.join(path_to_find_bin, 'dependency-check-report.json') - with open(json_file, 'r') as f: - jar_contents = json.load(f) - except Exception as ex: - logger.warning(f"Error to read dependency-check-report.json file : {ex}") - success = False - return + try: + json_file = os.path.join(path_to_find_bin, 'dependency-check-report.json') + with open(json_file, 'r') as f: + jar_contents = json.load(f) + except Exception as ex: + logger.debug(f"Error to read dependency-check-report.json file : {ex}") + success = False + return - dependencies = jar_contents.get("dependencies") + dependencies = jar_contents.get("dependencies") + try: for val in dependencies: bin_with_path = "" oss_name = "" @@ -181,11 +181,7 @@ def analyze_jar_file(path_to_find_bin): oss_ver = product_info['value'] # Get Vulnerability Info. - try: - vulnerability_items = get_vulnerability_info(file_with_path, vulnerability, vulnerability_items, remove_vulnerability_items) - except Exception as ex: - logger.info(f"Error to get vulnerability Info. : {ex}") - success = False + vulnerability_items = get_vulnerability_info(file_with_path, vulnerability, vulnerability_items, remove_vulnerability_items) if oss_name != "" or oss_ver != "" or oss_license != "" or oss_dl_url != "": oss = OssItem(oss_name, oss_ver, oss_license, oss_dl_url) @@ -197,7 +193,7 @@ def analyze_jar_file(path_to_find_bin): else: owasp_items[file_with_path] = [oss] except Exception as ex: - logger.warning(f"Error to use dependency-check : {ex}") + logger.debug(f"Error to get depency Info in jar_contets: {ex}") success = False return owasp_items, vulnerability_items, success diff --git a/src/fosslight_binary/binary_analysis.py b/src/fosslight_binary/binary_analysis.py index c01f1f2..f2f3a8d 100755 --- a/src/fosslight_binary/binary_analysis.py +++ b/src/fosslight_binary/binary_analysis.py @@ -159,7 +159,7 @@ def find_binaries(path_to_find_bin, output_dir, format, dburl=""): return_list = merge_binary_list(owasp_items, vulnerability_items, return_list) extended_header = JAR_VUL_HEADER else: - logger.warning("Fail to analyze jar file") + logger.warning("Could not find OSS information for some jar files.") return_list, db_loaded_cnt = get_oss_info_from_db(return_list, dburl) return_list = sorted(return_list, key=lambda row: (row.bin_name))