diff --git a/src/soliddriver_checks/cli.py b/src/soliddriver_checks/cli.py index 67b5c94..d364d5f 100644 --- a/src/soliddriver_checks/cli.py +++ b/src/soliddriver_checks/cli.py @@ -114,7 +114,7 @@ def check(dst): "out_format", type=click.Choice(FORMAT_TYPES), default="json", - help="Specify output format", + help="Specify output format (PDF is in Beta)", ) @click.option( "--query", diff --git a/src/soliddriver_checks/config/soliddriver-checks.conf b/src/soliddriver_checks/config/soliddriver-checks.conf index 7df9af0..d2e8cdd 100644 --- a/src/soliddriver_checks/config/soliddriver-checks.conf +++ b/src/soliddriver_checks/config/soliddriver-checks.conf @@ -12,6 +12,9 @@ { "name":"GPLv2" }, + { + "name":"GPL v2 only" + }, { "name":"GPL-2.0" }, diff --git a/src/soliddriver_checks/config/templates/kmp-checks.html.jinja b/src/soliddriver_checks/config/templates/kmp-checks.html.jinja index 7d14dfb..42de2b2 100644 --- a/src/soliddriver_checks/config/templates/kmp-checks.html.jinja +++ b/src/soliddriver_checks/config/templates/kmp-checks.html.jinja @@ -54,10 +54,15 @@ th.detail_6{ width:240px; background-color:#90EBCD; - }th.detail_7{ + } + th.detail_7{ width:240px; background-color:#90EBCD; } + th.detail_8{ + width:240px; + background-color:#90EBCD; + } td{ text-align:left; vertical-align: top; diff --git a/src/soliddriver_checks/utils/data_exporter.py b/src/soliddriver_checks/utils/data_exporter.py index b6b43dc..dce2a26 100644 --- a/src/soliddriver_checks/utils/data_exporter.py +++ b/src/soliddriver_checks/utils/data_exporter.py @@ -155,6 +155,7 @@ def _copy_work_sheep(self, title, ws): ws["A5"].value = _get_version() ws["A6"].value = _generate_timestamp() + ws.column_dimensions['A'].width = 200 class RPMsExporter: def __init__(self): @@ -168,20 +169,9 @@ def _summary_symbol_result(self, val): result = "" if unfound_no > 0: - # result = "Can not find symbols like {} ... in RPM! ".format( - # val["unfound"][0] - # ) - result = f"Number of symbols can not be found in KMP: {unfound_no}" if cs_mm_no > 0: - # result = ( - # result - # + "Symbols check sum like {} ... does not match in RPM!".format( - # val["checksum-mismatch"][0] - # ) - # ) - result = result + f" Number of symbols checksum does not match: {cs_mm_no}" return result @@ -231,15 +221,25 @@ def _get_summary_table(self, rpm_table): columns=[ "Vendor", "Total KMPs", - "Driver Checks", "License", "Signature", "Weak Module Invoked", + "Supported Flag/Signature Checks Failed", "Symbols Check Failed", + "Modalias Check Failed", ] ) - def drivers_check(rpms_sf, rpms_is): + def alias_check(alias): + num = 0 + + for i in range(len(alias)): + if alias.iat[i]["match_all"] or len(alias.iat[i]["unmatched_km_alias"]) > 0 or len(alias.iat[i]["unmatched_kmp_alias"]) > 0: + num += 1 + + return num + + def supported_sig_check(rpms_sf, rpms_is): """ rpms_sf: RPMs' driver supported flag rpm_is : RPMs' is_signed column @@ -258,7 +258,7 @@ def drivers_check(rpms_sf, rpms_is): if not found_issues: num += 1 - return num + return rpms - num def license_check(vld_lics, rpm_licenses, driver_license): count = 0 @@ -279,8 +279,8 @@ def license_check(vld_lics, rpm_licenses, driver_license): for v in vendors: df_vendor = df.loc[df["vendor"] == v] total = len(df_vendor.index) - dc = drivers_check(df_vendor["df-supported"], df_vendor["is-signed"]) - failed = len( + spc = supported_sig_check(df_vendor["df-supported"], df_vendor["is-signed"]) + symc = len( df_vendor.loc[df_vendor["sym-check"] == "failed", "sym-check"].index ) lic_check = license_check( @@ -294,16 +294,18 @@ def license_check(vld_lics, rpm_licenses, driver_license): ].index ) wm_invoked = len(df_vendor.loc[df_vendor["wm-invoked"], "wm-invoked"].index) + aliasc = alias_check(df_vendor["modalias"]) df_summary = df_summary.append( { "Vendor": v, "Total KMPs": total, - "Driver Checks": f"{dc} ({dc/total * 100:.2f}%)", "License": f"{lic_check} ({lic_check/total * 100:.2f}%)", "Signature": f"{no_sig} ({no_sig/total * 100:.2f}%)", "Weak Module Invoked": f"{wm_invoked} ({wm_invoked/total * 100:.2f}%)", - "Symbols Check Failed": f"{failed} ({failed/total * 100:.2f}%)", + "Supported Flag/Signature Checks Failed": f"{spc} ({spc/total * 100:.2f}%)", + "Symbols Check Failed": f"{symc} ({symc/total * 100:.2f}%)", + "Modalias Check Failed": f"{aliasc} ({aliasc/total * 100:.2f}%)", }, ignore_index=True, ) @@ -327,20 +329,22 @@ def _get_summary_table_html(self, rpm_table): for i, row in df_summary.iterrows(): vendor = row["Vendor"] total_rpms = row["Total KMPs"] - dc = row["Driver Checks"] + ssc = row["Supported Flag/Signature Checks Failed"] lic_check = row["License"] signature = row["Signature"] wm_invoked = row["Weak Module Invoked"] sym_failed = row["Symbols Check Failed"] + alias_failed = row["Modalias Check Failed"] row_passed = False if ( vendor != "" - and int(dc.split(" ")[0]) == total_rpms + and int(ssc.split(" ")[0]) == 0 and int(lic_check.split(" ")[0]) == total_rpms and int(signature.split(" ")[0]) == total_rpms and int(wm_invoked.split(" ")[0]) == total_rpms and int(sym_failed.split(" ")[0]) == 0 + and int(alias_failed.split(" ")[0]) == 0 ): row_passed = True with tr() as r: @@ -353,13 +357,6 @@ def _get_summary_table_html(self, rpm_table): tv.set_attribute("class", "important_failed") with td(total_rpms) as t: t.set_attribute("class", "summary_total") - with td(dc) as t: - if int(dc.split(" ")[0]) != total_rpms: - t.set_attribute( - "class", "critical_failed summary_number" - ) - else: - t.set_attribute("class", "summary_number") with td(lic_check) as t: if int(lic_check.split(" ")[0]) != total_rpms: t.set_attribute( @@ -381,6 +378,13 @@ def _get_summary_table_html(self, rpm_table): ) else: t.set_attribute("class", "summary_number") + with td(ssc) as t: + if int(ssc.split(" ")[0]) != 0: + t.set_attribute( + "class", "critical_failed summary_number" + ) + else: + t.set_attribute("class", "summary_number") with td(sym_failed) as t: if int(sym_failed.split(" ")[0]) != 0: t.set_attribute( @@ -388,6 +392,13 @@ def _get_summary_table_html(self, rpm_table): ) else: t.set_attribute("class", "summary_number") + with td(alias_failed) as t: + if int(alias_failed.split(" ")[0]) > 0: + t.set_attribute( + "class", "important_falied summary_number" + ) + else: + t.set_attribute("class", "summary_number") return tb @@ -399,11 +410,11 @@ def _rename_rpm_detail_columns(self, rpm_table): "path": "Path", "vendor": "Vendor", "signature": "Signature", - # "distribution": "Distribution", "license": "License", "wm-invoked": "Weak Module Invoked", - "df-supported": "Driver Checks", + "df-supported": "Supported Flag Check", "sym-check": "Symbols Check", + "modalias": "Modalias Check", "dv-licenses": "Driver Licenses", "is-signed": "is-signed", } @@ -435,6 +446,30 @@ def _fmt_driver_license_check(self, rpm_license, driver_licenses, vld_lics): ) return chk_result + + def _fmt_modalias_check(self, alias): + match_all = alias["match_all"] + km_unmatched = len(alias["unmatched_km_alias"]) + kmp_unmatched = len(alias["unmatched_kmp_alias"]) + + if match_all: + return "KMP can match all the devices! Highly not recommended!" + + message = "" + + if km_unmatched > 0: + message += "Alias found in kernel module but no match in it's package: " + for kmu in alias["unmatched_km_alias"]: + message += kmu + ", " + message += "\n" + + if kmp_unmatched > 0: + message += "Alias found in the package but no match in it's kernel module: " + for kmpu in alias["unmatched_kmp_alias"]: + message += kmpu + ", " + + return message + def _get_table_detail_html(self, rpm_table): df = self._rename_rpm_detail_columns(rpm_table) @@ -444,7 +479,7 @@ def _get_table_detail_html(self, rpm_table): tb.set_attribute("class", "table_center") with tr(): th("KMP Checks", colspan=6).set_attribute("class", f"detail_rpm") - th("Kernel Module Checks", colspan=2).set_attribute("class", f"detail_kernel_module") + th("Kernel Module Checks", colspan=3).set_attribute("class", f"detail_kernel_module") with tr(): th("Name").set_attribute("class", f"detail_0") th("Path").set_attribute("class", f"detail_1") @@ -453,7 +488,8 @@ def _get_table_detail_html(self, rpm_table): th(raw("LicenseKMP and it's kernel modules should use open source licenses.")).set_attribute("class", f"detail_4 tooltip") th(raw("Weak Module InvokedWeak Module is necessary to make 3rd party kernel modules installed for one kernel available to KABI-compatible kernels. ")).set_attribute("class", f"detail_5 tooltip") th(raw("Supported Flag/Signature\"supported\" flag:
\"yes\": Only supported by SUSE
\"external\": supported by both SUSE and vendor")).set_attribute("class", f"detail_6 tooltip") - th(raw("Symbolssymbols check is to check whether the symbols in kernel modules matches the symbols in its package.")).set_attribute("class", f"detail_7 tooltip") + th(raw("SymbolsSymbols check is to check whether the symbols in kernel modules matches the symbols in its package.")).set_attribute("class", f"detail_7 tooltip") + th(raw("ModaliasModalias check is to check whether the modalias in kernel modules matches the modalias in its package.")).set_attribute("class", f"detail_8 tooltip") for i, row in df.iterrows(): with tr() as r: @@ -467,9 +503,11 @@ def _get_table_detail_html(self, rpm_table): sym_check = self._get_sym_check_failed( row["Symbols Check"] ).replace("\n", "
") - supported = row["Driver Checks"] + supported = row["Supported Flag Check"] is_signed = row["is-signed"] - dc_err = self._combine_driver_check_errs(supported, is_signed) + alias = row["Modalias Check"] + alias_chk = self._fmt_modalias_check(alias) + dc_err = self._supported_sig_errs(supported, is_signed) dv_license = row["Driver Licenses"] lcs_chk = self._fmt_driver_license_check( license, dv_license, vld_lic @@ -490,7 +528,6 @@ def _get_table_detail_html(self, rpm_table): else: ts = td(signature) ts.set_attribute("class", "important_failed") - # td(distribution) if lcs_chk == "": # license check if license == "": tl = td("No License") @@ -525,10 +562,17 @@ def _get_table_detail_html(self, rpm_table): t_w = td(raw(sym_check)) t_w.set_attribute("class", "critical_failed") r.set_attribute("class", "critical_failed_row") + if alias_chk == "": + t = td("All passed!") + t.set_attribute("class", "detail_pass") + else: + t_w = td(raw(alias_chk.replace("\n", "
"))) + t_w.set_attribute("class", "important_failed") + r.set_attribute("class", "important_failed_row") return tb - def _combine_driver_check_errs(self, sffds, nsds): + def _supported_sig_errs(self, sffds, nsds): sfds_l = self._get_supported_driver_failed(sffds) nsds_l = self._get_no_signed_driver(nsds) @@ -658,7 +702,16 @@ def _xlsx_create_vendor_summary(self, wb, rpm_table): sym_failed = Rule(type="expression", dxf=ctc_style) sym_failed.formula = ['VALUE(LEFT($G2, FIND(" ", $G2) - 1)) <> 0'] ws_vs.conditional_formatting.add(f"G2:G{last_record_row_no}", sym_failed) - + + ws_vs.column_dimensions['A'].width = 30 + ws_vs.column_dimensions['B'].width = 15 + ws_vs.column_dimensions['C'].width = 15 + ws_vs.column_dimensions['D'].width = 15 + ws_vs.column_dimensions['E'].width = 15 + ws_vs.column_dimensions['F'].width = 15 + ws_vs.column_dimensions['G'].width = 15 + ws_vs.column_dimensions['H'].width = 15 + def _xlsx_create_rpm_details(self, wb, rpm_table): df = self._rename_rpm_detail_columns(rpm_table) ws_rd = wb.create_sheet("KMPs details") @@ -695,7 +748,7 @@ def _xlsx_create_rpm_details(self, wb, rpm_table): ws_rd['A1'].border = header_border ws_rd['A1'].alignment = center_align - ws_rd.merge_cells('H1:I1') + ws_rd.merge_cells('H1:L1') ws_rd['H1'] = "Kernel Module Checks" ws_rd['H1'].font = header_font ws_rd['H1'].fill = header_fill @@ -741,8 +794,8 @@ def _xlsx_create_rpm_details(self, wb, rpm_table): ws_rd[cell_no].font = ctc_font ws_rd[cell_no].fill = ctc_fill ws_rd[cell_no].border = ctc_border - elif cols[col_idx] == "Driver Checks": - val = self._combine_driver_check_errs(val, row["is-signed"]) + elif cols[col_idx] == "Supported Flag Check": + val = self._supported_sig_errs(val, row["is-signed"]) if len(val) > 0: ws_rd[cell_no] = '\n'.join(val) ws_rd[cell_no].font = ctc_font @@ -769,6 +822,18 @@ def _xlsx_create_rpm_details(self, wb, rpm_table): ws_rd[cell_no].font = imt_font ws_rd[cell_no].fill = imt_fill ws_rd[cell_no].border = imt_border + elif cols[col_idx] == "Modalias Check": + alias_check = self._fmt_modalias_check(row["Modalias Check"]) + if alias_check != "": + ws_rd[cell_no] = alias_check + ws_rd[cell_no].font = imt_font + ws_rd[cell_no].fill = imt_fill + ws_rd[cell_no].border = imt_border + else: + val = "All passed!" + ws_rd[cell_no] = val + ws_rd[cell_no].alignment = center_align + else: # no format needed. ws_rd[cell_no] = str(val) @@ -783,6 +848,16 @@ def _xlsx_create_rpm_details(self, wb, rpm_table): sig_rule = Rule(type="expression", dxf=ctc_style) sig_rule.formula = ['=OR($D2 = "", $D2 = "(none)")'] ws_rd.conditional_formatting.add(f"D2:D{records}", sig_rule) + + ws_rd.column_dimensions['A'].width = 30 + ws_rd.column_dimensions['B'].width = 40 + ws_rd.column_dimensions['C'].width = 30 + ws_rd.column_dimensions['D'].width = 30 + ws_rd.column_dimensions['F'].width = 20 + ws_rd.column_dimensions['G'].width = 15 + ws_rd.column_dimensions['H'].width = 40 + ws_rd.column_dimensions['I'].width = 40 + ws_rd.column_dimensions['L'].width = 40 def _xlsx_create_report_workbook(self): wb = Workbook() diff --git a/src/soliddriver_checks/utils/data_reader.py b/src/soliddriver_checks/utils/data_reader.py index e701ea4..656eb55 100644 --- a/src/soliddriver_checks/utils/data_reader.py +++ b/src/soliddriver_checks/utils/data_reader.py @@ -7,6 +7,7 @@ import re from collections import namedtuple import tempfile +import fnmatch import json from scp import SCPClient from paramiko.ssh_exception import NoValidConnectionsError, SSHException @@ -90,8 +91,35 @@ def __init__(self, progress): "sym-check", "dv-licenses", "is-signed", + "modalias", ] + def _get_driver_alias(self, driver): + alias = run_cmd("/usr/sbin/modinfo --field=alias %s | grep pci:" % driver) + return alias.splitlines() + + def _get_rpm_modalias(self, rpm): + modalias = namedtuple("modalias", "kernel_flavor pci_re") + ml_pci_re = re.compile(r"modalias\((.*):(.*\:.*)\)") # example: modalias(kernel-default:pci:v000019A2d00000712sv*sd*bc*sc*i*) + ml_all_re = re.compile(r"modalias\((.*):(.*)\)") # example: packageand(kernel-default:primergy-be2iscsi) + raw_modalias = run_cmd("rpm -q --supplements %s" %rpm) + + alias_re = [] + for line in raw_modalias.splitlines(): + line = str(line, "utf-8").strip() + pci_rst = ml_pci_re.match(line) + all_rst = ml_all_re.match(line) + if pci_rst: + ker_flavor, pci = pci_rst.groups() + if "pci:" in pci: # only check PCI devices + alias_re.append(pci) + elif all_rst: # match all (*) should not be allowed + ker_flavor, rst = all_rst.groups() + if rst == "*": + alias_re.append(rst) + + return alias_re + def _driver_symbols_check(self, rpm_symbols, driver): symvers = run_cmd("/usr/sbin/modprobe --dump-modversions %s" % driver) @@ -147,7 +175,7 @@ def _get_driver_supported(self, driver): if values[0].strip() == "supported": if supported != "": # only allow appears once. - return "Multiple" + supported = supported + ", " + ":".join(values[1:]).strip() else: supported = ":".join(values[1:]).strip() @@ -203,6 +231,50 @@ def _fmt_driver_symbol(self, drivers): symbols[d] = d_info return symbols + + def _fmt_driver_modalias(self, kmp_alias, drivers): + key_match_all = 'match_all' + key_unmatched_km_alias = 'unmatched_km_alias' + key_unmatched_kmp_alias = 'unmatched_kmp_alias' + for a in kmp_alias: + if a == "*": # "use default-kernel:* to match all the devices is always a bad idea." + return {key_match_all:True, + key_unmatched_km_alias:[], + key_unmatched_kmp_alias:[]} + + unmatched_ker_alias = [] + unmatched_kmp_alias = kmp_alias.copy() + + for d in drivers: + for ker_a in drivers[d]["alias"]: + ker_a = str(ker_a, "utf-8").strip() + found = False + for kmp_a in kmp_alias: + if fnmatch.fnmatch(ker_a.strip(), kmp_a.strip()): + found = True + for uk in unmatched_kmp_alias: + if uk.strip() == kmp_a.strip(): + unmatched_kmp_alias.remove(uk) + break + break + if not found: + unmatched_ker_alias.append(ker_a) + + # There has some packages have "%5" in the package but use "_" in the kernel module + # So we have to match it again, but equal is enough. + r_ukmpalias = [v.replace("_", "%5F").replace("-", "%2D").replace(".", "%2E") for v in unmatched_ker_alias] + for i in range(len(unmatched_kmp_alias) - 1, -1, -1): + found_match = False + for j in range(len(unmatched_ker_alias) - 1, -1, -1): + if r_ukmpalias[j] == unmatched_kmp_alias[i]: + unmatched_ker_alias.pop(j) + found_match = True + if found_match: + unmatched_kmp_alias.pop(i) + + return {key_match_all:False, + key_unmatched_km_alias:unmatched_ker_alias, + key_unmatched_kmp_alias:unmatched_kmp_alias} def _is_driver_signed(self, driver): raw_info = run_cmd("/usr/sbin/modinfo %s" % driver) @@ -242,6 +314,7 @@ def _driver_checks(self, rpm: str): item["supported"] = self._get_driver_supported(driver) item["license"] = self._get_driver_license(driver) item["is_signed"] = self._is_driver_signed(driver) + item["alias"] = self._get_driver_alias(driver) dpath = str(driver) dpath = dpath[dpath.startswith(tmp.name) + len(tmp.name) - 1 :] @@ -293,11 +366,13 @@ def _format_rpm_info(self, rpm_files, raw_output, row_handlers, query="all"): symbols = dict() d_licenses = dict() is_signed = dict() + km_modalias = dict() if driver_checks is not None: supported = self._fmt_driver_supported(driver_checks) symbols = self._fmt_driver_symbol(driver_checks) d_licenses = self._fmt_driver_license(driver_checks) is_signed = self._fmt_driver_is_signed(driver_checks) + km_modalias = self._fmt_driver_modalias(self._get_rpm_modalias(rpm), driver_checks) if not self._query_filter(supported, query): continue @@ -315,7 +390,8 @@ def _format_rpm_info(self, rpm_files, raw_output, row_handlers, query="all"): supported, symbols, d_licenses, - is_signed + is_signed, + km_modalias ] ) diff --git a/src/soliddriver_checks/version.py b/src/soliddriver_checks/version.py index af218a2..1847ab2 100644 --- a/src/soliddriver_checks/version.py +++ b/src/soliddriver_checks/version.py @@ -2,4 +2,4 @@ Global version information used in soliddriver-checks and the package """ -__VERSION__ = "2.0.7" +__VERSION__ = "2.0.8-beta" diff --git a/tests/Dockerfile b/tests/Dockerfile index 48faff1..49212b8 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -2,8 +2,9 @@ FROM opensuse/leap:latest WORKDIR /root -RUN zypper --non-interactive in python3-pip kmod +RUN zypper --non-interactive in python39 python39-pip kmod +RUN zypper --non-interactive dup -RUN pip3 install rich pdfkit pandas \ -openpyxl click paramiko dominate +RUN pip3.9 install rich pdfkit pandas \ +openpyxl click paramiko dominate build diff --git a/tests/create_container_mac.sh b/tests/create_container_mac.sh index 5e49fb2..afbc0af 100755 --- a/tests/create_container_mac.sh +++ b/tests/create_container_mac.sh @@ -1,6 +1,6 @@ docker run \ -it --rm \ --mount type=bind,source=/Users/$USER/projects/github.com/SUSE/soliddriver-checks,target=/root/source_codes \ ---mount type=bind,source=/Users/$USER/projects/Lenovo/SSDP/01-Apr-2022,target=/root/rpms \ +--mount type=bind,source=/Users/$USER/projects/fujitsu/kmps-15-sp2,target=/root/rpms \ --mount type=bind,source=/Users/$USER/codes/tmp,target=/root/output_dir \ opensuse-leap-soliddriver-checks:latest /bin/bash