Wip/hughsie/ifd

RELEASE

@@ -9,6 +9,10 @@ When forking main into a stable 2_1_X, be sure to disable the following CI jobs:
 Also update `SECURITY.md`, removing the oldest branch and add the new branch at the top.
 To make sure it's done right, you can reference commit 433e809318c68c9ab6d4ae50ee9c4312503185d8
 
+Check IFD parsing (if the files are available):
+
+    ../contrib/check-ifd-firmware.py ../../fwupd-test-roms/
+
 Write release entries:
 
  * ../contrib/generate-release.py

contrib/check-ifd-firmware.py

@@ -0,0 +1,104 @@
+#!/usr/bin/python3
+#
+# Copyright (C) 2024 Richard Hughes <richard@hughsie.com>
+#
+# SPDX-License-Identifier: LGPL-2.1+
+#
+# pylint: disable=invalid-name,missing-docstring,too-few-public-methods
+
+from typing import Dict
+import os
+import glob
+import sys
+import subprocess
+import json
+import argparse
+from collections import defaultdict
+from termcolor import colored
+
+
+def _scan_file(fn: str) -> Dict[str, int]:
+    new_map: Dict[str, int] = defaultdict(int)
+    try:
+        print(f"loading {fn}…")
+        args = [
+            "./src/fwupdtool",
+            "firmware-parse",
+            fn,
+            "ifd-firmware",
+            "--json",
+            "--no-timestamp",
+        ]
+        p = subprocess.run(args, check=True, capture_output=True)
+    except subprocess.CalledProcessError as e:
+        print(f"{' '.join(args)}: {e}")
+    else:
+        for line in p.stdout.decode().split("\n"):
+            new_map["Lines"] += 1
+            if line.find("gtype=") == -1:
+                continue
+            sections = line.split('"')
+            if not sections[1].startswith("Fu"):
+                continue
+            new_map[sections[1]] += 1
+        for line in p.stderr.decode().split("\n"):
+            if not line:
+                continue
+            new_map["WarningLines"] += 1
+            print(line)
+    return new_map
+
+
+def _scan_dir(path: str, force_save: bool = False) -> bool:
+    all_okay: bool = True
+    needs_save: bool = False
+
+    results: Dict[str, Dict[str, int]] = {}
+    for fn in glob.glob(f"{path}/*.bin"):
+        results[fn] = _scan_file(fn)
+
+    # go through each result
+    print(f"    {os.path.basename(sys.argv[0])}:")
+    for fn, new_map in results.items():
+        try:
+            with open(f"{fn}.json", "rb") as f:
+                old_map = json.loads(f.read().decode())
+        except FileNotFoundError:
+            old_map = {}
+        print(f"    {fn}")
+        for key in sorted(set(list(old_map.keys()) + list(new_map.keys()))):
+            cnt_old = old_map.get(key, 0)
+            cnt_new = new_map.get(key, 0)
+            if cnt_new > cnt_old:
+                key_str: str = colored(key, "green")
+                needs_save = True
+            elif cnt_new < cnt_old:
+                key_str = colored(key, "red")
+                if key != "WarningLines":
+                    all_okay = False
+            else:
+                continue
+            print(f"    {key_str:36}: {cnt_old} -> {cnt_new}")
+
+    # save new results if all better
+    if (needs_save and all_okay) or force_save:
+        for fn, new_map in results.items():
+            with open(f"{fn}.json", "wb") as f:
+                f.write(json.dumps(new_map, sort_keys=True, indent=4).encode())
+
+    return all_okay
+
+
+if __name__ == "__main__":
+    rc: int = 0
+
+    parser = argparse.ArgumentParser(
+        prog="check-ifd-firmware", description="Check IFD firmware parsing"
+    )
+    parser.add_argument("paths", nargs="+")
+    parser.add_argument("force_save", action="store_true")
+    _args = parser.parse_args()
+    for _path in _args.paths:
+        if not _scan_dir(_path, force_save=_args.force_save):
+            rc = 1
+    sys.exit(rc)

contrib/ci/oss-fuzz.py

@@ -196,7 +196,7 @@ def mkfuzztargets(self, globstr: str) -> None:
         builder_xmls = glob.glob(globstr)
         if not builder_xmls:
             print(f"failed to find {globstr}")
-            sys.exit(1)
+            return
         for fn_src in builder_xmls:
             fn_dst = fn_src.replace(".builder.xml", ".bin")
             if os.path.exists(fn_dst):
@@ -379,6 +379,7 @@ def _build(bld: Builder) -> None:
 
     # built in formats
     for fzr in [
+        Fuzzer("efi-lz77", pattern="efi-lz77-decompressor"),
         Fuzzer("csv"),
         Fuzzer("cab"),
         Fuzzer("dfuse"),

contrib/dump-mtd-ifd.py

@@ -0,0 +1,185 @@
+#!/usr/bin/python3
+#
+# Copyright (C) 2024 Richard Hughes <richard@hughsie.com>
+#
+# SPDX-License-Identifier: LGPL-2.1+
+#
+# pylint: disable=invalid-name,missing-docstring,too-few-public-methods,too-many-locals
+
+import sys
+import os
+import struct
+import io
+from typing import List, Optional
+import argparse
+
+
+class IfdPartition:
+    REGION_NAMES = [
+        "desc",
+        "bios",
+        "me",
+        "gbe",
+        "platform",
+        "devexp",
+        "bios2",
+        "ec",
+        "ie",
+        "10gbe",
+    ]
+
+    def __init__(self, region: int = 0, offset: int = 0, size: int = 0) -> None:
+        self.region: int = region
+        self.offset: int = offset
+        self.size: int = size
+
+    def __str__(self) -> str:
+        try:
+            region_name: str = IfdPartition.REGION_NAMES[self.region]
+        except IndexError:
+            region_name = "unknown"
+        return (
+            "IfdPartition("
+            f"region=0x{self.region} ({region_name}), "
+            f"offset=0x{self.offset:x}, "
+            f"size=0x{self.size:x})"
+        )
+
+
+def _read_partitions(f: io.BufferedReader) -> bytearray:
+    f.seek(0)
+    blob_ifd: bytes = f.read(0x1000)
+    (
+        signature,
+        descriptor_map0,
+        descriptor_map1,
+        descriptor_map2,
+    ) = struct.unpack_from("<16xIIII", blob_ifd, offset=0)
+    if signature != 0x0FF0A55A:
+        sys.exit(f"Not IFD signature 0x0FF0A55A, got 0x{signature:X}")
+
+    # read out the descriptor maps
+    print(f"descriptor_map0=0x{descriptor_map0:X}")
+    print(f"descriptor_map1=0x{descriptor_map1:X}")
+    print(f"descriptor_map2=0x{descriptor_map2:X}")
+
+    # default to all partitions
+    num_regions = (descriptor_map0 >> 24) & 0b111
+    if num_regions == 0:
+        num_regions = 10
+    print(f"num_regions=0x{num_regions:X}")
+
+    # read out FREGs
+    flash_region_base_addr = (descriptor_map0 >> 12) & 0x00000FF0
+    print(f"flash_region_base_addr=0x{flash_region_base_addr:X}")
+    flash_descriptor_regs: List[int] = []
+    for region in range(num_regions):
+        flash_descriptor_regs.append(
+            struct.unpack_from(
+                "<I", blob_ifd, offset=flash_region_base_addr + (region * 4)
+            )[0]
+        )
+    for region in range(num_regions):
+        print(f"flash_descriptor_reg{region}=0x{flash_descriptor_regs[region]:X}")
+
+    # parse each partition
+    fregs: List[IfdPartition] = []
+    for i in range(num_regions):
+        freg_base: int = (flash_descriptor_regs[i] << 12) & 0x07FFF000
+        freg_limit: int = ((flash_descriptor_regs[i] >> 4) & 0x07FFF000) | 0x00000FFF
+
+        # invalid
+        if freg_base > freg_limit:
+            continue
+
+        fregs.append(
+            IfdPartition(region=i, offset=freg_base, size=freg_limit - freg_base)
+        )
+
+    # create a binary blob big enough
+    image_size: int = 0
+    for freg in fregs:
+        if freg.offset + freg.size > image_size:
+            image_size = freg.offset + freg.size
+    print(f"image_size=0x{image_size:x}")
+    blob: bytearray = bytearray(image_size)
+
+    # copy each partition
+    for freg in fregs:
+        print("reading...", freg)
+        try:
+            f.seek(freg.offset)
+            blob[freg.offset : freg.offset + freg.size] = f.read(freg.size)
+        except OSError as e:
+            print(f"failed to read: {e}")
+    return blob
+
+
+def _read_device_to_file(devname: str, filename: Optional[str]) -> None:
+    # grab system info from sysfs
+    if not filename:
+        filename = ""
+        for sysfs_fn in [
+            "/sys/class/dmi/id/sys_vendor",
+            "/sys/class/dmi/id/product_family",
+            "/sys/class/dmi/id/product_name",
+            "/sys/class/dmi/id/product_sku",
+        ]:
+            try:
+                with open(sysfs_fn, "rb") as f:
+                    if filename:
+                        filename += "-"
+                    filename += (
+                        f.read().decode().replace("\n", "").replace(" ", "_").lower()
+                    )
+            except FileNotFoundError:
+                pass
+        if filename:
+            filename += ".bin"
+        else:
+            filename = "bios.bin"
+
+    # check this device name is what we expect
+    print(f"checking {devname}...")
+    try:
+        with open(f"/sys/class/mtd/{os.path.basename(devname)}/name", "rb") as f_name:
+            name = f_name.read().decode().replace("\n", "")
+    except FileNotFoundError as e:
+        sys.exit(str(e))
+    if name != "BIOS":
+        sys.exit(f"Not Intel Corporation PCH SPI Controller, got {name}")
+
+    # read the IFD header, then each partition
+    try:
+        with open(devname, "rb") as f_in:
+            print(f"reading from {devname}...")
+            blob = _read_partitions(f_in)
+    except PermissionError as e:
+        sys.exit(f"cannot read mtd device: {e}")
+    print(f"writing {filename}...")
+    with open(filename, "wb") as f_out:
+        f_out.write(blob)
+    print("done!")
+
+
+if __name__ == "__main__":
+    # both have defaults
+    parser = argparse.ArgumentParser(
+        prog="dump-mtd-ifd", description="Dump local SPI contents using MTD"
+    )
+    parser.add_argument(
+        "--filename",
+        action="store",
+        type=str,
+        help="Output filename",
+        default=None,
+    )
+    parser.add_argument(
+        "--devname",
+        action="store",
+        type=str,
+        help="Device name, e.g. /dev/mtd0",
+        default="/dev/mtd0",
+    )
+    args = parser.parse_args()
+    _read_device_to_file(args.devname, args.filename)

libfwupdplugin/fu-efi-firmware-common.c

@@ -30,10 +30,10 @@
 gboolean
 fu_efi_firmware_parse_sections(FuFirmware *firmware,
 			       GInputStream *stream,
+			       gsize offset,
 			       FwupdInstallFlags flags,
 			       GError **error)
 {
-	gsize offset = 0;
 	gsize streamsz = 0;
 
 	if (!fu_input_stream_size(stream, &streamsz, error))

libfwupdplugin/fu-efi-firmware-common.h

@@ -11,5 +11,6 @@
 gboolean
 fu_efi_firmware_parse_sections(FuFirmware *firmware,
 			       GInputStream *stream,
+			       gsize offset,
 			       FwupdInstallFlags flags,
 			       GError **error) G_GNUC_NON_NULL(1, 2);

libfwupdplugin/fu-efi-firmware-file.c

@@ -143,17 +143,6 @@ fu_efi_firmware_file_parse(FuFirmware *firmware,
 	/* add simple blob */
 	partial_stream = fu_partial_input_stream_new(stream, st->len, size - st->len);
 
-	/* add fv-image */
-	if (priv->type == FU_EFI_FILE_TYPE_FIRMWARE_VOLUME_IMAGE) {
-		if (!fu_efi_firmware_parse_sections(firmware, partial_stream, flags, error)) {
-			g_prefix_error(error, "failed to add firmware image: ");
-			return FALSE;
-		}
-	} else {
-		if (!fu_firmware_set_stream(firmware, partial_stream, error))
-			return FALSE;
-	}
-
 	/* verify data checksum */
 	if ((priv->attrib & FU_EFI_FILE_ATTRIB_CHECKSUM) > 0 &&
 	    (flags & FWUPD_INSTALL_FLAG_IGNORE_CHECKSUM) == 0) {
@@ -171,6 +160,17 @@ fu_efi_firmware_file_parse(FuFirmware *firmware,
 		}
 	}
 
+	/* add sections */
+	if (priv->type != FU_EFI_FILE_TYPE_FFS_PAD && priv->type != FU_EFI_FILE_TYPE_RAW) {
+		if (!fu_efi_firmware_parse_sections(firmware, partial_stream, 0, flags, error)) {
+			g_prefix_error(error, "failed to add firmware image: ");
+			return FALSE;
+		}
+	} else {
+		if (!fu_firmware_set_stream(firmware, partial_stream, error))
+			return FALSE;
+	}
+
 	/* align size for volume */
 	fu_firmware_set_size(firmware,
 			     fu_common_align_up(size, fu_firmware_get_alignment(firmware)));

libfwupdplugin/fu-efi-firmware-filesystem.c

@@ -36,7 +36,7 @@ fu_efi_firmware_filesystem_parse(FuFirmware *firmware,
 	gsize streamsz = 0;
 	if (!fu_input_stream_size(stream, &streamsz, error))
 		return FALSE;
-	while (offset + 0x18 < streamsz) {
+	while (offset < streamsz) {
 		g_autoptr(FuFirmware) img = fu_efi_firmware_file_new();
 		g_autoptr(GInputStream) stream_tmp = NULL;
 		gboolean is_freespace = TRUE;
@@ -51,9 +51,12 @@ fu_efi_firmware_filesystem_parse(FuFirmware *firmware,
 				break;
 			}
 		}
-		if (is_freespace)
+		if (is_freespace) {
+			g_debug("ignoring free space @0x%x of 0x%x",
+				(guint)offset,
+				(guint)streamsz);
 			break;
-
+		}
 		stream_tmp = fu_partial_input_stream_new(stream, offset, streamsz - offset);
 		if (!fu_firmware_parse_stream(img,
 					      stream_tmp,