From 2d88408c8dd2a5f0e9f0f1f5e46e1cae22b86860 Mon Sep 17 00:00:00 2001 From: Maksim An Date: Mon, 6 May 2024 16:52:58 -0700 Subject: [PATCH] verity-boot: append hash device to rootfs Turned out that dev nodes for SCSI devices may not be determenistic and the hash device and rootfs may end up appearing under /dev/sda and /dev/sdb respectively. Instead of mounting a separate hash device, append the verity Merkle tree to rootfs ext4 filesystem, similarly to how it's done for layer VHDs. Update the IVGM kernel init to reflect the changes. TODO: remove the code that assumes hash device in a separate VHD. Signed-off-by: Maksim An --- Makefile.bootfiles | 33 +++++++++++++++++------ internal/uvm/create_lcow.go | 54 ++++++++++++++++++++----------------- 2 files changed, 54 insertions(+), 33 deletions(-) diff --git a/Makefile.bootfiles b/Makefile.bootfiles index 5fb92f3414..72ff78f388 100644 --- a/Makefile.bootfiles +++ b/Makefile.bootfiles @@ -1,5 +1,6 @@ BASE:=base.tar.gz DEV_BUILD:=0 +VERITY_APPEND:=0 DELTA_TARGET=out/delta.tar.gz @@ -20,6 +21,16 @@ IGVM_TOOL:= KERNEL_PATH:= TAR2EXT4_TOOL:=bin/cmd/tar2ext4 +ROOTFS_DEVICE:=/dev/sda +VERITY_DEVICE:=/dev/sdb +ROOTFS_VHD:=out/rootfs.vhd +HASH_DEVICE_VHD:=out/rootfs.hash.vhd +ifeq "$(VERITY_APPEND)" "1" +VERITY_DEVICE:=/dev/sda +ROOTFS_VHD=out/rootfs.verity.vhd +HASH_DEVICE_VHD= +endif + .PHONY: all always rootfs test snp simple .DEFAULT_GOAL := all @@ -32,7 +43,7 @@ clean: rootfs: out/rootfs.vhd -snp: out/kernelinitrd.vmgs out/rootfs.hash.vhd out/rootfs.vhd out/v2056.vmgs +snp: out/kernelinitrd.vmgs $(ROOTFS_VHD) $(HASH_DEVICE_VHD) out/v2056.vmgs simple: out/simple.vmgs snp @@ -52,23 +63,21 @@ out/simple.bin: out/initrd.img $(PATH_PREFIX)/$(KERNEL_PATH) boot/startup_simple -rdinit out/initrd.img \ -vtl 0 -ROOTFS_DEVICE:=/dev/sda -VERITY_DEVICE:=/dev/sdb # Debug build for use with uvmtester. UVM with dm-verity protected vhd disk mounted directly via the kernel command line. Ignores corruption in dm-verity protected disk. (Use dmesg to see if dm-verity is ignoring data corruption.) -out/v2056.bin: out/rootfs.vhd out/rootfs.hash.vhd $(PATH_PREFIX)/$(KERNEL_PATH) out/rootfs.hash.datasectors out/rootfs.hash.datablocksize out/rootfs.hash.hashblocksize out/rootfs.hash.datablocks out/rootfs.hash.rootdigest out/rootfs.hash.salt boot/startup_v2056.sh +out/v2056.bin: $(ROOTFS_VHD) $(HASH_DEVICE_VHD) $(PATH_PREFIX)/$(KERNEL_PATH) out/rootfs.hash.datasectors out/rootfs.hash.datablocksize out/rootfs.hash.hashblocksize out/rootfs.hash.datablocks out/rootfs.hash.rootdigest out/rootfs.hash.deviceoffset out/rootfs.hash.salt boot/startup_v2056.sh rm -f $@ python3 $(PATH_PREFIX)/$(IGVM_TOOL) \ -o $@ -kernel $(PATH_PREFIX)/$(KERNEL_PATH) \ - -append "8250_core.nr_uarts=0 panic=-1 debug loglevel=7 root=/dev/dm-0 dm-mod.create=\"dmverity,,,ro,0 $(shell cat out/rootfs.hash.datasectors) verity 1 $(ROOTFS_DEVICE) $(VERITY_DEVICE) $(shell cat out/rootfs.hash.datablocksize) $(shell cat out/rootfs.hash.hashblocksize) $(shell cat out/rootfs.hash.datablocks) 0 sha256 $(shell cat out/rootfs.hash.rootdigest) $(shell cat out/rootfs.hash.salt) 1 ignore_corruption\" init=/startup_v2056.sh" \ + -append "8250_core.nr_uarts=0 panic=-1 debug loglevel=7 root=/dev/dm-0 dm-mod.create=\"dmverity,,,ro,0 $(shell cat out/rootfs.hash.datasectors) verity 1 $(ROOTFS_DEVICE) $(VERITY_DEVICE) $(shell cat out/rootfs.hash.datablocksize) $(shell cat out/rootfs.hash.hashblocksize) $(shell cat out/rootfs.hash.datablocks) $(shell cat out/rootfs.hash.deviceoffset) sha256 $(shell cat out/rootfs.hash.rootdigest) $(shell cat out/rootfs.hash.salt) 1 ignore_corruption\" init=/startup_v2056.sh" \ -vtl 0 # Full UVM with dm-verity protected vhd disk mounted directly via the kernel command line. -out/kernelinitrd.bin: out/rootfs.vhd out/rootfs.hash.vhd out/rootfs.hash.datasectors out/rootfs.hash.datablocksize out/rootfs.hash.hashblocksize out/rootfs.hash.datablocks out/rootfs.hash.rootdigest out/rootfs.hash.salt $(PATH_PREFIX)/$(KERNEL_PATH) boot/startup.sh +out/kernelinitrd.bin: $(ROOTFS_VHD) $(HASH_DEVICE_VHD) $(PATH_PREFIX)/$(KERNEL_PATH) out/rootfs.hash.datasectors out/rootfs.hash.datablocksize out/rootfs.hash.hashblocksize out/rootfs.hash.datablocks out/rootfs.hash.rootdigest out/rootfs.hash.deviceoffset out/rootfs.hash.salt boot/startup.sh rm -f $@ python3 $(PATH_PREFIX)/$(IGVM_TOOL) \ -o $@ \ -kernel $(PATH_PREFIX)/$(KERNEL_PATH) \ - -append "8250_core.nr_uarts=0 panic=-1 debug loglevel=7 root=/dev/dm-0 dm-mod.create=\"dmverity,,,ro,0 $(shell cat out/rootfs.hash.datasectors) verity 1 $(ROOTFS_DEVICE) $(VERITY_DEVICE) $(shell cat out/rootfs.hash.datablocksize) $(shell cat out/rootfs.hash.hashblocksize) $(shell cat out/rootfs.hash.datablocks) 0 sha256 $(shell cat out/rootfs.hash.rootdigest) $(shell cat out/rootfs.hash.salt)\" init=/startup.sh" \ + -append "8250_core.nr_uarts=0 panic=-1 debug loglevel=7 root=/dev/dm-0 dm-mod.create=\"dmverity,,,ro,0 $(shell cat out/rootfs.hash.datasectors) verity 1 $(ROOTFS_DEVICE) $(VERITY_DEVICE) $(shell cat out/rootfs.hash.datablocksize) $(shell cat out/rootfs.hash.hashblocksize) $(shell cat out/rootfs.hash.datablocks) $(shell cat out/rootfs.hash.deviceoffset) sha256 $(shell cat out/rootfs.hash.rootdigest) $(shell cat out/rootfs.hash.salt)\" init=/startup.sh" \ -vtl 0 # Rule to make a vhd from a file. This is used to create the rootfs.hash.vhd from rootfs.hash. @@ -79,7 +88,7 @@ out/kernelinitrd.bin: out/rootfs.vhd out/rootfs.hash.vhd out/rootfs.hash.datasec %.vhd: %.ext4 $(TAR2EXT4_TOOL) $(TAR2EXT4_TOOL) -only-vhd -i $< -o $@ -%.hash %.hash.info %.hash.datablocks %.hash.rootdigest %hash.datablocksize %.hash.datasectors %.hash.hashblocksize: %.ext4 %.hash.salt +%.hash %.hash.info %.hash.datablocks %.hash.rootdigest %hash.datablocksize %.hash.datasectors %.hash.hashblocksize %.hash.deviceoffset: %.ext4 %.hash.salt veritysetup format --no-superblock --salt $(shell cat out/rootfs.hash.salt) $< $*.hash > $*.hash.info # Retrieve info required by dm-verity at boot time # Get the blocksize of rootfs @@ -89,6 +98,10 @@ out/kernelinitrd.bin: out/rootfs.vhd out/rootfs.hash.vhd out/rootfs.hash.datasec cat $*.hash.info | awk '/^Hash block size:/{ print $$4 }' > $*.hash.hashblocksize cat $*.hash.info | awk '/^Data blocks:/{ print $$3 }' > $*.hash.datablocks echo $$(( $$(cat $*.hash.datablocks) * $$(cat $*.hash.datablocksize) / 512 )) > $*.hash.datasectors + echo "0" > $*.hash.deviceoffset + @if [ "$(VERITY_APPEND)" = "1" ]; then \ + cat $*.hash.datablocks > $*.hash.deviceoffset; \ + fi out/rootfs.hash.salt: hexdump -vn32 -e'8/4 "%08X" 1 "\n"' /dev/random > $@ @@ -97,6 +110,10 @@ out/rootfs.ext4: out/rootfs.tar.gz $(TAR2EXT4_TOOL) gzip -f -d ./out/rootfs.tar.gz $(TAR2EXT4_TOOL) -i ./out/rootfs.tar -o $@ +out/rootfs.verity: out/rootfs.ext4 out/rootfs.hash + cp out/rootfs.ext4 $@ + cat out/rootfs.hash >> $@ + out/rootfs.tar.gz: out/initrd.img rm -rf rootfs-conv mkdir rootfs-conv diff --git a/internal/uvm/create_lcow.go b/internal/uvm/create_lcow.go index ced3518dbd..d7cf1f5492 100644 --- a/internal/uvm/create_lcow.go +++ b/internal/uvm/create_lcow.go @@ -69,7 +69,7 @@ const ( InitrdFile = "initrd.img" // VhdFile is the default file name for a rootfs.vhd used to boot LCOW. VhdFile = "rootfs.vhd" - // DmVerityVhdFile is the default file name for a dmverity_rootfs.vhd which + // DefaultDmVerityRootfsVhd is the default file name for a dmverity_rootfs.vhd which // is mounted by the GuestStateFile during boot and used as the root file // system when booting in the SNP case. DefaultDmVerityRootfsVhd = "rootfs.vhd" @@ -163,7 +163,7 @@ func NewDefaultOptionsLCOW(id, owner string) *OptionsLCOW { RootFSFile: InitrdFile, KernelBootOptions: "", EnableGraphicsConsole: false, - ConsolePipe: "", + ConsolePipe: "\\\\.\\pipe\\vmpipe", UseGuestConnection: true, ExecCommandLine: fmt.Sprintf("/bin/gcs -v4 -log-format json -loglevel %s", logrus.StandardLogger().Level.String()), ForwardStdout: false, @@ -362,11 +362,11 @@ func makeLCOWVMGSDoc(ctx context.Context, opts *OptionsLCOW, uvm *UtilityVM) (_ return nil, fmt.Errorf("the DM Verity VHD file '%s' was not found", dmVerityRootfsTemplatePath) } - // The root file system comes from the dmverity vhd file which is mounted by the initrd in the vmgs file. - dmVerityHashTemplatePath := filepath.Join(opts.BootFilesPath, opts.DmVerityHashVhd) - if _, err := os.Stat(dmVerityHashTemplatePath); os.IsNotExist(err) { - return nil, fmt.Errorf("the DM Verity Hash file '%s' was not found", dmVerityHashTemplatePath) - } + //// The root file system comes from the dmverity vhd file which is mounted by the initrd in the vmgs file. + //dmVerityHashTemplatePath := filepath.Join(opts.BootFilesPath, opts.DmVerityHashVhd) + //if _, err := os.Stat(dmVerityHashTemplatePath); os.IsNotExist(err) { + // return nil, fmt.Errorf("the DM Verity Hash file '%s' was not found", dmVerityHashTemplatePath) + //} var processor *hcsschema.Processor2 processor, err = fetchProcessor(ctx, opts, uvm) @@ -394,17 +394,21 @@ func makeLCOWVMGSDoc(ctx context.Context, opts *OptionsLCOW, uvm *UtilityVM) (_ } }() - dmVerityHashFullPath := filepath.Join(opts.BundleDirectory, DefaultDmVerityHashVhd) - if err := copyfile.CopyFile(ctx, dmVerityHashTemplatePath, dmVerityHashFullPath, true); err != nil { - return nil, fmt.Errorf("failed to copy DM Verity hash template file: %w", err) - } - defer func() { - if err != nil { - os.Remove(dmVerityHashFullPath) - } - }() - - for _, filename := range []string{vmgsFileFullPath, dmVerityRootFsFullPath, dmVerityHashFullPath} { + //dmVerityHashFullPath := filepath.Join(opts.BundleDirectory, DefaultDmVerityHashVhd) + //if err := copyfile.CopyFile(ctx, dmVerityHashTemplatePath, dmVerityHashFullPath, true); err != nil { + // return nil, fmt.Errorf("failed to copy DM Verity hash template file: %w", err) + //} + //defer func() { + // if err != nil { + // os.Remove(dmVerityHashFullPath) + // } + //}() + + for _, filename := range []string{ + vmgsFileFullPath, + dmVerityRootFsFullPath, + //dmVerityHashFullPath, + } { if err := security.GrantVmGroupAccessWithMask(filename, security.AccessMaskAll); err != nil { return nil, fmt.Errorf("failed to grant VM group access ALL: %w", err) } @@ -491,16 +495,16 @@ func makeLCOWVMGSDoc(ctx context.Context, opts *OptionsLCOW, uvm *UtilityVM) (_ Path: dmVerityRootFsFullPath, ReadOnly: true, }, - "1": { - Type_: "VirtualDisk", - Path: dmVerityHashFullPath, - ReadOnly: true, - }, + //"1": { + // Type_: "VirtualDisk", + // Path: dmVerityHashFullPath, + // ReadOnly: true, + //}, }, }, } uvm.reservedSCSISlots = append(uvm.reservedSCSISlots, scsi.Slot{Controller: 0, LUN: 0}) - uvm.reservedSCSISlots = append(uvm.reservedSCSISlots, scsi.Slot{Controller: 0, LUN: 1}) + //uvm.reservedSCSISlots = append(uvm.reservedSCSISlots, scsi.Slot{Controller: 0, LUN: 1}) } } @@ -737,7 +741,7 @@ func makeLCOWDoc(ctx context.Context, opts *OptionsLCOW, uvm *UtilityVM) (_ *hcs uvm.vpmemDevicesDefault[0] = dev } } else { - kernelArgs = "root=/dev/sda ro rootwait init=/init" + kernelArgs = "loglevel=7 root=/dev/dm-0 dm-mod.create=\"dmverity,,,ro,0 173768 verity 1 /dev/sda /dev/sda 4096 4096 21721 21721 sha256 42896a788a58da77b6acb8ddf53aa744bd269c19146cfdf48eb8fc5529a52e62 a1c38923e44adffdd21f84e9185248c884fa28e767795d1025e5804e1c3df905\" init=/init" doc.VirtualMachine.Devices.Scsi[guestrequest.ScsiControllerGuids[0]].Attachments["0"] = hcsschema.Attachment{ Type_: "VirtualDisk", Path: rootfsFullPath,