kola: Add tests for encrypted root disks with TPM PCR binding

As documented in flatcar/flatcar-website#317
we can use PCR binding in Flatcar with some limitations and workarounds.
I think we should be able to get rid of the first boot PCR difference
by handling the first-boot flag detection in userspace and if we can
guarantee that the outcome of setting the first boot or not is the same,
i.e., we would have to always measure the effective Ignition config.
For the rebinding on update the best bet we have is to create a Flatcar
variant with sd-boot for signed PCR policies. Anyway, these are future
topics and it's already good that we can make some encryption setups
work.

kola/tests/misc/tpm.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/coreos/go-semver/semver"
 	"github.com/coreos/pkg/capnslog"
@@ -47,6 +48,127 @@ systemd:
         ExecStart=rm /etc/luks/rootencrypted
         [Install]
         WantedBy=multi-user.target
+`)
+	// Note: Keep the two below configs in sync with those
+	// documented in the Flatcar TPM docs.
+	// Ideally the reboot wouldn't be needed (or done in the initrd?)
+	IgnitionConfigRootCryptenrollPcrNoUpdate = conf.Butane(`---
+variant: flatcar
+version: 1.0.0
+storage:
+  files:
+    - path: /etc/flatcar/update.conf
+      overwrite: true
+      contents:
+        inline: |
+          SERVER=disabled
+  luks:
+  - name: rootencrypted
+    wipe_volume: true
+    device: "/dev/disk/by-partlabel/ROOT"
+  filesystems:
+    - device: /dev/mapper/rootencrypted
+      format: ext4
+      label: ROOT
+systemd:
+  units:
+    - name: cryptenroll-helper-first.service
+      enabled: true
+      contents: |
+        [Unit]
+        ConditionFirstBoot=true
+        OnFailure=emergency.target
+        OnFailureJobMode=isolate
+        After=first-boot-complete.target multi-user.target
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStart=systemd-cryptenroll --tpm2-device=auto --unlock-key-file=/etc/luks/rootencrypted --tpm2-pcrs= /dev/disk/by-partlabel/ROOT
+        ExecStart=mv /etc/luks/rootencrypted /etc/luks/rootencrypted-bind
+        ExecStart=sleep 10
+        ExecStart=systemctl reboot
+        [Install]
+        WantedBy=multi-user.target
+    - name: cryptenroll-helper-bind.service
+      enabled: true
+      contents: |
+        [Unit]
+        ConditionFirstBoot=false
+        ConditionPathExists=/etc/luks/rootencrypted-bind
+        OnFailure=emergency.target
+        OnFailureJobMode=isolate
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStart=systemd-cryptenroll --tpm2-device=auto --unlock-key-file=/etc/luks/rootencrypted-bind --tpm2-pcrs=4+7+8+9+11+12+13 --wipe-slot=tpm2 /dev/disk/by-partlabel/ROOT
+        ExecStart=mv /etc/luks/rootencrypted-bind /etc/luks/rootencrypted
+        [Install]
+        WantedBy=multi-user.target
+`)
+	// The rebinding for the update is due to how GRUB measures things and
+	// we can only make this work without rebinding if we switch to sd-boot
+	IgnitionConfigRootCryptenrollPcrWithUpdate = conf.Butane(`---
+variant: flatcar
+version: 1.0.0
+storage:
+  files:
+    - path: /oem/bin/oem-postinst
+      overwrite: true
+      mode: 0755
+      contents:
+        inline: |
+          #!/bin/bash
+          set -euo pipefail
+          # When the update fails to correctly apply, this runs again
+          if [ -e /etc/luks/rootencrypted-bound ]; then
+            mv /etc/luks/rootencrypted-bound /etc/luks/rootencrypted-bind
+          fi
+          # But since a reboot inbetween could have bound it again,
+          # remove the PCR binding for every run
+          systemd-cryptenroll --tpm2-device=auto --unlock-key-file=/etc/luks/rootencrypted-bind --wipe-slot=tpm2 --tpm2-pcrs= /dev/disk/by-partlabel/ROOT
+  luks:
+  - name: rootencrypted
+    wipe_volume: true
+    device: "/dev/disk/by-partlabel/ROOT"
+  filesystems:
+    - device: /dev/mapper/rootencrypted
+      format: ext4
+      label: ROOT
+systemd:
+  units:
+    - name: cryptenroll-helper-first.service
+      enabled: true
+      contents: |
+        [Unit]
+        ConditionFirstBoot=true
+        OnFailure=emergency.target
+        OnFailureJobMode=isolate
+        After=first-boot-complete.target multi-user.target
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStart=systemd-cryptenroll --tpm2-device=auto --unlock-key-file=/etc/luks/rootencrypted --tpm2-pcrs= /dev/disk/by-partlabel/ROOT
+        ExecStart=mv /etc/luks/rootencrypted /etc/luks/rootencrypted-bind
+        ExecStart=sleep 10
+        ExecStart=systemctl reboot
+        [Install]
+        WantedBy=multi-user.target
+    - name: cryptenroll-helper-bind.service
+      enabled: true
+      contents: |
+        [Unit]
+        ConditionFirstBoot=false
+        ConditionPathExists=/etc/luks/rootencrypted-bind
+        OnFailure=emergency.target
+        OnFailureJobMode=isolate
+        Before=update-engine.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStart=systemd-cryptenroll --tpm2-device=auto --unlock-key-file=/etc/luks/rootencrypted-bind --tpm2-pcrs=4+7+8+9+11+12+13 --wipe-slot=tpm2 /dev/disk/by-partlabel/ROOT
+        ExecStart=mv /etc/luks/rootencrypted-bind /etc/luks/rootencrypted-bound
+        [Install]
+        WantedBy=multi-user.target
 `)
 	IgnitionConfigRootTPM = `{
 		"ignition": {
@@ -124,7 +246,7 @@ systemd:
 
 func init() {
 	runRootTPMCryptenroll := func(c cluster.TestCluster) {
-		tpmTest(c, IgnitionConfigRootCryptenroll, "/")
+		tpmTest(c, IgnitionConfigRootCryptenroll, "/", false)
 	}
 	register.Register(&register.Test{
 		Run:         runRootTPMCryptenroll,
@@ -134,9 +256,31 @@ func init() {
 		Distros:     []string{"cl"},
 		MinVersion:  semver.Version{Major: 3913, Minor: 0, Patch: 1},
 	})
+	runRootTPMCryptenrollPcrNoUpdate := func(c cluster.TestCluster) {
+		tpmTest(c, IgnitionConfigRootCryptenrollPcrNoUpdate, "/", true)
+	}
+	register.Register(&register.Test{
+		Run:         runRootTPMCryptenrollPcrNoUpdate,
+		ClusterSize: 0,
+		Platforms:   []string{"qemu"},
+		Name:        "cl.tpm.root-cryptenroll-pcr-noupdate",
+		Distros:     []string{"cl"},
+		MinVersion:  semver.Version{Major: 3913, Minor: 0, Patch: 1},
+	})
+	runRootTPMCryptenrollPcrWithUpdate := func(c cluster.TestCluster) {
+		tpmTest(c, IgnitionConfigRootCryptenrollPcrWithUpdate, "/", true)
+	}
+	register.Register(&register.Test{
+		Run:         runRootTPMCryptenrollPcrWithUpdate,
+		ClusterSize: 0,
+		Platforms:   []string{"qemu"},
+		Name:        "cl.tpm.root-cryptenroll-pcr-withupdate",
+		Distros:     []string{"cl"},
+		MinVersion:  semver.Version{Major: 3913, Minor: 0, Patch: 1},
+	})
 
 	runRootTPM := func(c cluster.TestCluster) {
-		tpmTest(c, conf.Ignition(IgnitionConfigRootTPM), "/")
+		tpmTest(c, conf.Ignition(IgnitionConfigRootTPM), "/", false)
 	}
 	register.Register(&register.Test{
 		Run:         runRootTPM,
@@ -148,7 +292,7 @@ func init() {
 	})
 
 	runNonRootTPM := func(c cluster.TestCluster) {
-		tpmTest(c, conf.Ignition(IgnitionConfigNonRootTPM), "/mnt/data")
+		tpmTest(c, conf.Ignition(IgnitionConfigNonRootTPM), "/mnt/data", false)
 	}
 	register.Register(&register.Test{
 		Run:         runNonRootTPM,
@@ -160,7 +304,7 @@ func init() {
 	})
 }
 
-func tpmTest(c cluster.TestCluster, userData *conf.UserData, mountpoint string) {
+func tpmTest(c cluster.TestCluster, userData *conf.UserData, mountpoint string, wait bool) {
 	swtpm, err := startSwtpm()
 	if err != nil {
 		c.Fatalf("could not start software TPM emulation: %v", err)
@@ -188,6 +332,13 @@ func tpmTest(c cluster.TestCluster, userData *conf.UserData, mountpoint string)
 		c.Fatal(err)
 	}
 
+	if wait {
+		// Wait for the first reboot
+		time.Sleep(1 * time.Minute)
+		// Verify that the machine rebooted
+		_ = c.MustSSH(m, "grep -v flatcar.first_boot /proc/cmdline")
+	}
+
 	checkIfMountpointIsEncrypted(c, m, mountpoint)
 
 	// Make sure the change is reboot-safe. This is especially important for the case of an encrypted root disk because the