systemd: Fix systemd-{cryptenroll,cryptsetup} TPM2 and FIDO2 support (attempt #3)

[zhaofengli]

Description of changes

This PR makes it possible to use systemd-cryptenroll (and its integration in systemd-cryptsetup) to automatically unlock LUKS volumes using TPM2 or FIDO2.

Based on #171242 (attempt #1) and #179823 (attempt #2).

Relavent discussions about potential attack vectors: #179823 (comment)

How to test

You can test this PR with the minimal amount of rebuilds by pulling in the updated packages and NixOS module on my cryptenroll-master branch:

{ pkgs, config, ... }:
let
  nixpkgs-cryptenroll = builtins.fetchTarball {
    name = "nixpkgs-cryptenroll";
    url = "https://github.com/zhaofengli/nixpkgs/archive/4b00eacbb8bd00dc08b6ebef6d220b3d8e49ecb4.tar.gz";
    sha256 = "0776mjndy9j7bsaa077js6bsrsanb102rccnm4kf5xzyrsmrh4d9";
  };
  pkgsPatched = import nixpkgs-cryptenroll {
    system = pkgs.system;
    overlays = [];
  };

  # HACK to use the same version of tpm2-tss and libfido2
  overlaidModule = let
    mod = import (nixpkgs-cryptenroll + "/nixos/modules/system/boot/systemd/initrd.nix");
  in { lib, config, utils, pkgs, ... } @ args: mod (args // {
    pkgs = pkgs // {
      inherit (pkgsPatched) tpm2-tss libfido2;
    };
  });
in {
  disabledModules = [
    "system/boot/systemd/initrd.nix"
  ];

  imports = [
    overlaidModule
  ];

  boot.initrd.systemd = {
    enable = true;
    package = pkgsPatched.systemdStage1;
    emergencyAccess = true;
    initrdBin = with pkgs; [ gnugrep strace ]; # for debugging only
  };

  boot.initrd.luks.devices = {
    yourvolume = {
      # ... remaining configs ...

      crypttabExtraOpts = [
        #"fido2-device=auto"
        "tpm2-device=auto"
      ];
    };
  };

  boot.initrd.availableKernelModules = [ "tpm-tis" "tpm-crb" ];
}

Reboot into the new configuration. After rebooting, add a new LUKS key slot with the key sealed to the current PCR values:

sudo systemd-cryptenroll --tpm2-pcrs=0+2+4+8+9 --tpm2-device=auto /dev/disk/by-uuid/your-uuid-here

Note: The PCR list here is an example only! You need to understand what's being measured and when. One bootloader-specific thing is initrd: systemd/systemd#12716

Then, reboot again. This time the volume should be decrypted without prompting you for a password.

Things done

• Built on platform(s)
  • x86_64-linux
  • aarch64-linux
  • x86_64-darwin
  • aarch64-darwin
• For non-Linux: Is sandbox = true set in nix.conf? (See Nix manual)
• Tested, as applicable:
  • NixOS test(s) (look inside nixos/tests)
  • and/or package tests
  • or, for functions and "core" functionality, tests in lib/tests or pkgs/test
  • made sure NixOS tests are linked to the relevant packages
• Tested compilation of all packages that depend on this change using nix-shell -p nixpkgs-review --run "nixpkgs-review rev HEAD". Note: all changes have to be committed, also see nixpkgs-review usage
• Tested basic functionality of all binary files (usually in ./result/bin/)
• 22.11 Release Notes (or backporting 22.05 Release notes)
  • (Package updates) Added a release notes entry if the change is major or breaking
  • (Module updates) Added a release notes entry if the change is significant
  • (Module addition) Added a release notes entry if adding a new NixOS module
  • (Release notes changes) Ran nixos/doc/manual/md-to-db.sh to update generated release notes
• Fits CONTRIBUTING.md.

[RaitoBezarius]

Do you think you can add a NixOS test?

[JJJollyjim]

@RaitoBezarius I was already writing one when I looked back at this PR to check something and saw your comment… should be done in an hour or so

[JJJollyjim]

Here we go:

import ./make-test-python.nix ({ lib, pkgs, ... }: {
  name = "systemd-initrd-luks-tpm";

  nodes.machine = { pkgs, ... }: {
    # Use systemd-boot
    virtualisation = {
      emptyDiskImages = [ 512 ];
      useBootLoader = true;
      useEFIBoot = true;
      qemu.options = ["-chardev socket,id=chrtpm,path=/tmp/mytpm1/swtpm-sock -tpmdev emulator,id=tpm0,chardev=chrtpm -device tpm-tis,tpmdev=tpm0"];
    };
    boot.loader.systemd-boot.enable = true;

    boot.initrd.availableKernelModules = [ "tpm_tis" ];

    environment.systemPackages = with pkgs; [ cryptsetup ];
    boot.initrd.systemd = {
      enable = true;
    };

    specialisation.boot-luks.configuration = {
      boot.initrd.luks.devices = lib.mkVMOverride {
        cryptroot = {
          device = "/dev/vdc";
          crypttabExtraOpts = [ "tpm2-device=auto" ];
        };
      };
      virtualisation.bootDevice = "/dev/mapper/cryptroot";
    };
  };

  testScript = ''
    import subprocess
    import os
    import time


    class Tpm:
        def __init__(self):
            os.mkdir("/tmp/mytpm1")
            self.start()

        def start(self):
            self.proc = subprocess.Popen(["${pkgs.swtpm}/bin/swtpm", "socket", "--tpmstate", "dir=/tmp/mytpm1", "--ctrl", "type=unixio,path=/tmp/mytpm1/swtpm-sock", "--log", "level=20", "--tpm2"])

        def wait_for_death_then_restart(self):
            while self.proc.poll() is None:
                print("waiting for tpm to die")
                time.sleep(1)
            assert self.proc.returncode == 0
            self.start()

    tpm = Tpm()


    # Create encrypted volume
    machine.wait_for_unit("multi-user.target")
    machine.succeed("echo -n supersecret | cryptsetup luksFormat -q --iter-time=1 /dev/vdc -")
    machine.succeed("PASSWORD=supersecret SYSTEMD_LOG_LEVEL=debug systemd-cryptenroll --tpm2-pcrs= --tpm2-device=auto /dev/vdc |& systemd-cat")

    # Boot from the encrypted disk
    machine.succeed("bootctl set-default nixos-generation-1-specialisation-boot-luks.conf")
    machine.succeed("sync")
    machine.crash()

    tpm.wait_for_death_then_restart()

    # Boot and decrypt the disk
    machine.wait_for_unit("multi-user.target")
    assert "/dev/mapper/cryptroot on / type ext4" in machine.succeed("mount")
  '';
})

Based on the existing systemd-initrd-luks-* family of tests – feel free to add to the PR.

[JJJollyjim]

(it passes on master with this PR added, not much rebuilding required for such a simple config, took me about 10 minutes)

[Kranzes]

Can we add to storePaths only the stuff we need fro pkgs.tpm2-tss and pkgs.libfido2?

[Kranzes]

Here's another minor patch that will make the additional store paths optional.

diff --git a/nixos/modules/system/boot/systemd/initrd.nix b/nixos/modules/system/boot/systemd/initrd.nix
index f10a8c1313f..f0ec3db3b21 100644
--- a/nixos/modules/system/boot/systemd/initrd.nix
+++ b/nixos/modules/system/boot/systemd/initrd.nix
@@ -394,6 +394,14 @@ in {
         "${cfg.package}/lib/systemd/system-generators/systemd-hibernate-resume-generator"
         "${cfg.package}/lib/systemd/system-generators/systemd-run-generator"
 
+        # utilities needed by systemd
+        "${cfg.package.util-linux}/bin/mount"
+        "${cfg.package.util-linux}/bin/umount"
+        "${cfg.package.util-linux}/bin/sulogin"
+
+        # so NSS can look up usernames
+        "${pkgs.glibc}/lib/libnss_files.so.2"
+      ] ++ optionals cfg.package.withCryptsetup [
         # tpm2 and fido2 support
         "${cfg.package}/lib/cryptsetup/libcryptsetup-token-systemd-tpm2.so"
         "${cfg.package}/lib/cryptsetup/libcryptsetup-token-systemd-fido2.so"
@@ -402,14 +410,6 @@ in {
 
         # the unwrapped systemd-cryptsetup executable
         "${cfg.package}/lib/systemd/.systemd-cryptsetup-wrapped"
-
-        # utilities needed by systemd
-        "${cfg.package.util-linux}/bin/mount"
-        "${cfg.package.util-linux}/bin/umount"
-        "${cfg.package.util-linux}/bin/sulogin"
-
-        # so NSS can look up usernames
-        "${pkgs.glibc}/lib/libnss_files.so.2"
       ] ++ jobScripts;
 
       targets.initrd.aliases = ["default.target"];

[zhaofengli]

Can we add to storePaths only the stuff we need fro pkgs.tpm2-tss and pkgs.libfido2?

We could, but I'd prefer not to do that to make it less fragile and reduce work in the future in case tpm2-tss/libfido2 change their dependencies.

Here's another minor patch that will make the additional store paths optional.

Thanks, applied.

[Kranzes]

We could, but I'd prefer not to do that to make it less fragile and reduce work in the future in case tpm2-tss/libfido2 change their dependencies.

Can that not be said about the rest of the stuff in storePaths?

[zhaofengli]

Can that not be said about the rest of the stuff in storePaths?

No, because the dependencies are listed in the ELF (that make-initrd-ng picks up automatically), unlike tpm2-tss which dlopen's libraries dynamically depending on the interface.

I went ahead and did it anyways and looks like it saved us 1MB, but at what cost?

[Kranzes]

You're probably right, maybe just keep libfido's single library and still use pkgs.tpm2-tss?

[zhaofengli]

To measure the impact on the initrd size, let's use the following simple expression:

let
  pkgs = import ./. {};
  nixos = pkgs.nixos {
    boot.initrd.systemd.enable = true;
  };
in nixos.config.system.build.initialRamdisk

tpm2-tss	libfido2	Size (bytes)	Δ from baseline
Without added files		10948957	0
Slim	Slim	13613312	+2.5MiB
Slim	Full	13760201	+2.7MiB
Full	Slim	13908566	+2.8MiB
Full	Full	1405251	+3.0MiB

Note that the sizes here are after zstd compression. Including the percentage increases in the table would be misleading, because the baseline sizes in actual configs are larger due to the bundled kernel modules.

[oxalica]

Should we remove these assertions now? Seems there are no more concerns?

nixpkgs/nixos/modules/system/boot/luksroot.nix

Lines 908 to 911 in 74d81b4

	# TODO
	{ assertion = config.boot.initrd.systemd.enable -> !luks.fido2Support;
	message = "systemd stage 1 does not support FIDO2 yet.";
	}

[endocrimes]

~3MB of initrd growth for fairly large quality of life improvements seems pretty reasonable to me 👍

[oxalica]

For anyone interested, I wrote a FIDO2 test using QEMU with CanoKey support in oxalica@d75bc7b, which depends on #194254.

[lovesegfault]

I merged #194254, this PR can be rebased and @oxalica's commit cherry-picked :)

[zhaofengli]

Done! The test won't work in staging since #194254 isn't there yet, but it seems to pass on cryptenroll-master.

One weird thing though, if I add os.unlink("/tmp/canokey-file") before the second boot the test still passes for some reason. However, the cryptenroll functionality does seem to be used. @oxalica

[oxalica]

One weird thing though, if I add os.unlink("/tmp/canokey-file") before the second boot the test still passes for some reason.

Seems the private key of the virtual card is hard-coded.
https://github.com/canokeys/canokey-core/blob/56182bb4e8b0566954dff4e9af9762b26f580594/virt-card/fabrication.c

[colemickens]

Small notes:

1. This "blocked" startup in such a way that the usual hidpi font didn't kick in until after I performed the auth.
2. I didn't see any prompt on screen, I just noticed the yubikey flashing at me.
3. By pulling this and adding a specialisation for this boot method, I ran into and fixed nixos: luksroot: toString-ify keyFileSize usage #194530

Otherwise it worked great, thanks!

(EDIT: sorry, this can be ignored, I missed the context that @oxalica clarifies in the next reply.)

[oxalica]

@colemickens

1. This "blocked" startup in such a way that the usual hidpi font didn't kick in until after I performed the auth.
2. I didn't see any prompt on screen, I just noticed the yubikey flashing at me.

I believe that's unrelated and deserves an individual issue about systemd-initrd. You could try to shuffle around the loading order of your graphics driver, between boot.{,initrd.{available,}}kernelModules.

This PR is not just about systemd-initrd. Currently on master, even full systemd package doesn't support FIDO2 enrolling.

[arianvp]

@GrahamcOfBorg build cryptsetup systemd

[arianvp]

@GrahamcOfBorg test systemd cryptsetup

[arianvp]

@GrahamcOfBorg build cryptsetup.passthru.tests

For some reason it failed on x86? maybe flakey? just rerunning before merging

[zhaofengli]

Rebased to resolve conflicts. The failure seems to be unrelated, and the tests (systemd, systemd-initrd-luks-tpm2, systemd-initrd-luks-fido2) do succeed on the cryptsetup-master branch.

error: builder for '/nix/store/xpw7xrlq6gs8zm2cz071z45s6wbh1jxp-python3.10-eventlet-0.33.1.drv' failed with exit code 1;
       last 10 log lines:
       >   /build/source/eventlet/greenthread.py:221: DeprecationWarning: capitalize_response_headers is disabled.
       >    Please, make sure you know what you are doing.
       >    HTTP headers names are case-insensitive per RFC standard.
       >    Most likely, you need to fix HTTP parsing in your client software.
       >     result = function(*args, **kwargs)
       >
       > -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
       > =========================== short test summary info ============================
       > FAILED tests/patcher_test.py::test_fork_after_monkey_patch - AssertionError: ...
       > = 1 failed, 562 passed, 120 skipped, 17 deselected, 2483 warnings in 113.06s (0:01:53) =
       For full logs, run 'nix log /nix/store/xpw7xrlq6gs8zm2cz071z45s6wbh1jxp-python3.10-eventlet-0.33.1.drv'.

[Kranzes]

Just in case anyone is having a similar problem I created an issue on upstream: systemd/systemd#25128
(systemd-cryptsetup: cannot enroll a second FIDO2 slot after password was deleted)

[ajs124]

Can we maybe make this optional? Or conditional on if the kernel has this option set?

I have some custom kernel configs that really don't need this module and also use the new initrd and now the initrd build fails with modprobe: FATAL: Module tpm-tis not found in directory […].

[zhaofengli]

I guess we can have a new option under boot.initrd.luks.devices.<name> that indicates whether a LUKS makes use of cryptenroll-based TPM2/FIDO2 unlocking, and only insert the modules and extra files when a volume with those options enabled exists. However, this can be confusing for non-systemd-stage1 users, especially when boot.initrd.luks.devices.<name>.fido2 already exists.

[flokli]

I'd prefer if disabling these kernel modules would an opt-in config option.

We can't determine at evaluation time if there's a fido luks slot configured.

[ajs124]

fair enough. iirc we can determine at eval time if the kernel even has these modules enabled. maybe using system.requiredKernelConfig?

[ajs124]

looking at the kernel config, these options/modules don't even seem to be available on non-x86 systems or systems without ACPI, so I'm really not sure they should be enabled by default

[flokli]

Hmmh, that'd be a problem. Can we check if the kernel has these modules configured, and only include them in that case?

[ajs124]

I think something like this might work:

--- a/nixos/modules/system/boot/systemd/initrd.nix                                                                                                                                    
+++ b/nixos/modules/system/boot/systemd/initrd.nix
@@ -334,8 +334,11 @@ in {
               
     boot.initrd.availableKernelModules = [
       "autofs4"           # systemd needs this for some features
-      "tpm-tis" "tpm-crb" # systemd-cryptenroll              
-    ];                                                                                    
+    ]                                                                                     
+      # systemd-cryptenroll                                                                                                                                                          
+      ++ lib.optional (config.boot.kernelPackages.kernel.config.isEnabled "TCG_TIS") "tpm-tis"
+      ++ lib.optional (config.boot.kernelPackages.kernel.config.isEnabled "TCG_CRB") "tpm-crb"                                                                     
+    ;                                                                                     
                                
     boot.initrd.systemd = {                                                               
       initrdBin = [pkgs.bash pkgs.coreutils cfg.package.kmod cfg.package] ++ config.system.fsPackages;                                                                              

but only if we actually set these options and don't rely on kernel defaults. Maybe.

For some reason nixosTests.systemd-initrd-luks-fido2 and nixosTests.systemd-initrd-luks-tpm2 build even with:

     boot.initrd.availableKernelModules = [
       "autofs4"           # systemd needs this for some features
     ];

so I'm not sure how to test.

[zhaofengli]

The VM test passes without adding any extra modules but on (some?) actual hardware they are required.