Update the fscryptctl tests

- Add tests for all new functionality.  That primarily includes v2
  policy support in get_policy and set_policy; the new commands add_key,
  remove_key, and key_status; support for more encryption modes; and
  support for key sizes other than 64 bytes.

- Update to Python 3, as Python 2 is no longer supported by python.org.

- Change 'make test' to no longer require root.  It now just
  accepts an empty directory on a filesystem that supports encryption.
  It no longer requires that it be a mountpoint, and it no longer
  unmount and remounts the filesystem.  (Unmounting was used to test
  fully removing encryption keys for v1 policies, but that wasn't
  testing fscryptctl itself anyway, so having it in the fscryptctl test
  suite didn't add too much.  And it's not needed for v2 policies.)

- Change 'make test-setup' and 'make test-teardown' run sudo themselves,
  like they do in the Makefile for 'fscrypt', so that the user doesn't
  need to explicitly run them under 'sudo'.

- Add 'make test-all' as shorthand for test-setup + test +
  test-teardown.

- Support wrapping the fscryptctl binary with valgrind.

- Bump the kernel requirement to run the tests up to 5.4 or later with
  support for the AES-128-CBC and Adiantum encryption modes enabled.  We
  could skip tests that the kernel doesn't support, but that would take
  some extra work, and it's unclear that it would be worthwhile.

CONTRIBUTING.md

@@ -24,18 +24,24 @@ information on using pull requests.
 
 ## Before you submit a pull request
 
-When making any changes to `fscryptctl`, you will need the following commands:
-*   `make format` which formats all of the C code (requires `clang-format`)
-*   `make test` which runs the tests for fscryptctl (requires `python` and the
-    `pytest` and `keyutils` python packages). Note that to run all of the tests,
-    the environment variable `TEST_FILESYSTEM_ROOT` must be set to the
-    mountpoint of an ext4 filesystem setup for encryption that the user can
-    mount and unmount.
-*   `make all` - Runs the above commands and builds `fscryptctl`.
-
-Make sure all these commands are run and the tests pass before submitting a pull
-request. All the above dependencies can be installed with:
+When making any changes to `fscryptctl`, run the following commands:
+* `make format`, which formats the source code (requires `clang-format`)
+* `make test-all`, which builds `fscryptctl` and runs the tests.  The tests
+  require the `e2fsprogs` and `python3` packages, the `pytest` and `keyutils`
+  Python packages, and kernel support for ext4 encryption.
+
+The userspace dependencies can be installed with:
 ``` bash
-> sudo apt-get install python-pip libkeyutils-dev clang-format
-> sudo -H pip install -U pip pytest keyutils
+> sudo apt-get install e2fsprogs python3-pip libkeyutils-dev clang-format
+> sudo -H pip3 install -U pip pytest keyutils
+```
+
+Your Linux kernel must be version 5.4 or later and have the following
+configuration options enabled:
+```
+CONFIG_EXT4_FS
+CONFIG_FS_ENCRYPTION
+CONFIG_CRYPTO_ADIANTUM
+CONFIG_CRYPTO_SHA256
+CONFIG_CRYPTO_ESSIV	(if 5.5 or later)
 ```

Makefile

@@ -83,42 +83,50 @@ format-check:
 
 # Testing targets
 
-# IMAGE will be the path to our test ext4 image file.
-IMAGE ?= fscryptctl_image
-
-# MOUNT will be the path to the filesystem where our tests are run.
+# The 'test' target requires that $(TEST_DIR) point to a directory on a
+# filesystem that supports encryption.
+#
+# 'test-setup' sets up the default TEST_DIR to point to a directory on a
+# temporary ext4 filesystem on a loopback device.  'test-teardown' cleans up
+# afterwards.  Note that both of these use 'sudo'.
 #
-# Running "make test-setup MOUNT=/foo/bar" creates a test filesystem at that
-#	location. Be sure to also run "make test-teardown MOUNT=/foo/bar".
-# Running "make test MOUNT=/foo/bar" will run all tests on that filesystem. By
-#       default, it is the one created with "make test-setup".
-MOUNT ?= /mnt/fscryptctl_mount
-export TEST_FILESYSTEM_ROOT = $(MOUNT)
-
-.PHONY: root test
-root:
-ifneq ($(shell id -u),0)
-	$(error You must be root to execute this command)
-endif
-
-test: fscryptctl root
-ifeq ("$(wildcard $(MOUNT))","")
-	$(error mountpoint $(MOUNT) does not exist, run "make test-setup")
-endif
-	python -m pytest test.py -s -q
-
-.PHONY: test-setup test-teardown
-test-setup: root
-	dd if=/dev/zero of=$(IMAGE) bs=1M count=20
-	mkfs.ext4 -b 4096 -O encrypt -F $(IMAGE)
-	mkdir -p $(MOUNT)
-	mount -o rw,loop,user $(IMAGE) $(MOUNT)
-	chmod +777 $(MOUNT)
-
-test-teardown: root
-	umount $(MOUNT)
-	rmdir $(MOUNT)
-	rm -f $(IMAGE)
+# 'test-all' runs 'test-setup', 'test', and 'test-teardown'.
+
+TEST_IMAGE ?= /tmp/fscryptctl-test-image
+TEST_DIR ?= /tmp/fscryptctl-test-dir
+
+.PHONY: test test-setup test-teardown test-all
+
+test: fscryptctl
+	@if [ ! -e "$(TEST_DIR)" ]; then \
+		echo 1>&2 "Directory $(TEST_DIR) does not exist, run 'make test-setup'"; \
+		exit 1; \
+	fi
+	TEST_DIR="$(TEST_DIR)" PATH="$$PWD:$$PATH" \
+		 ENABLE_VALGRIND="$(ENABLE_VALGRIND)" \
+		 python3 -m pytest test.py -s -q
+
+# Depend on test-teardown so that anything already present is cleaned up first.
+test-setup:test-teardown
+	dd if=/dev/zero of="$(TEST_IMAGE)" bs=1M count=32
+	mkfs.ext4 -b 4096 -O encrypt -F "$(TEST_IMAGE)"
+	mkdir -p "$(TEST_DIR)"
+	sudo mount -o rw,loop "$(TEST_IMAGE)" "$(TEST_DIR)"
+	sudo sh -c 'chown $$SUDO_USER:$$SUDO_USER "$(TEST_DIR)"'
+	@echo
+	@echo "$(TEST_DIR) is now set up."
+
+test-teardown:
+	if mountpoint "$(TEST_DIR)" &> /dev/null; then \
+		sudo umount "$(TEST_DIR)"; \
+	fi
+	rm -rf "$(TEST_DIR)"
+	rm -f "$(TEST_IMAGE)"
+
+test-all:
+	$(MAKE) test-setup
+	$(MAKE) test
+	$(MAKE) test-teardown
 
 .PHONY: travis-install travis-script
 travis-install: test-setup
@@ -138,6 +146,6 @@ uninstall:
 	rm -f $(DESTDIR)$(BINDIR)/fscryptctl
 
 clean:
-	rm -f fscryptctl *.o *.pyc $(IMAGE)
+	rm -f fscryptctl *.o *.pyc
 	rm -rf __pycache__
-	rm -rf .cache
+	rm -rf .pytest_cache

generate_test_key_identifiers.py

@@ -0,0 +1,50 @@
+#
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+#
+
+"""This program generates the key descriptors and key identifiers for the test
+keys in test.py."""
+
+import hashlib
+import test
+
+# For HKDF-SHA512; see
+# https://www.pycryptodome.org/en/latest/src/protocol/kdf.html#hkdf
+import Crypto.Hash.SHA512
+import Crypto.Protocol.KDF
+
+
+def compute_key_descriptor(raw):
+    return hashlib.sha512(hashlib.sha512(raw).digest()).hexdigest()[:16]
+
+
+def compute_key_identifier(raw):
+    return Crypto.Protocol.KDF.HKDF(raw, 16, "", Crypto.Hash.SHA512,
+                                    context=b"fscrypt\0\1").hex()
+
+
+for key in test.TEST_KEYS:
+    raw = key["raw"]
+    descriptor = compute_key_descriptor(raw)
+    identifier = compute_key_identifier(raw)
+    if "descriptor" in key:
+        assert descriptor == key["descriptor"]
+    if "identifier" in key:
+        assert identifier == key["identifier"]
+    print("... = {")
+    print("    raw: " + str(raw) + ",")
+    print('    "descriptor": "' + descriptor + '",')
+    print('    "identifier": "' + identifier + '",')
+    print("}")

input_fail.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Exit with 1 if any input is provided. Print the input to stdout, unless an
 # argument is specified. In that case, print the argument instead.