Build for measured rootfs improvements

src/runtime-rs/Makefile

@@ -452,12 +452,15 @@ optimize: $(SOURCES) | show-summary show-header
 	@RUSTFLAGS="-C link-arg=-s $(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES)
 
 ##TARGET clean: clean build
-clean:
+clean: clean-generated-files
 	@cargo clean
-	@rm -f $(GENERATED_FILES)
 	@rm -f tarpaulin-report.html
 	@rm -f $(CONFIGS)
 
+##TARGET clean-generated-files: clean generated files
+clean-generated-files:
+	@rm -f $(GENERATED_FILES)
+
 vendor:
 	@cargo vendor
 

src/runtime/Makefile

@@ -879,15 +879,16 @@ vendor: handle_vendor
 
 static-checks-build: $(GENERATED_FILES)
 
-clean:
+clean: clean-generated-files
 	$(QUIET_CLEAN)rm -f \
-		$(CONFIGS) \
-		$(GENERATED_FILES) \
 		$(MONITOR) \
 		$(SHIMV2) \
 		$(TARGET) \
 		.git-commit .git-commit.tmp
 
+clean-generated-files:
+	$(QUIET_CLEAN)rm -f $(GENERATED_FILES)
+
 show-usage: show-header
 	@printf "• Overview:\n"
 	@printf "\n"
@@ -904,6 +905,7 @@ show-usage: show-header
 	@printf "\tfast-test                  : run tests with failfast option.\n"
 	@printf "\tcheck                      : run code checks.\n"
 	@printf "\tclean                      : remove built files.\n"
+	@printf "\tclean-generated-files      : remove generated files.\n"
 	@printf "\tcontainerd-shim-v2         : only build containerd shim v2.\n"
 	@printf "\tcoverage                   : run coverage tests.\n"
 	@printf "\tdefault                    : same as 'make build' (or just 'make').\n"

tests/integration/kubernetes/k8s-measured-rootfs.bats

@@ -0,0 +1,52 @@
+#!/usr/bin/env bats
+#
+# Copyright (c) 2023 Red Hat
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+load "${BATS_TEST_DIRNAME}/lib.sh"
+load "${BATS_TEST_DIRNAME}/tests_common.sh"
+
+check_and_skip() {
+	# Currently the only kernel built with measured rootfs support is
+	# the kernel-tdx-experimental.
+	[ "${KATA_HYPERVISOR}" = "qemu-tdx" ] || \
+		skip "measured rootfs tests not implemented for hypervisor: $KATA_HYPERVISOR"
+}
+
+setup() {
+	check_and_skip
+	setup_common
+}
+
+teardown() {
+	check_and_skip
+
+	kubectl describe -f "${pod_config}" || true
+	kubectl delete -f "${pod_config}" || true
+}
+
+@test "Test cannnot launch pod with measured boot enabled and incorrect hash" {
+	pod_config="$(new_pod_config nginx "kata-${KATA_HYPERVISOR}")"
+
+	incorrect_hash="5180b1568c2ba972e4e06ee0a55976acae8329f2a5d1d2004395635e1ec4a76e"
+
+	# Despite the kernel being built with support, it is not currently enabled
+	# on configuration.toml. To avoid editing that file on the worker node,
+	# here it will be enabled via pod annotations.
+	set_metadata_annotation "$pod_config" \
+		"io.katacontainers.config.hypervisor.kernel_params" \
+		"rootfs_verity.scheme=dm-verity rootfs_verity.hash=$incorrect_hash"
+	# Run on a specific node so we know from where to inspect the logs
+	set_node "$pod_config" "$node"
+
+	# For debug sake
+	echo "Pod $pod_config file:"
+	cat $pod_config
+
+	assert_pod_fail "$pod_config"
+
+	assert_logs_contain "$node" kata "$node_start_time" \
+		'verity: .* metadata block .* is corrupted'
+}

tests/integration/kubernetes/lib.sh

@@ -0,0 +1,185 @@
+#!/bin/bash
+# Copyright (c) 2021, 2022 IBM Corporation
+# Copyright (c) 2022, 2023 Red Hat
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# This provides generic functions to use in the tests.
+#
+set -e
+
+# Delete all pods if any exist, otherwise just return
+#
+k8s_delete_all_pods_if_any_exists() {
+	[ -z "$(kubectl get --no-headers pods)" ] || \
+		kubectl delete --all pods
+}
+
+FIXTURES_DIR="${BATS_TEST_DIRNAME}/runtimeclass_workloads"
+
+# Wait until the pod is not 'Ready'. Fail if it hits the timeout.
+#
+# Parameters:
+#	$1 - the sandbox ID
+#	$2 - wait time in seconds. Defaults to 120. (optional)
+#
+k8s_wait_pod_be_ready() {
+	local pod_name="$1"
+	local wait_time="${2:-120}"
+
+	kubectl wait --timeout="${wait_time}s" --for=condition=ready "pods/$pod_name"
+}
+
+# Create a pod and wait it be ready, otherwise fail.
+#
+# Parameters:
+#	$1 - the pod configuration file.
+#
+k8s_create_pod() {
+	local config_file="$1"
+	local pod_name=""
+
+	if [ ! -f "${config_file}" ]; then
+		echo "Pod config file '${config_file}' does not exist"
+		return 1
+	fi
+
+	kubectl apply -f "${config_file}"
+	if ! pod_name=$(kubectl get pods -o jsonpath='{.items..metadata.name}'); then
+		echo "Failed to create the pod"
+		return 1
+	fi
+
+	if ! k8s_wait_pod_be_ready "$pod_name"; then
+		# TODO: run this command for debugging. Maybe it should be
+		#       guarded by DEBUG=true?
+		kubectl get pods "$pod_name"
+		return 1
+	fi
+}
+
+# Check the logged messages on host have a given message.
+#
+# Parameters:
+#	$1 - the k8s worker node name
+#	$2 - the syslog identifier as in journalctl's -t option
+#	$3 - only logs since date/time (%Y-%m-%d %H:%M:%S)
+#	$4 - the message
+#
+assert_logs_contain() {
+	local node="$1"
+	local log_id="$2"
+	local datetime="$3"
+	local message="$4"
+
+	# Note: with image-rs we get more than the default 1000 lines of logs
+	print_node_journal "$node" "$log_id" --since "$datetime" -n 100000 \
+		grep "$message"
+}
+
+# Create a pod then assert it fails to run. Use in tests that you expect the
+# pod creation to fail.
+#
+# Note: a good testing practice is to afterwards check that the pod creation
+# failed because of the expected reason.
+#
+# Parameters:
+#	$1 - the pod configuration file.
+#
+assert_pod_fail() {
+	local container_config="$1"
+	echo "In assert_pod_fail: $container_config"
+
+	echo "Attempt to create the container but it should fail"
+	! k8s_create_pod "$container_config" || /bin/false
+}
+
+# Create a pod configuration out of a template file.
+#
+# Parameters:
+#	$1 - the container image.
+#	$2 - the runtimeclass
+#
+# Return:
+# 	the path to the configuration file. The caller should not care about
+# 	its removal afterwards as it is created under the bats temporary
+# 	directory.
+#
+new_pod_config() {
+	local base_config="${FIXTURES_DIR}/pod-config.yaml.in"
+	local image="$1"
+	local runtimeclass="$2"
+	local new_config
+
+	# The runtimeclass is not optional.
+	[ -n "$runtimeclass" ] || return 1
+
+	new_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename "${base_config}").XXX")
+	IMAGE="$image" RUNTIMECLASS="$runtimeclass" envsubst < "$base_config" > "$new_config"
+	echo "$new_config"
+}
+
+# Set an annotation on configuration metadata.
+#
+# Usually you will pass a pod configuration file where the 'metadata'
+# is relative to the 'root' path. Other configuration files like deployments,
+# the annotation should be set on 'spec.template.metadata', so use the 4th
+# parameter of this function to pass the base metadata path (for deployments
+# cases, it will be 'spec.template' for example).
+#
+# Parameters:
+#	$1 - the yaml file
+#	$2 - the annotation key
+#	$3 - the annotation value
+#	$4 - (optional) base metadata path
+set_metadata_annotation() {
+	local yaml="${1}"
+	local key="${2}"
+	local value="${3}"
+	local metadata_path="${4:-}"
+	local annotation_key=""
+
+	[ -n "$metadata_path" ] && annotation_key+="${metadata_path}."
+
+	# yaml annotation key name.
+	annotation_key+="metadata.annotations.\"${key}\""
+
+	echo "$annotation_key"
+	# yq set annotations in yaml. Quoting the key because it can have
+	# dots.
+	yq w -i --style=double "${yaml}" "${annotation_key}" "${value}"
+}
+
+# Set the node name on configuration spec.
+#
+# Parameters:
+#	$1 - the yaml file
+#	$2 - the node name
+#
+set_node() {
+	local yaml="$1"
+	local node="$2"
+	[ -n "$node" ] || return 1
+
+	yq w -i "${yaml}" "spec.nodeName" "$node"
+}
+
+# Get the systemd's journal from a worker node
+#
+# Parameters:
+#	$1 - the k8s worker node name
+#	$2 - the syslog identifier as in journalctl's -t option
+#	$N - (optional) any extra parameters to journalctl
+#
+print_node_journal() {
+	local node="$1"
+	local id="$2"
+	shift 2
+	local img="quay.io/prometheus/busybox"
+
+	kubectl debug --image "$img" -q -it "node/${node}" \
+		-- chroot /host journalctl -x -t "$id" --no-pager "$@"
+	# Delete the debugger pod
+	kubectl get pods -o name | grep "node-debugger-${node}" | \
+		xargs kubectl delete > /dev/null
+}

tests/integration/kubernetes/run_kubernetes_tests.sh

@@ -38,6 +38,7 @@ else
 		"k8s-kill-all-process-in-container.bats" \
 		"k8s-limit-range.bats" \
 		"k8s-liveness-probes.bats" \
+		"k8s-measured-rootfs.bats" \
 		"k8s-memory.bats" \
 		"k8s-nested-configmap-secret.bats" \
 		"k8s-oom.bats" \

tests/integration/kubernetes/runtimeclass_workloads/pod-config.yaml.in

@@ -0,0 +1,14 @@
+# Copyright (c) 2021, 2022 IBM Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-e2e
+spec:
+  runtimeClassName: $RUNTIMECLASS
+  containers:
+  - name: test-container
+    image: $IMAGE
+    imagePullPolicy: Always

tests/integration/kubernetes/tests_common.sh

@@ -33,6 +33,23 @@ dragonball_limitations="https://github.com/kata-containers/kata-containers/issue
 # overwrite it.
 export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/config}"
 
+# Common setup for tests.
+#
+# Global variables exported:
+#	$node	             - random picked node that has kata installed
+#	$node_start_date     - start date/time at the $node for the sake of
+#                          fetching logs
+#
+setup_common() {
+	node=$(get_one_kata_node)
+	[ -n "$node" ]
+	node_start_time=$(exec_host "$node" date +\"%Y-%m-%d %H:%M:%S\")
+	[ -n "$node_start_time" ]
+	export node node_start_time
+
+	k8s_delete_all_pods_if_any_exists || true
+}
+
 get_pod_config_dir() {
 	pod_config_dir="${BATS_TEST_DIRNAME}/runtimeclass_workloads_work"
 	info "k8s configured to use runtimeclass"

tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh

@@ -25,7 +25,6 @@ readonly versions_yaml="${repo_root_dir}/versions.yaml"
 readonly agent_builder="${static_build_dir}/agent/build.sh"
 readonly clh_builder="${static_build_dir}/cloud-hypervisor/build-static-clh.sh"
 readonly firecracker_builder="${static_build_dir}/firecracker/build-static-firecracker.sh"
-readonly initramfs_builder="${static_build_dir}/initramfs/build.sh"
 readonly kernel_builder="${static_build_dir}/kernel/build.sh"
 readonly ovmf_builder="${static_build_dir}/ovmf/build.sh"
 readonly qemu_builder="${static_build_dir}/qemu/build-static-qemu.sh"
@@ -300,7 +299,7 @@ install_cached_kernel_tarball_component() {
 install_kernel_helper() {
 	local kernel_version_yaml_path="${1}"
 	local kernel_name="${2}"
-	local extra_cmd=${3}
+	local extra_cmd="${3:-}"
 
 	export kernel_version="$(get_from_kata_deps ${kernel_version_yaml_path})"
 	export kernel_kata_config_version="$(cat ${repo_root_dir}/tools/packaging/kernel/kata_config_version)"
@@ -314,11 +313,6 @@ install_kernel_helper() {
 
 	install_cached_kernel_tarball_component ${kernel_name} ${module_dir} && return 0
 
-	if [ "${MEASURED_ROOTFS}" == "yes" ]; then
-		info "build initramfs for cc kernel"
-		"${initramfs_builder}"
-	fi
-
 	info "build ${kernel_name}"
 	info "Kernel version ${kernel_version}"
 	DESTDIR="${destdir}" PREFIX="${prefix}" "${kernel_builder}" -v "${kernel_version}" ${extra_cmd}
@@ -605,18 +599,7 @@ install_shimv2() {
 	export GO_VERSION
 	export RUST_VERSION
 
-	if [ "${MEASURED_ROOTFS}" == "yes" ]; then
-	        extra_opts="DEFSERVICEOFFLOAD=true"
-		if [ -f "${repo_root_dir}/tools/osbuilder/root_hash.txt" ]; then
-			root_hash=$(sudo sed -e 's/Root hash:\s*//g;t;d' "${repo_root_dir}/tools/osbuilder//root_hash.txt")
-			root_measure_config="rootfs_verity.scheme=dm-verity rootfs_verity.hash=${root_hash}"
-			extra_opts+=" ROOTMEASURECONFIG=\"${root_measure_config}\""
-		fi
-
-		DESTDIR="${destdir}" PREFIX="${prefix}" EXTRA_OPTS="${extra_opts}" "${shimv2_builder}"
-	else
-		DESTDIR="${destdir}" PREFIX="${prefix}" "${shimv2_builder}"
-	fi
+	DESTDIR="${destdir}" PREFIX="${prefix}" "${shimv2_builder}"
 }
 
 install_ovmf() {