Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

gha: kata-deploy: Revert containerd config break #8679

Merged
merged 2 commits into from
Dec 20, 2023
Merged

gha: kata-deploy: Revert containerd config break #8679

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9e718b4
gha: kata-deploy: Add containerd status check
stevenhorsman Dec 15, 2023
ee5fa08
Revert "kata-deploy: Use tomlq to configure containerd"
stevenhorsman Dec 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
28 changes: 21 additions & 7 deletions tests/functional/kata-deploy/kata-deploy.bats
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,17 +36,17 @@ setup() {
yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[4].value' --tag '!!str' "true"
# Let the `kata-deploy` create the default `kata` runtime class
yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[5].value' --tag '!!str' "true"

if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then
yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[+].name' "HOST_OS"
yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[-1].value' "${KATA_HOST_OS}"
fi

echo "::group::Final kata-deploy.yaml that is used in the test"
cat "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" || die "Failed to setup the tests image"
echo "::endgroup::"

kubectl apply -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
if [ "${KUBERNETES}" = "k0s" ]; then
kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k0s"
Expand All @@ -64,7 +64,7 @@ setup() {
sleep 30s
}

@test "Test runtimeclasses are being properly created" {
@test "Test runtimeclasses are being properly created and container runtime not broken" {
# We filter `kata-mshv-vm-isolation` out as that's present on AKS clusters, but that's not coming from kata-deploy
current_runtime_classes=$(kubectl get runtimeclasses | grep -v "kata-mshv-vm-isolation" | grep "kata" | wc -l)
[[ ${current_runtime_classes} -eq ${expected_runtime_classes} ]]
Expand All @@ -73,6 +73,20 @@ setup() {
do
kubectl get runtimeclass | grep -E "${handler_re}"
done

# Ensure that kata-deploy didn't corrupt containerd config, by trying to get the container runtime and node status
echo "::group::kubectl node debug"
kubectl get node -o wide
kubectl describe nodes
echo "::endgroup::"

# Wait to see if the nodes get back into Ready state - if not then containerd might be having issues
kubectl wait nodes --timeout=60s --all --for condition=Ready=True

# Check that the container runtime verison doesn't have unknown, which happens when containerd can't start properly
container_runtime_version=$(kubectl get nodes --no-headers -o custom-columns=CONTAINER_RUNTIME:.status.nodeInfo.containerRuntimeVersion)
[[ ${container_runtime_version} != *"containerd://Unknown"* ]]

}

teardown() {
Expand All @@ -94,7 +108,7 @@ teardown() {

kubectl delete ${deploy_spec}
kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod

# Let the `kata-deploy` script take care of the runtime class creation / removal
yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" 'spec.template.spec.containers[0].env[4].value' --tag '!!str' "true"
# Create the runtime class only for the shim that's being tested
Expand All @@ -103,14 +117,14 @@ teardown() {
yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" 'spec.template.spec.containers[0].env[3].value' "${KATA_HYPERVISOR}"
# Let the `kata-deploy` create the default `kata` runtime class
yq write -i "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[5].value' --tag '!!str' "true"

sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
cat "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" || die "Failed to setup the tests image"

kubectl apply ${cleanup_spec}
sleep 30s

kubectl delete ${cleanup_spec}
kubectl delete -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
}
54 changes: 39 additions & 15 deletions tools/packaging/kata-deploy/scripts/kata-deploy.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,8 +94,8 @@ function get_container_runtime() {
else
echo "k3s"
fi
# Note: we assumed you used a conventional k0s setup and k0s will generate a systemd entry k0scontroller.service and k0sworker.service respectively
# and it is impossible to run this script without a kubelet, so this k0s controller must also have worker mode enabled
# Note: we assumed you used a conventional k0s setup and k0s will generate a systemd entry k0scontroller.service and k0sworker.service respectively
# and it is impossible to run this script without a kubelet, so this k0s controller must also have worker mode enabled
elif host_systemctl is-active --quiet k0scontroller; then
echo "k0s-controller"
elif host_systemctl is-active --quiet k0sworker; then
Expand Down Expand Up @@ -344,24 +344,48 @@ function configure_containerd_runtime() {
local runtime="kata-${shim}"
local configuration="configuration-${shim}"
local pluginid=cri

# if we are running k0s auto containerd.toml generation, the base template is by default version 2
# we can safely assume to reference the newer version of cri
if grep -q "version = 2\>" $containerd_conf_file || [ "$1" == "k0s-worker" ] || [ "$1" == "k0s-controller" ]; then
pluginid=\"io.containerd.grpc.v1.cri\"
fi
local runtime_table=".plugins.${pluginid}.containerd.runtimes.\"${runtime}\""
local runtime_options_table="${runtime_table}.options"
local runtime_type=\"io.containerd."${runtime}".v2\"
local runtime_config_path=\"$(get_kata_containers_config_path "${shim}")/${configuration}.toml\"

tomlq -i -t $(printf '%s.runtime_type=%s' ${runtime_table} ${runtime_type}) ${containerd_conf_file}
tomlq -i -t $(printf '%s.privileged_without_host_devices=true' ${runtime_table}) ${containerd_conf_file}
tomlq -i -t $(printf '%s.pod_annotations=["io.katacontainers.*"]' ${runtime_table}) ${containerd_conf_file}
tomlq -i -t $(printf '%s.ConfigPath=%s' ${runtime_options_table} ${runtime_config_path}) ${containerd_conf_file}

local runtime_table="plugins.${pluginid}.containerd.runtimes.$runtime"
local runtime_type="io.containerd.$runtime.v2"
local options_table="$runtime_table.options"
local config_path="$(get_kata_containers_config_path "${shim}")/$configuration.toml"
if grep -q "\[$runtime_table\]" $containerd_conf_file; then
echo "Configuration exists for $runtime_table, overwriting"
sed -i "/\[$runtime_table\]/,+1s#runtime_type.*#runtime_type = \"${runtime_type}\"#" $containerd_conf_file
else
cat <<EOF | tee -a "$containerd_conf_file"
[$runtime_table]
runtime_type = "${runtime_type}"
privileged_without_host_devices = true
pod_annotations = ["io.katacontainers.*"]
EOF
fi

if grep -q "\[$options_table\]" $containerd_conf_file; then
echo "Configuration exists for $options_table, overwriting"
sed -i "/\[$options_table\]/,+1s#ConfigPath.*#ConfigPath = \"${config_path}\"#" $containerd_conf_file
else
cat <<EOF | tee -a "$containerd_conf_file"
[$options_table]
ConfigPath = "${config_path}"
EOF
fi

if [ "${DEBUG}" == "true" ]; then
tomlq -i -t '.debug.level = "debug"' ${containerd_conf_file}
if grep -q "\[debug\]" $containerd_conf_file; then
sed -i 's/level.*/level = \"debug\"/' $containerd_conf_file
else
cat <<EOF | tee -a "$containerd_conf_file"
[debug]
level = "debug"
EOF
fi

fi
}

Expand Down Expand Up @@ -461,7 +485,7 @@ function main() {
containerd_conf_file="${containerd_conf_tmpl_file}"
containerd_conf_file_backup="${containerd_conf_file}.bak"
elif [ "$runtime" == "k0s-worker" ] || [ "$runtime" == "k0s-controller" ]; then
# From 1.27.1 onwards k0s enables dynamic configuration on containerd CRI runtimes.
# From 1.27.1 onwards k0s enables dynamic configuration on containerd CRI runtimes.
# This works by k0s creating a special directory in /etc/k0s/containerd.d/ where user can drop-in partial containerd configuration snippets.
# k0s will automatically pick up these files and adds these in containerd configuration imports list.
containerd_conf_file="/etc/containerd/kata-containers.toml"
Expand Down