diff --git a/files/bootstrap.sh b/files/bootstrap.sh index 7d2ce8098..f9b0fafd8 100755 --- a/files/bootstrap.sh +++ b/files/bootstrap.sh @@ -222,6 +222,13 @@ ENABLE_LOCAL_OUTPOST="${ENABLE_LOCAL_OUTPOST:-}" CLUSTER_ID="${CLUSTER_ID:-}" LOCAL_DISKS="${LOCAL_DISKS:-}" +##allow --reserved-cpus options via kubelet arg directly. Disable default reserved cgroup option in such cases +USE_RESERVED_CGROUPS=true +if [[ ${KUBELET_EXTRA_ARGS} == *'--reserved-cpus'* ]]; then + USE_RESERVED_CGROUPS=false + log "INFO: --kubelet-extra-args includes --reserved-cpus, so kube/system-reserved cgroups will not be used." +fi + if [[ ! -z ${LOCAL_DISKS} ]]; then setup-local-disks "${LOCAL_DISKS}" fi @@ -565,8 +572,11 @@ if [[ "$CONTAINER_RUNTIME" = "containerd" ]]; then sudo sed -i s,SANDBOX_IMAGE,$PAUSE_CONTAINER,g /etc/eks/containerd/containerd-config.toml echo "$(jq '.cgroupDriver="systemd"' "${KUBELET_CONFIG}")" > "${KUBELET_CONFIG}" - echo "$(jq '.systemReservedCgroup="/system"' "${KUBELET_CONFIG}")" > "${KUBELET_CONFIG}" - echo "$(jq '.kubeReservedCgroup="/runtime"' "${KUBELET_CONFIG}")" > "${KUBELET_CONFIG}" + ##allow --reserved-cpus options via kubelet arg directly. Disable default reserved cgroup option in such cases + if [[ "${USE_RESERVED_CGROUPS}" = true ]]; then + echo "$(jq '.systemReservedCgroup="/system"' "${KUBELET_CONFIG}")" > "${KUBELET_CONFIG}" + echo "$(jq '.kubeReservedCgroup="/runtime"' "${KUBELET_CONFIG}")" > "${KUBELET_CONFIG}" + fi # Check if the containerd config file is the same as the one used in the image build. # If different, then restart containerd w/ proper config diff --git a/test/cases/reserved-cpus-kubelet-arg.sh b/test/cases/reserved-cpus-kubelet-arg.sh new file mode 100755 index 000000000..2002b7060 --- /dev/null +++ b/test/cases/reserved-cpus-kubelet-arg.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +set -euo pipefail + +echo "-> Should not set systemReservedCgroup and kubeReservedCgroup when --reserved-cpus is set with containerd" +exit_code=0 +export KUBELET_VERSION=v1.24.15-eks-ba74326 +/etc/eks/bootstrap.sh \ + --b64-cluster-ca dGVzdA== \ + --apiserver-endpoint http://my-api-endpoint \ + --kubelet-extra-args '--node-labels=cnf=cnf1 --reserved-cpus=0-3 --cpu-manager-policy=static' \ + test || exit_code=$? + +if [[ ${exit_code} -ne 0 ]]; then + echo "❌ Test Failed: expected a non-zero exit code but got '${exit_code}'" + exit 1 +fi + +KUBELET_CONFIG=/etc/kubernetes/kubelet/kubelet-config.json +if grep -q systemReservedCgroup ${KUBELET_CONFIG}; then + echo "❌ Test Failed: expected systemReservedCgroup to be absent in ${KUBELET_CONFIG}.Found: $(grep systemReservedCgroup ${KUBELET_CONFIG})" + exit 1 +fi + +if grep -q kubeReservedCgroup ${KUBELET_CONFIG}; then + echo "❌ Test Failed: expected kubeReservedCgroup to be absent ${KUBELET_CONFIG}.Found: $(grep kubeReservedCgroup ${KUBELET_CONFIG})" + exit 1 +fi + +echo "-> Should set systemReservedCgroup and kubeReservedCgroup when --reserved-cpus is not set with containerd" +exit_code=0 +export KUBELET_VERSION=v1.24.15-eks-ba74326 +/etc/eks/bootstrap.sh \ + --b64-cluster-ca dGVzdA== \ + --apiserver-endpoint http://my-api-endpoint \ + test || exit_code=$? + +if [[ ${exit_code} -ne 0 ]]; then + echo "❌ Test Failed: expected a non-zero exit code but got '${exit_code}'" + exit 1 +fi + +if ! $(grep -q systemReservedCgroup ${KUBELET_CONFIG}); then + echo "❌ Test Failed: expected systemReservedCgroup to be present in ${KUBELET_CONFIG}. Found: $(grep systemReservedCgroup ${KUBELET_CONFIG})" + exit 1 +fi + +if ! $(grep -q kubeReservedCgroup ${KUBELET_CONFIG}); then + echo "❌ Test Failed: expected kubeReservedCgroup to be present ${KUBELET_CONFIG}.Found: $(grep kubeReservedCgroup ${KUBELET_CONFIG})" + exit 1 +fi + +echo "-> Should set systemReservedCgroup and kubeReservedCgroup when --reserved-cpus is set with dockerd" +exit_code=0 +export KUBELET_VERSION=v1.23.15-eks-ba74326 +/etc/eks/bootstrap.sh \ + --b64-cluster-ca dGVzdA== \ + --apiserver-endpoint http://my-api-endpoint \ + test || exit_code=$? + +if [[ ${exit_code} -ne 0 ]]; then + echo "❌ Test Failed: expected a non-zero exit code but got '${exit_code}'" + exit 1 +fi + +if ! $(grep -q systemReservedCgroup ${KUBELET_CONFIG}); then + echo "❌ Test Failed: expected systemReservedCgroup to be present in ${KUBELET_CONFIG}.Found: $(grep systemReservedCgroup ${KUBELET_CONFIG})" + exit 1 +fi + +if ! $(grep -q kubeReservedCgroup ${KUBELET_CONFIG}); then + echo "❌ Test Failed: expected kubeReservedCgroup to be present ${KUBELET_CONFIG}.Found: $(grep kubeReservedCgroup ${KUBELET_CONFIG})" + exit 1 +fi