Added kubeReserved calculation support for mixed instance NodeGroups

Support for calculating CPU, RAM, and Storage reservations via `kubeReserved` was added to eksctl in v0.16. However, it only supported NodeGroups with a single instance type. ASGs can have multiple instance types and were not included in the calculation. This change enables calculation of kubeReserved for mixed instances too. Since multiple instance types are supported it calculates the reservation based on the smallest instance type in the distribution. Additional kubeReserved tests covering unknown innstance types. The new tests ensure that unknown instance types are properly ignored during calculation of kubeReserved.
eksctl-io · Jul 6, 2020 · aa0c44c · aa0c44c
1 parent 6a108dc
commit aa0c44c
Show file tree

Hide file tree

Showing 3 changed files with 139 additions and 15 deletions.
diff --git a/pkg/nodebootstrap/userdata.go b/pkg/nodebootstrap/userdata.go
@@ -91,6 +91,14 @@ func clusterDNS(spec *api.ClusterConfig, ng *api.NodeGroup) string {
 	return "10.100.0.10"
 }
 
+func getKubeReserved(info InstanceTypeInfo) api.InlineDocument {
+	return api.InlineDocument{
+		"ephemeral-storage": info.DefaultStorageToReserve(),
+		"cpu":               info.DefaultCPUToReserve(),
+		"memory":            info.DefaultMemoryToReserve(),
+	}
+}
+
 func makeKubeletConfigYAML(spec *api.ClusterConfig, ng *api.NodeGroup) ([]byte, error) {
 	data, err := Asset("kubelet.yaml")
 	if err != nil {
@@ -110,12 +118,34 @@ func makeKubeletConfigYAML(spec *api.ClusterConfig, ng *api.NodeGroup) ([]byte,
 
 	// Set default reservations if specs about instance is available
 	if info, ok := instanceTypeInfos[ng.InstanceType]; ok {
+		// This is a NodeGroup with a single instanceType defined
 		if _, ok := obj["kubeReserved"]; !ok {
 			obj["kubeReserved"] = api.InlineDocument{}
 		}
-		obj["kubeReserved"].(api.InlineDocument)["ephemeral-storage"] = info.DefaultStorageToReserve()
-		obj["kubeReserved"].(api.InlineDocument)["cpu"] = info.DefaultCPUToReserve()
-		obj["kubeReserved"].(api.InlineDocument)["memory"] = info.DefaultMemoryToReserve()
+		obj["kubeReserved"] = getKubeReserved(info)
+	} else if ng.InstancesDistribution != nil {
+		// This is a NodeGroup using mixed instance types
+		var minCPU, minRAM int64
+		for _, instanceType := range ng.InstancesDistribution.InstanceTypes {
+			if info, ok := instanceTypeInfos[instanceType]; ok {
+				if instanceCPU := info.CPU; minCPU == 0 || instanceCPU < minCPU {
+					minCPU = instanceCPU
+				}
+				if instanceRAM := info.Memory; minRAM == 0 || instanceRAM < minRAM {
+					minRAM = instanceRAM
+				}
+			}
+		}
+		if minCPU > 0 && minRAM > 0 {
+			info = InstanceTypeInfo{
+				Memory: minRAM,
+				CPU:    minCPU,
+			}
+			if _, ok := obj["kubeReserved"]; !ok {
+				obj["kubeReserved"] = api.InlineDocument{}
+			}
+			obj["kubeReserved"] = getKubeReserved(info)
+		}
 	}
 
 	// Add extra configuration from configfile

diff --git a/pkg/nodebootstrap/userdata_test.go b/pkg/nodebootstrap/userdata_test.go
@@ -46,6 +46,17 @@ var _ = Describe("User data", func() {
 			Expect(errUnmarshal).ToNot(HaveOccurred())
 		})
 
+		It("does not contain default kube reservations for unknown instances", func() {
+			ng.InstanceType = "dne.small"
+			data, err := makeKubeletConfigYAML(clusterConfig, ng)
+			Expect(err).ToNot(HaveOccurred())
+
+			kubelet := kubeletapi.KubeletConfiguration{}
+			err = yaml.UnmarshalStrict(data, &kubelet)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(kubelet.KubeReserved).To(BeNil())
+		})
+
 		It("contains default kube reservations", func() {
 			ng.InstanceType = "i3.metal"
 			data, err := makeKubeletConfigYAML(clusterConfig, ng)
@@ -61,8 +72,76 @@ var _ = Describe("User data", func() {
 			}))
 		})
 
+		It("contains default kube reservations for mixed instance NGs", func() {
+			ng.InstancesDistribution = &api.NodeGroupInstancesDistribution{}
+			ng.InstancesDistribution.InstanceTypes = []string{
+				"c5.xlarge",
+				"c5.2xlarge",
+				"c5.4xlarge",
+			}
+			data, err := makeKubeletConfigYAML(clusterConfig, ng)
+			Expect(err).ToNot(HaveOccurred())
+
+			kubelet := kubeletapi.KubeletConfiguration{}
+			err = yaml.UnmarshalStrict(data, &kubelet)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(kubelet.KubeReserved).To(Equal(map[string]string{
+				"ephemeral-storage": "1Gi",
+				"cpu":               "80m",
+				"memory":            "1843Mi",
+			}))
+		})
+
+		It("contains default kube reservations for mixed instance NGs with at least one known instance type", func() {
+			ng.InstancesDistribution = &api.NodeGroupInstancesDistribution{}
+			ng.InstancesDistribution.InstanceTypes = []string{
+				"c5.xlarge",
+				"dne.small",
+				"dne.large",
+			}
+			data, err := makeKubeletConfigYAML(clusterConfig, ng)
+			Expect(err).ToNot(HaveOccurred())
+
+			kubelet := kubeletapi.KubeletConfiguration{}
+			err = yaml.UnmarshalStrict(data, &kubelet)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(kubelet.KubeReserved).To(Equal(map[string]string{
+				"ephemeral-storage": "1Gi",
+				"cpu":               "80m",
+				"memory":            "1843Mi",
+			}))
+		})
+
 		It("the kubelet config contains the overwritten values", func() {
+			ng.KubeletExtraConfig = &api.InlineDocument{
+				"kubeReserved": &map[string]string{
+					"cpu":               "300m",
+					"memory":            "300Mi",
+					"ephemeral-storage": "1Gi",
+				},
+				"featureGates": map[string]bool{
+					"HugePages":            false,
+					"DynamicKubeletConfig": true,
+				},
+			}
+			data, err := makeKubeletConfigYAML(clusterConfig, ng)
+			Expect(err).ToNot(HaveOccurred())
 
+			kubelet := &kubeletapi.KubeletConfiguration{}
+
+			errUnmarshal := yaml.UnmarshalStrict(data, kubelet)
+			Expect(errUnmarshal).ToNot(HaveOccurred())
+
+			Expect(kubelet.KubeReserved).ToNot(BeNil())
+			Expect(kubelet.KubeReserved["cpu"]).To(Equal("300m"))
+			Expect(kubelet.KubeReserved["memory"]).To(Equal("300Mi"))
+			Expect(kubelet.KubeReserved["ephemeral-storage"]).To(Equal("1Gi"))
+			Expect(kubelet.FeatureGates["HugePages"]).To(Equal(false))
+			Expect(kubelet.FeatureGates["DynamicKubeletConfig"]).To(Equal(true))
+			Expect(kubelet.FeatureGates["RotateKubeletServerCertificate"]).To(Equal(false))
+		})
+
+		It("the kubelet config contains the overwritten values for mixed instance NGs", func() {
 			ng.KubeletExtraConfig = &api.InlineDocument{
 				"kubeReserved": &map[string]string{
 					"cpu":               "300m",
@@ -74,6 +153,12 @@ var _ = Describe("User data", func() {
 					"DynamicKubeletConfig": true,
 				},
 			}
+			ng.InstancesDistribution = &api.NodeGroupInstancesDistribution{}
+			ng.InstancesDistribution.InstanceTypes = []string{
+				"c5.xlarge",
+				"c5.2xlarge",
+				"c5.4xlarge",
+			}
 			data, err := makeKubeletConfigYAML(clusterConfig, ng)
 			Expect(err).ToNot(HaveOccurred())
 

diff --git a/userdocs/src/usage/customizing-the-kubelet.md b/userdocs/src/usage/customizing-the-kubelet.md
@@ -2,18 +2,18 @@
 
 ## Customizing kubelet configuration
 
-System resources can be reserved through the configuration of the kubelet. This is recommended, because in the case 
+System resources can be reserved through the configuration of the kubelet. This is recommended, because in the case
 of resource starvation the kubelet might not be able to evict pods and eventually make the node become `NotReady`. To
- do this, config files can include the `kubeletExtraConfig` field which accepts a free form yaml that will be embedded 
+ do this, config files can include the `kubeletExtraConfig` field which accepts a free form yaml that will be embedded
  into the `kubelet.yaml`.
 
- 
-Some fields in the `kubelet.yaml` are set by eksctl and therefore are not overwritable, such as the `address`, 
+
+Some fields in the `kubelet.yaml` are set by eksctl and therefore are not overwritable, such as the `address`,
 `clusterDomain`, `authentication`, `authorization`, or `serverTLSBootstrap`.
 
-The following example config file creates a nodegroup that reserves `300m` vCPU, `300Mi` of memory and `1Gi` of 
-ephemeral-storage for the kubelet; `300m` vCPU, `300Mi` of memory and `1Gi`of ephemeral storage for OS system 
-daemons; and kicks in eviction of pods when there is less than `200Mi` of memory available or less than  10% of the 
+The following example config file creates a nodegroup that reserves `300m` vCPU, `300Mi` of memory and `1Gi` of
+ephemeral-storage for the kubelet; `300m` vCPU, `300Mi` of memory and `1Gi`of ephemeral storage for OS system
+daemons; and kicks in eviction of pods when there is less than `200Mi` of memory available or less than  10% of the
 root filesystem.
 
 ```yaml
@@ -48,12 +48,21 @@ nodeGroups:
 
 In this example, given instances of type `m5a.xlarge` which have 4 vCPUs and 16GiB of memory, the `Allocatable` amount
 of CPUs would be 3.4 and 15.4 GiB of memory. In addition, the `DynamicKubeletConfig` feature gate is also enabled. It is
-important to know that the values specified in the config file for the the fields in `kubeletExtraconfig` will 
+important to know that the values specified in the config file for the the fields in `kubeletExtraconfig` will
 completely overwrite the default values specified by eksctl. However, omitting one or more `kubeReserved` parameters
 will cause the missing parameters to be defaulted to sane values based on the aws instance type being used.
 
-!!!warning
-    By default `eksctl` sets `featureGates.RotateKubeletServerCertificate=true`, but when custom `featureGates` are provided, it will be unset. You should always include 
-    `featureGates.RotateKubeletServerCertificate=true`, unless you have to disable it.
-
+### A note on the `kubeReserved` calculation for NodeGroups with mixed instances
+
+While it is generally recommended to configure a mixed instance NodeGroup to use instances with the same CPU and RAM
+configuration; that's not a strict requirement. Therefore the `kubeReserved` calculation uses the _smallest instance_ in
+the `InstanceDistribution.InstanceTypes` field. This way NodeGroups with disparate instance types will not reserve too
+many resources on the smallest instance. However, this could lead to a reservation that is too small for the largest
+instance type.
+
+### Warning
+By default `eksctl` sets `featureGates.RotateKubeletServerCertificate=true`, but when custom `featureGates` are
+provided, it will be unset. You should always include `featureGates.RotateKubeletServerCertificate=true`, unless
+you have to disable it.
+