Added kubeReserved calculation support for Spot Fleet

Support for calculating CPU, RAM, and Storage reservations via
`kubeReserved` was added to eksctl in v0.16. However, it only supported
NodeGroups with a single instance type. Fleets can have multiple
instance types and were not included in the calculation.

This change enables calculation of kubeReserved for Fleets too. Since
multiple instance types are supported it calculates the reservation
based on the smallest instance type in the distribution.

pkg/nodebootstrap/userdata.go

@@ -110,12 +110,38 @@ func makeKubeletConfigYAML(spec *api.ClusterConfig, ng *api.NodeGroup) ([]byte,
 
 	// Set default reservations if specs about instance is available
 	if info, ok := instanceTypeInfos[ng.InstanceType]; ok {
+		// This is a NodeGroup with a single instanceType defined
 		if _, ok := obj["kubeReserved"]; !ok {
 			obj["kubeReserved"] = api.InlineDocument{}
 		}
 		obj["kubeReserved"].(api.InlineDocument)["ephemeral-storage"] = info.DefaultStorageToReserve()
 		obj["kubeReserved"].(api.InlineDocument)["cpu"] = info.DefaultCPUToReserve()
 		obj["kubeReserved"].(api.InlineDocument)["memory"] = info.DefaultMemoryToReserve()
+	} else if ng.InstancesDistribution != nil {
+		// This is a NodeGroup using Spot Fleet
+		var minCPU, minRAM int64
+		for _, instanceType := range ng.InstancesDistribution.InstanceTypes {
+			if info, ok := instanceTypeInfos[instanceType]; ok {
+				if instanceCPU := info.CPU; minCPU == 0 || instanceCPU < minCPU {
+					minCPU = instanceCPU
+				}
+				if instanceRAM := info.Memory; minRAM == 0 || instanceRAM < minRAM {
+					minRAM = instanceRAM
+				}
+			}
+		}
+		if minCPU > 0 && minRAM > 0 {
+			info = InstanceTypeInfo{
+				Memory: minRAM,
+				CPU:    minCPU,
+			}
+			if _, ok := obj["kubeReserved"]; !ok {
+				obj["kubeReserved"] = api.InlineDocument{}
+			}
+			obj["kubeReserved"].(api.InlineDocument)["ephemeral-storage"] = info.DefaultStorageToReserve()
+			obj["kubeReserved"].(api.InlineDocument)["cpu"] = info.DefaultCPUToReserve()
+			obj["kubeReserved"].(api.InlineDocument)["memory"] = info.DefaultMemoryToReserve()
+		}
 	}
 
 	// Add extra configuration from configfile

pkg/nodebootstrap/userdata_test.go

@@ -61,8 +61,56 @@ var _ = Describe("User data", func() {
 			}))
 		})
 
+		It("contains default kube reservations for fleets", func() {
+			ng.InstancesDistribution = &api.NodeGroupInstancesDistribution{}
+			ng.InstancesDistribution.InstanceTypes = []string{
+				"c5.xlarge",
+				"c5.2xlarge",
+				"c5.4xlarge",
+			}
+			data, err := makeKubeletConfigYAML(clusterConfig, ng)
+			Expect(err).ToNot(HaveOccurred())
+
+			kubelet := kubeletapi.KubeletConfiguration{}
+			err = yaml.UnmarshalStrict(data, &kubelet)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(kubelet.KubeReserved).To(Equal(map[string]string{
+				"ephemeral-storage": "1Gi",
+				"cpu":               "80m",
+				"memory":            "1843Mi",
+			}))
+		})
+
 		It("the kubelet config contains the overwritten values", func() {
+			ng.KubeletExtraConfig = &api.InlineDocument{
+				"kubeReserved": &map[string]string{
+					"cpu":               "300m",
+					"memory":            "300Mi",
+					"ephemeral-storage": "1Gi",
+				},
+				"featureGates": map[string]bool{
+					"HugePages":            false,
+					"DynamicKubeletConfig": true,
+				},
+			}
+			data, err := makeKubeletConfigYAML(clusterConfig, ng)
+			Expect(err).ToNot(HaveOccurred())
+
+			kubelet := &kubeletapi.KubeletConfiguration{}
+
+			errUnmarshal := yaml.UnmarshalStrict(data, kubelet)
+			Expect(errUnmarshal).ToNot(HaveOccurred())
 
+			Expect(kubelet.KubeReserved).ToNot(BeNil())
+			Expect(kubelet.KubeReserved["cpu"]).To(Equal("300m"))
+			Expect(kubelet.KubeReserved["memory"]).To(Equal("300Mi"))
+			Expect(kubelet.KubeReserved["ephemeral-storage"]).To(Equal("1Gi"))
+			Expect(kubelet.FeatureGates["HugePages"]).To(Equal(false))
+			Expect(kubelet.FeatureGates["DynamicKubeletConfig"]).To(Equal(true))
+			Expect(kubelet.FeatureGates["RotateKubeletServerCertificate"]).To(Equal(false))
+		})
+
+		It("the kubelet config contains the overwritten values for fleets", func() {
 			ng.KubeletExtraConfig = &api.InlineDocument{
 				"kubeReserved": &map[string]string{
 					"cpu":               "300m",
@@ -74,6 +122,12 @@ var _ = Describe("User data", func() {
 					"DynamicKubeletConfig": true,
 				},
 			}
+			ng.InstancesDistribution = &api.NodeGroupInstancesDistribution{}
+			ng.InstancesDistribution.InstanceTypes = []string{
+				"c5.xlarge",
+				"c5.2xlarge",
+				"c5.4xlarge",
+			}
 			data, err := makeKubeletConfigYAML(clusterConfig, ng)
 			Expect(err).ToNot(HaveOccurred())
 

userdocs/src/usage/customizing-the-kubelet.md

@@ -2,18 +2,18 @@
 
 ## Customizing kubelet configuration
 
-System resources can be reserved through the configuration of the kubelet. This is recommended, because in the case 
+System resources can be reserved through the configuration of the kubelet. This is recommended, because in the case
 of resource starvation the kubelet might not be able to evict pods and eventually make the node become `NotReady`. To
- do this, config files can include the `kubeletExtraConfig` field which accepts a free form yaml that will be embedded 
+ do this, config files can include the `kubeletExtraConfig` field which accepts a free form yaml that will be embedded
  into the `kubelet.yaml`.
 
- 
-Some fields in the `kubelet.yaml` are set by eksctl and therefore are not overwritable, such as the `address`, 
+
+Some fields in the `kubelet.yaml` are set by eksctl and therefore are not overwritable, such as the `address`,
 `clusterDomain`, `authentication`, `authorization`, or `serverTLSBootstrap`.
 
-The following example config file creates a nodegroup that reserves `300m` vCPU, `300Mi` of memory and `1Gi` of 
-ephemeral-storage for the kubelet; `300m` vCPU, `300Mi` of memory and `1Gi`of ephemeral storage for OS system 
-daemons; and kicks in eviction of pods when there is less than `200Mi` of memory available or less than  10% of the 
+The following example config file creates a nodegroup that reserves `300m` vCPU, `300Mi` of memory and `1Gi` of
+ephemeral-storage for the kubelet; `300m` vCPU, `300Mi` of memory and `1Gi`of ephemeral storage for OS system
+daemons; and kicks in eviction of pods when there is less than `200Mi` of memory available or less than  10% of the
 root filesystem.
 
 ```yaml
@@ -48,12 +48,20 @@ nodeGroups:
 
 In this example, given instances of type `m5a.xlarge` which have 4 vCPUs and 16GiB of memory, the `Allocatable` amount
 of CPUs would be 3.4 and 15.4 GiB of memory. In addition, the `DynamicKubeletConfig` feature gate is also enabled. It is
-important to know that the values specified in the config file for the the fields in `kubeletExtraconfig` will 
+important to know that the values specified in the config file for the the fields in `kubeletExtraconfig` will
 completely overwrite the default values specified by eksctl. However, omitting one or more `kubeReserved` parameters
 will cause the missing parameters to be defaulted to sane values based on the aws instance type being used.
 
+### A note on the `kubeReserved` calculation for Spot Fleets
+
+While it is generally recommended to configure a Fleet to use instances with the same CPU and RAM configuration; that's
+not a strict requirement. Therefore the `kubeReserved` calculation uses the _smallest instance_ in the
+`InstanceDistribution.InstanceTypes` field. This way Fleets with disparate instance types will not reserve too many
+resources on the smallest instance. However, this could lead to a reservation that is too small for the largest instance
+type.
+
 !!!warning
-    By default `eksctl` sets `featureGates.RotateKubeletServerCertificate=true`, but when custom `featureGates` are provided, it will be unset. You should always include 
+    By default `eksctl` sets `featureGates.RotateKubeletServerCertificate=true`, but when custom `featureGates` are provided, it will be unset. You should always include
     `featureGates.RotateKubeletServerCertificate=true`, unless you have to disable it.
- 
+