Adding AL2 UserData Merge Support

pkg/cloudprovider/aws/amifamily/al2.go

@@ -58,6 +58,7 @@ func (a AL2) UserData(kubeletConfig *v1alpha5.KubeletConfiguration, taints []cor
 			Taints:                  taints,
 			Labels:                  labels,
 			CABundle:                caBundle,
+			CustomUserData:          customUserData,
 		},
 	}
 }

pkg/cloudprovider/aws/amifamily/bootstrap/eksbootstrap.go

@@ -17,7 +17,14 @@ package bootstrap
 import (
 	"bytes"
 	"encoding/base64"
+	"errors"
 	"fmt"
+	"io"
+	"mime"
+	"mime/multipart"
+	"net/mail"
+	"net/textproto"
+	"sort"
 	"strings"
 	"sync"
 
@@ -29,6 +36,12 @@ type EKS struct {
 	ContainerRuntime string
 }
 
+const (
+	Boundary                      = "//"
+	MIMEVersionHeader             = "MIME-Version: 1.0"
+	MIMEContentTypeHeaderTemplate = "Content-Type: multipart/mixed; boundary=\"%s\""
+)
+
 func (e EKS) Script() (string, error) {
 	var caBundleArg string
 	if e.CABundle != nil {
@@ -55,7 +68,15 @@ func (e EKS) Script() (string, error) {
 	if e.KubeletConfig != nil && len(e.KubeletConfig.ClusterDNS) > 0 {
 		userData.WriteString(fmt.Sprintf(" \\\n--dns-cluster-ip '%s'", e.KubeletConfig.ClusterDNS[0]))
 	}
-	return base64.StdEncoding.EncodeToString(userData.Bytes()), nil
+	userData, err := e.mergeCustomUserData(userData)
+	if err != nil {
+		return "", err
+	}
+	// The mime/multipart package adds carriage returns, while the rest of our logic does not. Remove all
+	// carriage returns for consistency.
+	userDataBytes := bytes.Replace(userData.Bytes(), []byte{13}, []byte{}, -1)
+	userDataString := base64.StdEncoding.EncodeToString(userDataBytes)
+	return userDataString, nil
 }
 
 func (e EKS) nodeTaintArg() string {
@@ -73,12 +94,91 @@ func (e EKS) nodeLabelArg() string {
 	nodeLabelArg := ""
 	labelStrings := []string{}
 	var once sync.Once
-	for k, v := range e.Labels {
-		if v1alpha5.LabelDomainExceptions.Has(k) {
+	keys := make([]string, 0, len(e.Labels))
+	for k := range e.Labels {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys) // ensures this list is deterministic, for easy testing.
+	for _, key := range keys {
+		if v1alpha5.LabelDomainExceptions.Has(key) {
 			continue
 		}
 		once.Do(func() { nodeLabelArg = "--node-labels=" })
-		labelStrings = append(labelStrings, fmt.Sprintf("%s=%v", k, v))
+		labelStrings = append(labelStrings, fmt.Sprintf("%s=%v", key, e.Labels[key]))
 	}
 	return fmt.Sprintf("%s%s", nodeLabelArg, strings.Join(labelStrings, ","))
 }
+
+func (e EKS) mergeCustomUserData(userData bytes.Buffer) (bytes.Buffer, error) {
+	var outputBuffer bytes.Buffer
+	writer := multipart.NewWriter(&outputBuffer)
+	if err := writer.SetBoundary(Boundary); err != nil {
+		return outputBuffer, fmt.Errorf("defining boundary for merged user data %w", err)
+	}
+	outputBuffer.WriteString(MIMEVersionHeader + "\n")
+	outputBuffer.WriteString(fmt.Sprintf(MIMEContentTypeHeaderTemplate, Boundary) + "\n\n")
+	// Step 1 - Copy over customer bootstrapping
+	if err := copyCustomUserDataParts(writer, e.Options.CustomUserData); err != nil {
+		return outputBuffer, err
+	}
+	// Step 2 - Add Karpenter's bootstrapping logic
+	shellScriptContentHeader := textproto.MIMEHeader{"Content-Type": []string{"text/x-shellscript; charset=\"us-ascii\""}}
+	partWriter, err := writer.CreatePart(shellScriptContentHeader)
+	if err != nil {
+		return outputBuffer, fmt.Errorf("unable to add Karpenter managed user data %w", err)
+	}
+	_, err = partWriter.Write(userData.Bytes())
+	if err != nil {
+		return outputBuffer, fmt.Errorf("unable to create merged user data content %w", err)
+	}
+	writer.Close()
+	return outputBuffer, nil
+}
+
+func copyCustomUserDataParts(writer *multipart.Writer, customUserData *string) error {
+	if customUserData == nil || *customUserData == "" {
+		// No custom user data specified, so nothing to copy over.
+		return nil
+	}
+	reader, err := getMultiPartReader(*customUserData)
+	if err != nil {
+		return fmt.Errorf("parsing custom user data input %w", err)
+	}
+	for {
+		p, err := reader.NextPart()
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			return fmt.Errorf("parsing custom user data input %w", err)
+		}
+		slurp, err := io.ReadAll(p)
+		if err != nil {
+			return fmt.Errorf("parsing custom user data input %w", err)
+		}
+		partWriter, err := writer.CreatePart(p.Header)
+		if err != nil {
+			return fmt.Errorf("parsing custom user data input %w", err)
+		}
+		_, err = partWriter.Write(slurp)
+		if err != nil {
+			return fmt.Errorf("parsing custom user data input %w", err)
+		}
+	}
+	return nil
+}
+
+func getMultiPartReader(userData string) (*multipart.Reader, error) {
+	mailMsg, err := mail.ReadMessage(strings.NewReader(userData))
+	if err != nil {
+		return nil, fmt.Errorf("unreadable user data %w", err)
+	}
+	mediaType, params, err := mime.ParseMediaType(mailMsg.Header.Get("Content-Type"))
+	if err != nil {
+		return nil, fmt.Errorf("user data does not define a content-type header %w", err)
+	}
+	if !strings.HasPrefix(mediaType, "multipart/") {
+		return nil, fmt.Errorf("user data is not in multipart MIME format")
+	}
+	return multipart.NewReader(mailMsg.Body, params["boundary"]), nil
+}

pkg/cloudprovider/aws/amifamily/ubuntu.go

@@ -46,6 +46,7 @@ func (u Ubuntu) UserData(kubeletConfig *v1alpha5.KubeletConfiguration, taints []
 			Taints:                  taints,
 			Labels:                  labels,
 			CABundle:                caBundle,
+			CustomUserData:          customUserData,
 		},
 	}
 }

pkg/cloudprovider/aws/suite_test.go

@@ -860,6 +860,73 @@ var _ = Describe("Allocation", func() {
 					ExpectNotScheduled(ctx, env.Client, pod)
 				})
 			})
+			Context("AL2 Custom UserData", func() {
+				It("should merge in custom user data", func() {
+					opts.AWSENILimitedPodDensity = false
+					provider, _ := v1alpha1.Deserialize(provisioner.Spec.Provider)
+					content, _ := ioutil.ReadFile("testdata/al2_userdata_input.golden")
+					providerRefName := strings.ToLower(randomdata.SillyName())
+					providerRef := &v1alpha5.ProviderRef{
+						Name: providerRefName,
+					}
+					nodeTemplate := test.AWSNodeTemplate(test.AWSNodeTemplateOptions{
+						UserData:   aws.String(string(content)),
+						ObjectMeta: metav1.ObjectMeta{Name: providerRefName}})
+					ExpectApplied(ctx, env.Client, nodeTemplate)
+					controller = provisioning.NewController(injection.WithOptions(ctx, opts), cfg, env.Client, clientSet.CoreV1(), recorder, cloudProvider, cluster)
+					newProvisioner := test.Provisioner(test.ProvisionerOptions{Provider: provider, ProviderRef: providerRef})
+					ExpectApplied(ctx, env.Client, newProvisioner)
+					pod := ExpectProvisioned(ctx, env.Client, controller, test.UnschedulablePod())[0]
+					ExpectScheduled(ctx, env.Client, pod)
+					Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(1))
+					input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop().(*ec2.CreateLaunchTemplateInput)
+					userData, _ := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData)
+					content, _ = ioutil.ReadFile("testdata/al2_userdata_merged.golden")
+					expectedUserData := fmt.Sprintf(string(content), newProvisioner.Name)
+					Expect(expectedUserData).To(Equal(string(userData)))
+				})
+				It("should handle empty custom user data", func() {
+					opts.AWSENILimitedPodDensity = false
+					provider, _ := v1alpha1.Deserialize(provisioner.Spec.Provider)
+					providerRefName := strings.ToLower(randomdata.SillyName())
+					providerRef := &v1alpha5.ProviderRef{
+						Name: providerRefName,
+					}
+					nodeTemplate := test.AWSNodeTemplate(test.AWSNodeTemplateOptions{
+						UserData:   nil,
+						ObjectMeta: metav1.ObjectMeta{Name: providerRefName}})
+					ExpectApplied(ctx, env.Client, nodeTemplate)
+					controller = provisioning.NewController(injection.WithOptions(ctx, opts), cfg, env.Client, clientSet.CoreV1(), recorder, cloudProvider, cluster)
+					newProvisioner := test.Provisioner(test.ProvisionerOptions{Provider: provider, ProviderRef: providerRef})
+					ExpectApplied(ctx, env.Client, newProvisioner)
+					pod := ExpectProvisioned(ctx, env.Client, controller, test.UnschedulablePod())[0]
+					ExpectScheduled(ctx, env.Client, pod)
+					Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(1))
+					input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop().(*ec2.CreateLaunchTemplateInput)
+					userData, _ := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData)
+					content, _ := ioutil.ReadFile("testdata/al2_userdata_unmerged.golden")
+					expectedUserData := fmt.Sprintf(string(content), newProvisioner.Name)
+					Expect(expectedUserData).To(Equal(string(userData)))
+				})
+				It("should not bootstrap invalid MIME UserData", func() {
+					opts.AWSENILimitedPodDensity = false
+					provider, _ := v1alpha1.Deserialize(provisioner.Spec.Provider)
+					providerRefName := strings.ToLower(randomdata.SillyName())
+					providerRef := &v1alpha5.ProviderRef{
+						Name: providerRefName,
+					}
+					nodeTemplate := test.AWSNodeTemplate(test.AWSNodeTemplateOptions{
+						UserData:   aws.String("#/bin/bash\n ./not-mime.sh"),
+						ObjectMeta: metav1.ObjectMeta{Name: providerRefName}})
+					ExpectApplied(ctx, env.Client, nodeTemplate)
+					controller = provisioning.NewController(injection.WithOptions(ctx, opts), cfg, env.Client, clientSet.CoreV1(), recorder, cloudProvider, cluster)
+					newProvisioner := test.Provisioner(test.ProvisionerOptions{Provider: provider, ProviderRef: providerRef})
+					ExpectApplied(ctx, env.Client, newProvisioner)
+					pod := ExpectProvisioned(ctx, env.Client, controller, test.UnschedulablePod())[0]
+					// This will not be scheduled since userData cannot be generated for the prospective node.
+					ExpectNotScheduled(ctx, env.Client, pod)
+				})
+			})
 			Context("Kubelet Args", func() {
 				It("should specify the --dns-cluster-ip flag when clusterDNSIP is set", func() {
 					ExpectApplied(ctx, env.Client, test.Provisioner(test.ProvisionerOptions{

pkg/cloudprovider/aws/testdata/al2_userdata_input.golden

@@ -0,0 +1,10 @@
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="BOUNDARY"
+
+--BOUNDARY
+Content-Type: text/x-shellscript; charset="us-ascii"
+
+#!/bin/bash
+echo "Running custom user data script"
+
+--BOUNDARY--

pkg/cloudprovider/aws/testdata/al2_userdata_merged.golden

@@ -0,0 +1,19 @@
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="//"
+
+--//
+Content-Type: text/x-shellscript; charset="us-ascii"
+
+#!/bin/bash
+echo "Running custom user data script"
+
+--//
+Content-Type: text/x-shellscript; charset="us-ascii"
+
+#!/bin/bash -xe
+exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
+/etc/eks/bootstrap.sh 'test-cluster' --apiserver-endpoint 'https://test-cluster' --b64-cluster-ca 'ca-bundle' \
+--use-max-pods false \
+--container-runtime containerd \
+--kubelet-extra-args '--node-labels=karpenter.sh/capacity-type=on-demand,karpenter.sh/provisioner-name=%s  --max-pods=110'
+--//--

pkg/cloudprovider/aws/testdata/al2_userdata_unmerged.golden

@@ -0,0 +1,13 @@
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="//"
+
+--//
+Content-Type: text/x-shellscript; charset="us-ascii"
+
+#!/bin/bash -xe
+exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
+/etc/eks/bootstrap.sh 'test-cluster' --apiserver-endpoint 'https://test-cluster' --b64-cluster-ca 'ca-bundle' \
+--use-max-pods false \
+--container-runtime containerd \
+--kubelet-extra-args '--node-labels=karpenter.sh/capacity-type=on-demand,karpenter.sh/provisioner-name=%s  --max-pods=110'
+--//--

website/content/en/preview/AWS/provisioning.md

@@ -16,11 +16,11 @@ kind: Provisioner
 metadata:
   name: default
 spec:
-  requirements:                      
+  requirements:
     - key: karpenter.sh/capacity-type         # optional, set to on-demand by default, spot if both are listed
       operator: In
       values: ["spot"]
-  limits:   
+  limits:
     resources:
       cpu: 1000                               # optional, recommended to limit total provisioned CPUs
       memory: 1000Gi                          # optional, recommended to limit total provisioned memory
@@ -243,75 +243,10 @@ spec:
 
 ### UserData
 
-In order to specify custom user data, you must include it within the AWSNodeTemplate resource. You can then reference the AWSNodeTemplate resource through `spec.providerRef` in your provisioner.
+You can control the UserData that needs to be applied to your worker nodes via the `spec.providerRef` field in your provisioner.
+Review the [Custom UserData documentation](../user-data/) to learn the necessary steps.
 
-**Examples**
-
-Your UserData can be added to `spec.userData` in the `AWSNodeTemplate` resource like this -
-```
-apiVersion: karpenter.k8s.aws/v1alpha1
-kind: AWSNodeTemplate
-metadata:
-  name: mynodetemplate
-spec:
-  userData:  |
-    [settings.kubernetes]
-    kube-api-qps = 30
-    [settings.kubernetes.eviction-hard]
-    "memory.available" = "20%"
-```
-
-The AWSNodeTemplate CRD can then be referenced within the provisioner through `providerRef` -
-```
-spec:
-  provider:
-    amiFamily: Bottlerocket
-    instanceProfile: MyInstanceProfile
-    subnetSelector:
-      karpenter.sh/discovery: my-cluster
-    securityGroupSelector:
-      karpenter.sh/discovery: my-cluster
-  providerRef:
-    name: mynodetemplate
-```
-
-*Supporting UserData via the providerRef is currently only supported for the Bottlerocket AMIFamily*.
-
-**Semantics for Bottlerocket**
-* Your UserData must be valid TOML.
-* Karpenter will automatically merge settings to ensure successful bootstrap including `cluster-name`, `api-server` and `cluster-certificate`. Any labels and taints that need to be set based on pod requirements will also be specified in the final merged UserData.
-   * All Kubelet settings that Karpenter applies will override the corresponding settings in the provided UserData. For example, if you've specified `settings.kubernetes.cluster-name`, it will be overridden.
-   * If MaxPods is specified via the binary arg to Karpenter, the value will override anything specified in the UserData.
-   * If ClusterDNS is specified via `spec.kubeletConfiguration`, then that value will override anything specified in the UserData.
-* Unknown TOML fields will be ignored when the final merged UserData is generated by Karpenter.
-
-Consider the following example to understand how your customUserData settings will be merged in.
-
-Your UserData -
-```toml
-[settings.kubernetes.eviction-hard]
-"memory.available" = "12%"
-[settings.kubernetes]
-"unknown-setting" = "unknown"
-```
-
-Final merged UserData -
-```toml
-[settings]
-[settings.kubernetes]
-api-server = 'https://cluster'
-cluster-certificate = 'ca-bundle'
-cluster-name = 'cluster'
-
-[settings.kubernetes.node-labels]
-'karpenter.sh/capacity-type' = 'on-demand'
-'karpenter.sh/provisioner-name' = 'provisioner'
-
-[settings.kubernetes.node-taints]
-
-[settings.kubernetes.eviction-hard]
-'memory.available' = '12%%'
-```
+If you need to specify a launch template in addition to UserData, then review the [Launch Template documentation](../launch-templates/) instead and utilize the `spec.provider.launchTemplate` field.
 
 ## Other Resources