feat(eksapi): accept and configure nodeadm feature gates

internal/deployers/eksapi/deployer.go

@@ -67,25 +67,26 @@ type deployerOptions struct {
 	EmitMetrics                 bool          `flag:"emit-metrics" desc:"Record and emit metrics to CloudWatch"`
 	ExpectedAMI                 string        `flag:"expected-ami" desc:"Expected AMI of nodes. Up will fail if the actual nodes are not utilizing the expected AMI. Defaults to --ami if defined."`
 	// TODO: remove this once it's no longer used in downstream jobs
-	GenerateSSHKey      bool          `flag:"generate-ssh-key" desc:"Generate an SSH key to use for tests. The generated key should not be used in production, as it will not have a passphrase."`
-	InstanceTypes       []string      `flag:"instance-types" desc:"Node instance types. Cannot be used with --instance-type-archs"`
-	InstanceTypeArchs   []string      `flag:"instance-type-archs" desc:"Use default node instance types for specific architectures. Cannot be used with --instance-types"`
-	IPFamily            string        `flag:"ip-family" desc:"IP family for the cluster (ipv4 or ipv6)"`
-	KubeconfigPath      string        `flag:"kubeconfig" desc:"Path to kubeconfig"`
-	KubernetesVersion   string        `flag:"kubernetes-version" desc:"cluster Kubernetes version"`
-	LogBucket           string        `flag:"log-bucket" desc:"S3 bucket for storing logs for each run. If empty, logs will not be stored."`
-	NodeCreationTimeout time.Duration `flag:"node-creation-timeout" desc:"Time to wait for nodes to be created/launched. This should consider instance availability."`
-	NodeReadyTimeout    time.Duration `flag:"node-ready-timeout" desc:"Time to wait for all nodes to become ready"`
-	Nodes               int           `flag:"nodes" desc:"number of nodes to launch in cluster"`
-	NodeNameStrategy    string        `flag:"node-name-strategy" desc:"Specifies the naming strategy for node. Allowed values: ['SessionName', 'EC2PrivateDNSName'], default to EC2PrivateDNSName"`
-	Region              string        `flag:"region" desc:"AWS region for EKS cluster"`
-	SkipNodeReadinessChecks bool 	  `flag:"skip-node-readiness-checks" desc:"Skip performing readiness checks on created nodes"`
-	StaticClusterName   string        `flag:"static-cluster-name" desc:"Optional when re-use existing cluster and node group by querying the kubeconfig and run test"`
-	TuneVPCCNI          bool          `flag:"tune-vpc-cni" desc:"Apply tuning parameters to the VPC CNI DaemonSet"`
-	UnmanagedNodes      bool          `flag:"unmanaged-nodes" desc:"Use an AutoScalingGroup instead of an EKS-managed nodegroup. Requires --ami"`
-	UpClusterHeaders    []string      `flag:"up-cluster-header" desc:"Additional header to add to eks:CreateCluster requests. Specified in the same format as curl's -H flag."`
-	UserDataFormat      string        `flag:"user-data-format" desc:"Format of the node instance user data"`
-	ZoneType            string        `flag:"zone-type" desc:"Type of zone to use for infrastructure (availability-zone, local-zone, etc). Defaults to availability-zone"`
+	GenerateSSHKey          bool          `flag:"generate-ssh-key" desc:"Generate an SSH key to use for tests. The generated key should not be used in production, as it will not have a passphrase."`
+	InstanceTypes           []string      `flag:"instance-types" desc:"Node instance types. Cannot be used with --instance-type-archs"`
+	InstanceTypeArchs       []string      `flag:"instance-type-archs" desc:"Use default node instance types for specific architectures. Cannot be used with --instance-types"`
+	IPFamily                string        `flag:"ip-family" desc:"IP family for the cluster (ipv4 or ipv6)"`
+	KubeconfigPath          string        `flag:"kubeconfig" desc:"Path to kubeconfig"`
+	KubernetesVersion       string        `flag:"kubernetes-version" desc:"cluster Kubernetes version"`
+	LogBucket               string        `flag:"log-bucket" desc:"S3 bucket for storing logs for each run. If empty, logs will not be stored."`
+	NodeadmFeatureGates     []string      `flag:"nodeadm-feature-gates" desc:"Feature gates to enable for nodeadm (key=value pairs)"`
+	NodeCreationTimeout     time.Duration `flag:"node-creation-timeout" desc:"Time to wait for nodes to be created/launched. This should consider instance availability."`
+	NodeReadyTimeout        time.Duration `flag:"node-ready-timeout" desc:"Time to wait for all nodes to become ready"`
+	Nodes                   int           `flag:"nodes" desc:"number of nodes to launch in cluster"`
+	NodeNameStrategy        string        `flag:"node-name-strategy" desc:"Specifies the naming strategy for node. Allowed values: ['SessionName', 'EC2PrivateDNSName'], default to EC2PrivateDNSName"`
+	Region                  string        `flag:"region" desc:"AWS region for EKS cluster"`
+	SkipNodeReadinessChecks bool          `flag:"skip-node-readiness-checks" desc:"Skip performing readiness checks on created nodes"`
+	StaticClusterName       string        `flag:"static-cluster-name" desc:"Optional when re-use existing cluster and node group by querying the kubeconfig and run test"`
+	TuneVPCCNI              bool          `flag:"tune-vpc-cni" desc:"Apply tuning parameters to the VPC CNI DaemonSet"`
+	UnmanagedNodes          bool          `flag:"unmanaged-nodes" desc:"Use an AutoScalingGroup instead of an EKS-managed nodegroup. Requires --ami"`
+	UpClusterHeaders        []string      `flag:"up-cluster-header" desc:"Additional header to add to eks:CreateCluster requests. Specified in the same format as curl's -H flag."`
+	UserDataFormat          string        `flag:"user-data-format" desc:"Format of the node instance user data"`
+	ZoneType                string        `flag:"zone-type" desc:"Type of zone to use for infrastructure (availability-zone, local-zone, etc). Defaults to availability-zone"`
 }
 
 // NewDeployer implements deployer.New for EKS using the EKS (and other AWS) API(s) directly (no cloudformation)

internal/deployers/eksapi/node.go

@@ -349,7 +349,7 @@ func (m *nodeManager) createUnmanagedNodegroup(infra *Infrastructure, cluster *C
 	var capacityReservationId string
 	stackName := m.getUnmanagedNodegroupStackName()
 	klog.Infof("creating unmanaged nodegroup stack %s...", stackName)
-	userData, userDataIsMimePart, err := generateUserData(opts.UserDataFormat, cluster, opts)
+	userData, userDataIsMimePart, err := generateUserData(cluster, opts)
 	if err != nil {
 		return err
 	}

internal/deployers/eksapi/templates/templates.go

@@ -67,6 +67,7 @@ type UserDataTemplateData struct {
 	CIDR                 string
 	APIServerEndpoint    string
 	KubeletFeatureGates  map[string]bool
+	NodeadmFeatureGates  map[string]bool
 }
 
 var (

internal/deployers/eksapi/templates/userdata_nodeadm.yaml.mimepart.template

@@ -5,6 +5,12 @@ MIME-Version: 1.0
 apiVersion: node.eks.aws/v1alpha1
 kind: NodeConfig
 spec:
+{{- if .NodeadmFeatureGates}}
+  featureGates:
+    {{- range $gate, $value := .NodeadmFeatureGates }}
+    {{$gate}}: {{$value}}
+    {{- end }}
+{{- end }}
   cluster:
     name: {{.Name}}
     apiServerEndpoint: {{.APIServerEndpoint}}

internal/deployers/eksapi/userdata.go

@@ -3,15 +3,17 @@ package eksapi
 import (
 	"bytes"
 	"fmt"
+	"strconv"
+	"strings"
 	"text/template"
 
 	"github.com/aws/aws-k8s-tester/internal/deployers/eksapi/templates"
 )
 
-func generateUserData(format string, cluster *Cluster, opts *deployerOptions) (string, bool, error) {
+func generateUserData(cluster *Cluster, opts *deployerOptions) (string, bool, error) {
 	userDataIsMimePart := true
 	var t *template.Template
-	switch format {
+	switch opts.UserDataFormat {
 	case "bootstrap.sh":
 		t = templates.UserDataBootstrapSh
 	case "nodeadm":
@@ -21,7 +23,7 @@ func generateUserData(format string, cluster *Cluster, opts *deployerOptions) (s
 		t = templates.UserDataBottlerocket
 		userDataIsMimePart = false
 	default:
-		return "", false, fmt.Errorf("uknown user data format: '%s'", format)
+		return "", false, fmt.Errorf("unknown user data format: '%s'", opts.UserDataFormat)
 	}
 
 	kubeletFeatureGates := map[string]bool{}
@@ -30,15 +32,37 @@ func generateUserData(format string, cluster *Cluster, opts *deployerOptions) (s
 		kubeletFeatureGates["DynamicResourceAllocation"] = true
 	}
 
+	nodeadmFeatureGates, err := extractFeatureGates(opts.NodeadmFeatureGates)
+	if err != nil {
+		return "", false, err
+	}
+
 	var buf bytes.Buffer
 	if err := t.Execute(&buf, templates.UserDataTemplateData{
 		APIServerEndpoint:    cluster.endpoint,
 		CertificateAuthority: cluster.certificateAuthorityData,
 		CIDR:                 cluster.cidr,
 		Name:                 cluster.name,
 		KubeletFeatureGates:  kubeletFeatureGates,
+		NodeadmFeatureGates:  nodeadmFeatureGates,
 	}); err != nil {
 		return "", false, err
 	}
 	return buf.String(), userDataIsMimePart, nil
 }
+
+func extractFeatureGates(featureGatePairs []string) (map[string]bool, error) {
+	featureGateMap := make(map[string]bool)
+	for _, keyValuePair := range featureGatePairs {
+		components := strings.Split(keyValuePair, "=")
+		if len(components) != 2 {
+			return featureGateMap, fmt.Errorf("expected key=value pairs but %s has %d components", keyValuePair, len(components))
+		}
+		boolValue, err := strconv.ParseBool(components[1])
+		if err != nil {
+			return featureGateMap, fmt.Errorf("expected bool value in %s: %v", keyValuePair, err)
+		}
+		featureGateMap[components[0]] = boolValue
+	}
+	return featureGateMap, nil
+}

internal/deployers/eksapi/userdata_test.go

@@ -35,6 +35,41 @@ spec:
     certificateAuthority: certificateAuthority
     cidr: 10.100.0.0/16
 `
+
+const nodeadmUserDataKubeletDRA = `Content-Type: application/node.eks.aws
+MIME-Version: 1.0
+
+---
+apiVersion: node.eks.aws/v1alpha1
+kind: NodeConfig
+spec:
+  cluster:
+    name: cluster
+    apiServerEndpoint: https://example.com
+    certificateAuthority: certificateAuthority
+    cidr: 10.100.0.0/16
+  kubelet:
+    config:
+      featureGates:
+        DynamicResourceAllocation: true
+`
+
+const nodeadmUserDataFeatureGate = `Content-Type: application/node.eks.aws
+MIME-Version: 1.0
+
+---
+apiVersion: node.eks.aws/v1alpha1
+kind: NodeConfig
+spec:
+  featureGates:
+    foo: true
+  cluster:
+    name: cluster
+    apiServerEndpoint: https://example.com
+    certificateAuthority: certificateAuthority
+    cidr: 10.100.0.0/16
+`
+
 const bottlerocketUserData = `[settings.kubernetes]
 "cluster-name" = "cluster"
 "api-server" = "https://example.com"
@@ -47,9 +82,12 @@ device-ownership-from-security-context = true
 
 func Test_generateUserData(t *testing.T) {
 	cases := []struct {
-		format             string
-		expected           string
-		expectedIsMimePart bool
+		format              string
+		expected            string
+		expectedIsMimePart  bool
+		kubernetesVersion   string
+		NodeadmFeatureGates []string
+		wantErr             bool
 	}{
 		{
 			format:             "bootstrap.sh",
@@ -66,10 +104,28 @@ func Test_generateUserData(t *testing.T) {
 			expected:           bottlerocketUserData,
 			expectedIsMimePart: false,
 		},
+		{
+			format:             "nodeadm",
+			expected:           nodeadmUserDataKubeletDRA,
+			kubernetesVersion:  "1.33",
+			expectedIsMimePart: true,
+		},
+		{
+			format:              "nodeadm",
+			expected:            nodeadmUserDataFeatureGate,
+			kubernetesVersion:   "1.30",
+			NodeadmFeatureGates: []string{"foo=true"},
+			expectedIsMimePart:  true,
+		},
 	}
 	for _, c := range cases {
 		t.Run(c.format, func(t *testing.T) {
-			actual, isMimePart, err := generateUserData(c.format, &cluster, &deployerOptions{})
+			deployerOpts := &deployerOptions{
+				KubernetesVersion:   c.kubernetesVersion,
+				NodeadmFeatureGates: c.NodeadmFeatureGates,
+				UserDataFormat:      c.format,
+			}
+			actual, isMimePart, err := generateUserData(&cluster, deployerOpts)
 			if err != nil {
 				t.Log(err)
 				t.Error(err)
@@ -79,3 +135,36 @@ func Test_generateUserData(t *testing.T) {
 		})
 	}
 }
+
+func Test_extractFeatureGates(t *testing.T) {
+	testCases := []struct {
+		input     []string
+		expected  map[string]bool
+		expectErr bool
+	}{
+		{
+			input: []string{"foo=true", "bar=false"},
+			expected: map[string]bool{
+				"foo": true,
+				"bar": false,
+			},
+		},
+		{
+			input:     []string{"foo:true"},
+			expectErr: true,
+		},
+		{
+			input:     []string{"foo=bar"},
+			expectErr: true,
+		},
+	}
+	for _, testCase := range testCases {
+		output, err := extractFeatureGates(testCase.input)
+		if testCase.expectErr {
+			assert.Error(t, err)
+		} else {
+			assert.NoError(t, err)
+			assert.Equal(t, testCase.expected, output)
+		}
+	}
+}