diff --git a/internal/deployers/eksapi/deployer.go b/internal/deployers/eksapi/deployer.go index fedf7932b..6a3d38526 100644 --- a/internal/deployers/eksapi/deployer.go +++ b/internal/deployers/eksapi/deployer.go @@ -67,25 +67,26 @@ type deployerOptions struct { EmitMetrics bool `flag:"emit-metrics" desc:"Record and emit metrics to CloudWatch"` ExpectedAMI string `flag:"expected-ami" desc:"Expected AMI of nodes. Up will fail if the actual nodes are not utilizing the expected AMI. Defaults to --ami if defined."` // TODO: remove this once it's no longer used in downstream jobs - GenerateSSHKey bool `flag:"generate-ssh-key" desc:"Generate an SSH key to use for tests. The generated key should not be used in production, as it will not have a passphrase."` - InstanceTypes []string `flag:"instance-types" desc:"Node instance types. Cannot be used with --instance-type-archs"` - InstanceTypeArchs []string `flag:"instance-type-archs" desc:"Use default node instance types for specific architectures. Cannot be used with --instance-types"` - IPFamily string `flag:"ip-family" desc:"IP family for the cluster (ipv4 or ipv6)"` - KubeconfigPath string `flag:"kubeconfig" desc:"Path to kubeconfig"` - KubernetesVersion string `flag:"kubernetes-version" desc:"cluster Kubernetes version"` - LogBucket string `flag:"log-bucket" desc:"S3 bucket for storing logs for each run. If empty, logs will not be stored."` - NodeCreationTimeout time.Duration `flag:"node-creation-timeout" desc:"Time to wait for nodes to be created/launched. This should consider instance availability."` - NodeReadyTimeout time.Duration `flag:"node-ready-timeout" desc:"Time to wait for all nodes to become ready"` - Nodes int `flag:"nodes" desc:"number of nodes to launch in cluster"` - NodeNameStrategy string `flag:"node-name-strategy" desc:"Specifies the naming strategy for node. Allowed values: ['SessionName', 'EC2PrivateDNSName'], default to EC2PrivateDNSName"` - Region string `flag:"region" desc:"AWS region for EKS cluster"` - SkipNodeReadinessChecks bool `flag:"skip-node-readiness-checks" desc:"Skip performing readiness checks on created nodes"` - StaticClusterName string `flag:"static-cluster-name" desc:"Optional when re-use existing cluster and node group by querying the kubeconfig and run test"` - TuneVPCCNI bool `flag:"tune-vpc-cni" desc:"Apply tuning parameters to the VPC CNI DaemonSet"` - UnmanagedNodes bool `flag:"unmanaged-nodes" desc:"Use an AutoScalingGroup instead of an EKS-managed nodegroup. Requires --ami"` - UpClusterHeaders []string `flag:"up-cluster-header" desc:"Additional header to add to eks:CreateCluster requests. Specified in the same format as curl's -H flag."` - UserDataFormat string `flag:"user-data-format" desc:"Format of the node instance user data"` - ZoneType string `flag:"zone-type" desc:"Type of zone to use for infrastructure (availability-zone, local-zone, etc). Defaults to availability-zone"` + GenerateSSHKey bool `flag:"generate-ssh-key" desc:"Generate an SSH key to use for tests. The generated key should not be used in production, as it will not have a passphrase."` + InstanceTypes []string `flag:"instance-types" desc:"Node instance types. Cannot be used with --instance-type-archs"` + InstanceTypeArchs []string `flag:"instance-type-archs" desc:"Use default node instance types for specific architectures. Cannot be used with --instance-types"` + IPFamily string `flag:"ip-family" desc:"IP family for the cluster (ipv4 or ipv6)"` + KubeconfigPath string `flag:"kubeconfig" desc:"Path to kubeconfig"` + KubernetesVersion string `flag:"kubernetes-version" desc:"cluster Kubernetes version"` + LogBucket string `flag:"log-bucket" desc:"S3 bucket for storing logs for each run. If empty, logs will not be stored."` + NodeadmFeatureGates []string `flag:"nodeadm-feature-gates" desc:"Feature gates to enable for nodeadm (key=value pairs)"` + NodeCreationTimeout time.Duration `flag:"node-creation-timeout" desc:"Time to wait for nodes to be created/launched. This should consider instance availability."` + NodeReadyTimeout time.Duration `flag:"node-ready-timeout" desc:"Time to wait for all nodes to become ready"` + Nodes int `flag:"nodes" desc:"number of nodes to launch in cluster"` + NodeNameStrategy string `flag:"node-name-strategy" desc:"Specifies the naming strategy for node. Allowed values: ['SessionName', 'EC2PrivateDNSName'], default to EC2PrivateDNSName"` + Region string `flag:"region" desc:"AWS region for EKS cluster"` + SkipNodeReadinessChecks bool `flag:"skip-node-readiness-checks" desc:"Skip performing readiness checks on created nodes"` + StaticClusterName string `flag:"static-cluster-name" desc:"Optional when re-use existing cluster and node group by querying the kubeconfig and run test"` + TuneVPCCNI bool `flag:"tune-vpc-cni" desc:"Apply tuning parameters to the VPC CNI DaemonSet"` + UnmanagedNodes bool `flag:"unmanaged-nodes" desc:"Use an AutoScalingGroup instead of an EKS-managed nodegroup. Requires --ami"` + UpClusterHeaders []string `flag:"up-cluster-header" desc:"Additional header to add to eks:CreateCluster requests. Specified in the same format as curl's -H flag."` + UserDataFormat string `flag:"user-data-format" desc:"Format of the node instance user data"` + ZoneType string `flag:"zone-type" desc:"Type of zone to use for infrastructure (availability-zone, local-zone, etc). Defaults to availability-zone"` } // NewDeployer implements deployer.New for EKS using the EKS (and other AWS) API(s) directly (no cloudformation) diff --git a/internal/deployers/eksapi/node.go b/internal/deployers/eksapi/node.go index 697e04084..604488e20 100644 --- a/internal/deployers/eksapi/node.go +++ b/internal/deployers/eksapi/node.go @@ -349,7 +349,7 @@ func (m *nodeManager) createUnmanagedNodegroup(infra *Infrastructure, cluster *C var capacityReservationId string stackName := m.getUnmanagedNodegroupStackName() klog.Infof("creating unmanaged nodegroup stack %s...", stackName) - userData, userDataIsMimePart, err := generateUserData(opts.UserDataFormat, cluster, opts) + userData, userDataIsMimePart, err := generateUserData(cluster, opts) if err != nil { return err } diff --git a/internal/deployers/eksapi/templates/templates.go b/internal/deployers/eksapi/templates/templates.go index 0695472b0..544a39e09 100644 --- a/internal/deployers/eksapi/templates/templates.go +++ b/internal/deployers/eksapi/templates/templates.go @@ -67,6 +67,7 @@ type UserDataTemplateData struct { CIDR string APIServerEndpoint string KubeletFeatureGates map[string]bool + NodeadmFeatureGates map[string]bool } var ( diff --git a/internal/deployers/eksapi/templates/userdata_nodeadm.yaml.mimepart.template b/internal/deployers/eksapi/templates/userdata_nodeadm.yaml.mimepart.template index 6eb100d6a..2e9e13082 100644 --- a/internal/deployers/eksapi/templates/userdata_nodeadm.yaml.mimepart.template +++ b/internal/deployers/eksapi/templates/userdata_nodeadm.yaml.mimepart.template @@ -5,6 +5,12 @@ MIME-Version: 1.0 apiVersion: node.eks.aws/v1alpha1 kind: NodeConfig spec: +{{- if .NodeadmFeatureGates}} + featureGates: + {{- range $gate, $value := .NodeadmFeatureGates }} + {{$gate}}: {{$value}} + {{- end }} +{{- end }} cluster: name: {{.Name}} apiServerEndpoint: {{.APIServerEndpoint}} diff --git a/internal/deployers/eksapi/userdata.go b/internal/deployers/eksapi/userdata.go index e370bc2e6..43b5d8b87 100644 --- a/internal/deployers/eksapi/userdata.go +++ b/internal/deployers/eksapi/userdata.go @@ -3,15 +3,17 @@ package eksapi import ( "bytes" "fmt" + "strconv" + "strings" "text/template" "github.com/aws/aws-k8s-tester/internal/deployers/eksapi/templates" ) -func generateUserData(format string, cluster *Cluster, opts *deployerOptions) (string, bool, error) { +func generateUserData(cluster *Cluster, opts *deployerOptions) (string, bool, error) { userDataIsMimePart := true var t *template.Template - switch format { + switch opts.UserDataFormat { case "bootstrap.sh": t = templates.UserDataBootstrapSh case "nodeadm": @@ -21,7 +23,7 @@ func generateUserData(format string, cluster *Cluster, opts *deployerOptions) (s t = templates.UserDataBottlerocket userDataIsMimePart = false default: - return "", false, fmt.Errorf("uknown user data format: '%s'", format) + return "", false, fmt.Errorf("unknown user data format: '%s'", opts.UserDataFormat) } kubeletFeatureGates := map[string]bool{} @@ -30,6 +32,11 @@ func generateUserData(format string, cluster *Cluster, opts *deployerOptions) (s kubeletFeatureGates["DynamicResourceAllocation"] = true } + nodeadmFeatureGates, err := extractFeatureGates(opts.NodeadmFeatureGates) + if err != nil { + return "", false, err + } + var buf bytes.Buffer if err := t.Execute(&buf, templates.UserDataTemplateData{ APIServerEndpoint: cluster.endpoint, @@ -37,8 +44,25 @@ func generateUserData(format string, cluster *Cluster, opts *deployerOptions) (s CIDR: cluster.cidr, Name: cluster.name, KubeletFeatureGates: kubeletFeatureGates, + NodeadmFeatureGates: nodeadmFeatureGates, }); err != nil { return "", false, err } return buf.String(), userDataIsMimePart, nil } + +func extractFeatureGates(featureGatePairs []string) (map[string]bool, error) { + featureGateMap := make(map[string]bool) + for _, keyValuePair := range featureGatePairs { + components := strings.Split(keyValuePair, "=") + if len(components) != 2 { + return featureGateMap, fmt.Errorf("expected key=value pairs but %s has %d components", keyValuePair, len(components)) + } + boolValue, err := strconv.ParseBool(components[1]) + if err != nil { + return featureGateMap, fmt.Errorf("expected bool value in %s: %v", keyValuePair, err) + } + featureGateMap[components[0]] = boolValue + } + return featureGateMap, nil +} diff --git a/internal/deployers/eksapi/userdata_test.go b/internal/deployers/eksapi/userdata_test.go index f14c7beb5..f2203bfe3 100644 --- a/internal/deployers/eksapi/userdata_test.go +++ b/internal/deployers/eksapi/userdata_test.go @@ -35,6 +35,41 @@ spec: certificateAuthority: certificateAuthority cidr: 10.100.0.0/16 ` + +const nodeadmUserDataKubeletDRA = `Content-Type: application/node.eks.aws +MIME-Version: 1.0 + +--- +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: cluster + apiServerEndpoint: https://example.com + certificateAuthority: certificateAuthority + cidr: 10.100.0.0/16 + kubelet: + config: + featureGates: + DynamicResourceAllocation: true +` + +const nodeadmUserDataFeatureGate = `Content-Type: application/node.eks.aws +MIME-Version: 1.0 + +--- +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + featureGates: + foo: true + cluster: + name: cluster + apiServerEndpoint: https://example.com + certificateAuthority: certificateAuthority + cidr: 10.100.0.0/16 +` + const bottlerocketUserData = `[settings.kubernetes] "cluster-name" = "cluster" "api-server" = "https://example.com" @@ -47,9 +82,12 @@ device-ownership-from-security-context = true func Test_generateUserData(t *testing.T) { cases := []struct { - format string - expected string - expectedIsMimePart bool + format string + expected string + expectedIsMimePart bool + kubernetesVersion string + NodeadmFeatureGates []string + wantErr bool }{ { format: "bootstrap.sh", @@ -66,10 +104,28 @@ func Test_generateUserData(t *testing.T) { expected: bottlerocketUserData, expectedIsMimePart: false, }, + { + format: "nodeadm", + expected: nodeadmUserDataKubeletDRA, + kubernetesVersion: "1.33", + expectedIsMimePart: true, + }, + { + format: "nodeadm", + expected: nodeadmUserDataFeatureGate, + kubernetesVersion: "1.30", + NodeadmFeatureGates: []string{"foo=true"}, + expectedIsMimePart: true, + }, } for _, c := range cases { t.Run(c.format, func(t *testing.T) { - actual, isMimePart, err := generateUserData(c.format, &cluster, &deployerOptions{}) + deployerOpts := &deployerOptions{ + KubernetesVersion: c.kubernetesVersion, + NodeadmFeatureGates: c.NodeadmFeatureGates, + UserDataFormat: c.format, + } + actual, isMimePart, err := generateUserData(&cluster, deployerOpts) if err != nil { t.Log(err) t.Error(err) @@ -79,3 +135,36 @@ func Test_generateUserData(t *testing.T) { }) } } + +func Test_extractFeatureGates(t *testing.T) { + testCases := []struct { + input []string + expected map[string]bool + expectErr bool + }{ + { + input: []string{"foo=true", "bar=false"}, + expected: map[string]bool{ + "foo": true, + "bar": false, + }, + }, + { + input: []string{"foo:true"}, + expectErr: true, + }, + { + input: []string{"foo=bar"}, + expectErr: true, + }, + } + for _, testCase := range testCases { + output, err := extractFeatureGates(testCase.input) + if testCase.expectErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, testCase.expected, output) + } + } +}