Fixes and address comments

k8s/crds/kops.k8s.io_clusters.yaml

@@ -3121,22 +3121,9 @@ spec:
                     description: AmazonVPCNetworkingSpec declares that we want Amazon
                       VPC CNI networking
                     properties:
-                      awsVpcK8sCniConfigureRpFilter:
-                        description: Specifies whether ipamd should configure rp filter
-                          for primary interface. Setting this to false will require
-                          rp filter to be configured through init container.
-                        type: string
-                      disableTCPEarlyDemux:
-                        description: If ENABLE_POD_ENI is set to true, in order for
-                          the kubelet to connect via TCP (for liveness or readiness
-                          probes) to pods that are using per pod security groups,
-                          DISABLE_TCP_EARLY_DEMUX should be set to true for amazon-k8s-cni-init
-                          container under initcontainers. This will increase the local
-                          TCP connection latency slightly. To use this setting, a
-                          Linux kernel version of at least 4.6 is needed on the worker
-                          node.
-                        type: string
                       env:
+                        description: Env is a list of environment variables to set
+                          in the container.
                         items:
                           description: EnvVar represents an environment variable present
                             in a Container.
@@ -3161,10 +3148,11 @@ spec:
                           type: object
                         type: array
                       imageName:
-                        description: The container image name to use
+                        description: ImageName is the container image name to use.
                         type: string
                       initImageName:
-                        description: The init container image name to use
+                        description: InitImageName is the init container image name
+                          to use.
                         type: string
                     type: object
                   calico:

pkg/apis/kops/networking.go

@@ -248,19 +248,12 @@ type RomanaNetworkingSpec struct {
 
 // AmazonVPCNetworkingSpec declares that we want Amazon VPC CNI networking
 type AmazonVPCNetworkingSpec struct {
-	// Specifies whether ipamd should configure rp filter for primary interface.
-	// Setting this to false will require rp filter to be configured through init container.
-	AwsVpcK8sCniConfigureRpFilter string   `json:"awsVpcK8sCniConfigureRpFilter,omitempty"`
-	Env                           []EnvVar `json:"env,omitempty"`
-	// If ENABLE_POD_ENI is set to true, in order for the kubelet to connect via TCP (for liveness or readiness probes) to pods that are using per pod security groups,
-	// DISABLE_TCP_EARLY_DEMUX should be set to true for amazon-k8s-cni-init container under initcontainers.
-	// This will increase the local TCP connection latency slightly.
-	// To use this setting, a Linux kernel version of at least 4.6 is needed on the worker node.
-	DisableTCPEarlyDemux string `json:"disableTCPEarlyDemux,omitempty"`
-	// The container image name to use
+	// ImageName is the container image name to use.
 	ImageName string `json:"imageName,omitempty"`
-	// The init container image name to use
+	// InitImageName is the init container image name to use.
 	InitImageName string `json:"initImageName,omitempty"`
+	// Env is a list of environment variables to set in the container.
+	Env []EnvVar `json:"env,omitempty"`
 }
 
 const CiliumIpamEni = "eni"

pkg/apis/kops/v1alpha2/networking.go

@@ -248,19 +248,12 @@ type RomanaNetworkingSpec struct {
 
 // AmazonVPCNetworkingSpec declares that we want Amazon VPC CNI networking
 type AmazonVPCNetworkingSpec struct {
-	// Specifies whether ipamd should configure rp filter for primary interface.
-	// Setting this to false will require rp filter to be configured through init container.
-	AwsVpcK8sCniConfigureRpFilter string   `json:"awsVpcK8sCniConfigureRpFilter,omitempty"`
-	Env                           []EnvVar `json:"env,omitempty"`
-	// If ENABLE_POD_ENI is set to true, in order for the kubelet to connect via TCP (for liveness or readiness probes) to pods that are using per pod security groups,
-	// DISABLE_TCP_EARLY_DEMUX should be set to true for amazon-k8s-cni-init container under initcontainers.
-	// This will increase the local TCP connection latency slightly.
-	// To use this setting, a Linux kernel version of at least 4.6 is needed on the worker node.
-	DisableTCPEarlyDemux string `json:"disableTCPEarlyDemux,omitempty"`
-	// The container image name to use
+	// ImageName is the container image name to use.
 	ImageName string `json:"imageName,omitempty"`
-	// The init container image name to use
+	// InitImageName is the init container image name to use.
 	InitImageName string `json:"initImageName,omitempty"`
+	// Env is a list of environment variables to set in the container.
+	Env []EnvVar `json:"env,omitempty"`
 }
 
 const CiliumIpamEni = "eni"

upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template

@@ -106,9 +106,20 @@
                 - "fargate"
       "containers":
       - "env":
-        - "name": "AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER"
-          "value": "{{ AwsVpcK8sCniConfigureRpFilter }}"
-        # The below envs are commented-out on purpose. See https://github.com/kubernetes/kops/issues/11144 for more context.
+        {{- range $name, $value := AmazonVpcEnvVars }}
+        - "name": "{{ $name }}"
+          "value": "{{ $value }}"
+        {{- end }}
+        # The below envs are commented-out on purpose and replaced by the above range. 
+        # See https://github.com/kubernetes/kops/issues/11144 for more context.
+        # - "name": "ADDITIONAL_ENI_TAGS"
+        #   "value": "{}"
+        # - "name": "AWS_VPC_CNI_NODE_PORT_SUPPORT"
+        #   "value": "true"
+        # - "name": "AWS_VPC_ENI_MTU"
+        #   "value": "9001"
+        # - "name": "AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER"
+        #   "value": "false"
         # - "name": "AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG"
         #   "value": "false"
         # - "name": "AWS_VPC_K8S_CNI_EXTERNALSNAT"
@@ -131,16 +142,18 @@
         #   "value": "false"
         # - "name": "ENABLE_POD_ENI"
         #   "value": "false"
+        # - "name": "MY_NODE_NAME"
+        #   "valueFrom":
+        #     "fieldRef":
+        #       "fieldPath": "spec.nodeName"
+        # - "name": "WARM_ENI_TARGET"
+        #   "value": "1"
         - "name": "MY_NODE_NAME"
           "valueFrom":
             "fieldRef":
               "fieldPath": "spec.nodeName"
         - "name": "CLUSTER_NAME"
           "value": "{{ ClusterName }}"
-        {{- range .Networking.AmazonVPC.Env }}
-        - "name": "{{ .Name }}"
-          "value": "{{ .Value }}"
-        {{- end }}
         "image": "{{- or .Networking.AmazonVPC.ImageName "602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:v1.8.0" }}"
         "imagePullPolicy": "Always"
         "livenessProbe":
@@ -183,7 +196,7 @@
       "initContainers":
       - "env":
         - "name": "DISABLE_TCP_EARLY_DEMUX"
-          "value": " {{ DisableTCPEarlyDemux }}"
+          "value": "false"
         "image": "{{- or .Networking.AmazonVPC.InitImageName "602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni-init:v1.8.0" }}"
         "imagePullPolicy": "Always"
         "name": "aws-vpc-cni-init"

upup/pkg/fi/cloudup/template_functions.go

@@ -151,17 +151,14 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap, secretStore fi.SecretS
 
 	if cluster.Spec.Networking != nil && cluster.Spec.Networking.AmazonVPC != nil {
 		c := cluster.Spec.Networking.AmazonVPC
-		dest["AwsVpcK8sCniConfigureRpFilter"] = func() string {
-			if c.AwsVpcK8sCniConfigureRpFilter != "" {
-				return c.AwsVpcK8sCniConfigureRpFilter
+		dest["AmazonVpcEnvVars"] = func() map[string]string {
+			envVars := map[string]string{
+				"AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER": "false",
 			}
-			return "false"
-		}
-		dest["DisableTCPEarlyDemux"] = func() string {
-			if c.DisableTCPEarlyDemux != "" {
-				return c.DisableTCPEarlyDemux
+			for _, e := range c.Env {
+				envVars[e.Name] = e.Value
 			}
-			return "false"
+			return envVars
 		}
 	}
 

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc-containerd/manifest.yaml

@@ -54,7 +54,7 @@ spec:
     version: 1.17.0
   - id: k8s-1.16
     manifest: networking.amazon-vpc-routed-eni/k8s-1.16.yaml
-    manifestHash: 102ad54bf34da0a827df22e8cbaae453b83b0db9
+    manifestHash: fea48420a1f8d5e929b59e08fb96612032d8e651
     name: networking.amazon-vpc-routed-eni
     needsRollingUpdate: all
     selector:

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc-containerd/networking.amazon-vpc-routed-eni-k8s-1.16.yaml

@@ -134,16 +134,16 @@ spec:
       - env:
         - name: AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER
           value: "false"
+        - name: AWS_VPC_K8S_CNI_LOGLEVEL
+          value: debug
+        - name: WARM_IP_TARGET
+          value: "10"
         - name: MY_NODE_NAME
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
         - name: CLUSTER_NAME
           value: minimal.example.com
-        - name: WARM_IP_TARGET
-          value: "10"
-        - name: AWS_VPC_K8S_CNI_LOGLEVEL
-          value: debug
         image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:v1.8.0
         imagePullPolicy: Always
         livenessProbe:
@@ -186,7 +186,7 @@ spec:
       initContainers:
       - env:
         - name: DISABLE_TCP_EARLY_DEMUX
-          value: ' false'
+          value: "false"
         image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni-init:v1.8.0
         imagePullPolicy: Always
         name: aws-vpc-cni-init

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/manifest.yaml

@@ -54,7 +54,7 @@ spec:
     version: 1.17.0
   - id: k8s-1.16
     manifest: networking.amazon-vpc-routed-eni/k8s-1.16.yaml
-    manifestHash: 102ad54bf34da0a827df22e8cbaae453b83b0db9
+    manifestHash: fea48420a1f8d5e929b59e08fb96612032d8e651
     name: networking.amazon-vpc-routed-eni
     needsRollingUpdate: all
     selector:

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/networking.amazon-vpc-routed-eni-k8s-1.16.yaml

@@ -134,16 +134,16 @@ spec:
       - env:
         - name: AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER
           value: "false"
+        - name: AWS_VPC_K8S_CNI_LOGLEVEL
+          value: debug
+        - name: WARM_IP_TARGET
+          value: "10"
         - name: MY_NODE_NAME
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
         - name: CLUSTER_NAME
           value: minimal.example.com
-        - name: WARM_IP_TARGET
-          value: "10"
-        - name: AWS_VPC_K8S_CNI_LOGLEVEL
-          value: debug
         image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:v1.8.0
         imagePullPolicy: Always
         livenessProbe:
@@ -186,7 +186,7 @@ spec:
       initContainers:
       - env:
         - name: DISABLE_TCP_EARLY_DEMUX
-          value: ' false'
+          value: "false"
         image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni-init:v1.8.0
         imagePullPolicy: Always
         name: aws-vpc-cni-init