Add [experimental] option for using IPVS proxy mode

core/controlplane/config/config.go

@@ -106,6 +106,13 @@ func NewDefaultCluster() *Cluster {
 		},
 	}
 
+	ipvsMode := IPVSMode{
+		Enabled:       false,
+		Scheduler:     "rr",
+		SyncPeriod:    "60s",
+		MinSyncPeriod: "10s",
+	}
+
 	return &Cluster{
 		DeploymentSettings: DeploymentSettings{
 			ClusterName:        "kubernetes",
@@ -131,6 +138,9 @@ func NewDefaultCluster() *Cluster {
 					interval: 60,
 				},
 			},
+			KubeProxy: KubeProxy{
+				IPVSMode: ipvsMode,
+			},
 			KubeDns: KubeDns{
 				NodeLocalResolver: false,
 			},
@@ -423,6 +433,7 @@ type DeploymentSettings struct {
 	CloudWatchLogging       `yaml:"cloudWatchLogging,omitempty"`
 	AmazonSsmAgent          `yaml:"amazonSsmAgent,omitempty"`
 	CloudFormationStreaming bool `yaml:"cloudFormationStreaming,omitempty"`
+	KubeProxy               `yaml:"kubeProxy,omitempty"`
 	KubeDns                 `yaml:"kubeDns,omitempty"`
 	KubernetesDashboard     `yaml:"kubernetesDashboard,omitempty"`
 	// Images repository
@@ -637,6 +648,17 @@ type TargetGroup struct {
 	SecurityGroupIds []string `yaml:"securityGroupIds"`
 }
 
+type KubeProxy struct {
+	IPVSMode IPVSMode `yaml:"ipvsMode"`
+}
+
+type IPVSMode struct {
+	Enabled       bool   `yaml:"enabled"`
+	Scheduler     string `yaml:"scheduler"`
+	SyncPeriod    string `yaml:"syncPeriod"`
+	MinSyncPeriod string `yaml:"minSyncPeriod"`
+}
+
 type KubeDns struct {
 	NodeLocalResolver bool `yaml:"nodeLocalResolver"`
 }

core/controlplane/config/templates/cloud-config-controller

@@ -1912,12 +1912,20 @@ write_files:
         namespace: kube-system
       data:
         kube-proxy-config.yaml: |
-          apiVersion: componentconfig/v1alpha1
+          apiVersion: {{if ge .K8sVer "v1.9"}}kubeproxy.config.k8s.io{{else}}componentconfig{{end}}/v1alpha1
           kind: KubeProxyConfiguration
           bindAddress: 0.0.0.0
           clientConnection:
             kubeconfig: /etc/kubernetes/kubeconfig/kube-proxy.yaml
           clusterCIDR: {{.PodCIDR}}
+          {{if .KubeProxy.IPVSMode.Enabled -}}
+          featureGates: "SupportIPVSProxyMode=true"
+          mode: ipvs
+          ipvs:
+            scheduler: {{.KubeProxy.IPVSMode.Scheduler}}
+            syncPeriod: {{.KubeProxy.IPVSMode.SyncPeriod}}
+            minSyncPeriod: {{.KubeProxy.IPVSMode.MinSyncPeriod}}
+          {{end}}
 
   - path: /srv/kubernetes/manifests/kube-proxy-ds.yaml
     content: |
@@ -1959,13 +1967,23 @@ write_files:
                 securityContext:
                   privileged: true
                 volumeMounts:
+                {{if .KubeProxy.IPVSMode.Enabled -}}
+                - mountPath: /lib/modules
+                  name: lib-modules
+                  readOnly: true
+                {{end -}}
                 - mountPath: /etc/kubernetes/kubeconfig
                   name: kubeconfig
                   readOnly: true
                 - mountPath: /etc/kubernetes/kube-proxy
                   name: kube-proxy-config
                   readOnly: true
               volumes:
+              {{if .KubeProxy.IPVSMode.Enabled -}}
+              - name: lib-modules
+                hostPath:
+                  path: /lib/modules
+              {{end -}}
               - name: kubeconfig
                 hostPath:
                   path: /etc/kubernetes/kubeconfig

core/controlplane/config/templates/cluster.yaml

@@ -1176,6 +1176,19 @@ kubernetesDashboard:
 #kubeDns:
 # nodeLocalResolver: false
 
+kubeProxy:
+  # Use IPVS kube-proxy mode instead of [default] iptables one (requires Kubernetes 1.9.0+ to work reliably)
+  # This is intended to address performance issues of iptables mode for clusters with big number of nodes and services
+  # FIXME For those who use hyperkube version 'v1.9.0' / 'v1.9.0_coreos.0', your image may lack `ipset` utility
+  # FIXME Please see: https://github.com/kubernetes/kubernetes/issues/57321 (next Kubernetes release will have a fix)
+  # FIXME https://github.com/kubernetes/kubernetes/commit/787a55bb67ccd2da14aa6e7f91289c859beecb5f#diff-bf0f8d724d18f700f3c821aa5a74f4cf
+  # FIXME IPVS integration is still green, proceed with care! You may get fixed hyperkube image from 'ivanilves/hyperkube' Docker repo
+  ipvsMode:
+    enabled: false
+    scheduler: rr
+    syncPeriod: 300s
+    minSyncPeriod: 60s
+
 # When enabled, CloudFormation events will stream to stdout during kube-aws 'update | up'.
 # It is enabled by default.
 #cloudFormationStreaming: true

test/integration/maincluster_test.go

@@ -520,6 +520,65 @@ apiEndpoints:
 				},
 			},
 		},
+		{
+			context:    "WithKubeProxyIPVSModeDisabledByDefault",
+			configYaml: minimalValidConfigYaml,
+			assertConfig: []ConfigTester{
+				func(c *config.Config, t *testing.T) {
+					if c.KubeProxy.IPVSMode.Enabled != false {
+						t.Errorf("kube-proxy IPVS mode must be disabled by default")
+					}
+
+					expectedScheduler := "rr"
+					if c.KubeProxy.IPVSMode.Scheduler != expectedScheduler {
+						t.Errorf("IPVS scheduler should be by default set to: %s (actual = %s)", expectedScheduler, c.KubeProxy.IPVSMode.Scheduler)
+					}
+
+					expectedSyncPeriod := "60s"
+					if c.KubeProxy.IPVSMode.SyncPeriod != expectedSyncPeriod {
+						t.Errorf("Sync period should be by default set to: %s (actual = %s)", expectedSyncPeriod, c.KubeProxy.IPVSMode.SyncPeriod)
+					}
+
+					expectedMinSyncPeriod := "10s"
+					if c.KubeProxy.IPVSMode.MinSyncPeriod != expectedMinSyncPeriod {
+						t.Errorf("Minimal sync period should be by default set to: %s (actual = %s)", expectedMinSyncPeriod, c.KubeProxy.IPVSMode.MinSyncPeriod)
+					}
+				},
+			},
+		},
+		{
+			context: "WithKubeProxyIPVSModeEnabled",
+			configYaml: minimalValidConfigYaml + `
+kubeProxy:
+  ipvsMode:
+    enabled: true
+    scheduler: lc
+    syncPeriod: 90s
+    minSyncPeriod: 15s
+`,
+			assertConfig: []ConfigTester{
+				func(c *config.Config, t *testing.T) {
+					if c.KubeProxy.IPVSMode.Enabled != true {
+						t.Errorf("kube-proxy IPVS mode must be enabled")
+					}
+
+					expectedScheduler := "lc"
+					if c.KubeProxy.IPVSMode.Scheduler != expectedScheduler {
+						t.Errorf("IPVS scheduler should be set to: %s (actual = %s)", expectedScheduler, c.KubeProxy.IPVSMode.Scheduler)
+					}
+
+					expectedSyncPeriod := "90s"
+					if c.KubeProxy.IPVSMode.SyncPeriod != expectedSyncPeriod {
+						t.Errorf("Sync period should be set to: %s (actual = %s)", expectedSyncPeriod, c.KubeProxy.IPVSMode.SyncPeriod)
+					}
+
+					expectedMinSyncPeriod := "15s"
+					if c.KubeProxy.IPVSMode.MinSyncPeriod != expectedMinSyncPeriod {
+						t.Errorf("Minimal sync period should be set to: %s (actual = %s)", expectedMinSyncPeriod, c.KubeProxy.IPVSMode.MinSyncPeriod)
+					}
+				},
+			},
+		},
 		{
 			// See https://github.com/kubernetes-incubator/kube-aws/issues/365
 			context:    "WithClusterNameContainsHyphens",