feat: add support for single stack IPv6 (Azure#2781)

(cherry picked from commit 1b9beb4)
bowen5 · Mar 30, 2020 · e036001 · e036001
1 parent 11d7372
commit e036001
Show file tree

Hide file tree

Showing 25 changed files with 423 additions and 92 deletions.
diff --git a/parts/k8s/addons/1.18/kubernetesmasteraddons-kube-proxy-daemonset.yaml b/parts/k8s/addons/1.18/kubernetesmasteraddons-kube-proxy-daemonset.yaml
@@ -9,6 +9,15 @@ data:
       kubeconfig: /var/lib/kubelet/kubeconfig
     clusterCIDR: "{{ContainerConfig "cluster-cidr"}}"
     mode: "{{ContainerConfig "proxy-mode"}}"
+    {{- if ContainerConfig "bind-address"}}
+    bindAddress: "{{ContainerConfig "bind-address"}}"
+    {{end}}
+    {{- if ContainerConfig "healthz-bind-address"}}
+    healthzBindAddress: "{{ContainerConfig "healthz-bind-address"}}"
+    {{end}}
+    {{- if ContainerConfig "metrics-bind-address"}}
+    metricsBindAddress: "{{ContainerConfig "metrics-bind-address"}}"
+    {{end}}
     featureGates:
       {{ContainerConfig "featureGates"}}
 metadata:

diff --git a/parts/k8s/addons/coredns.yaml b/parts/k8s/addons/coredns.yaml
@@ -155,6 +155,9 @@ spec:
           effect: NoSchedule
       nodeSelector:
         beta.kubernetes.io/os: linux
+        {{- if ContainerConfig "use-host-network"}}
+        kubernetes.azure.com/role: agent
+        {{end}}
       containers:
       - name: coredns
         image: {{ContainerImage "coredns"}}
@@ -206,6 +209,9 @@ spec:
             - all
           readOnlyRootFilesystem: true
       dnsPolicy: Default
+      {{- if ContainerConfig "use-host-network"}}
+      hostNetwork: {{ContainerConfig "use-host-network"}}
+      {{end}}
       volumes:
         - name: config-volume
           configMap:

diff --git a/parts/k8s/cloud-init/artifacts/cse_config.sh b/parts/k8s/cloud-init/artifacts/cse_config.sh
@@ -295,7 +295,7 @@ ensureKMS() {
 }
 {{end}}
 
-{{if IsIPv6DualStackFeatureEnabled}}
+{{if IsIPv6Enabled}}
 ensureDHCPv6() {
     wait_for_file 3600 1 {{GetDHCPv6ServiceCSEScriptFilepath}} || exit $ERR_FILE_WATCH_TIMEOUT
     wait_for_file 3600 1 {{GetDHCPv6ConfigCSEScriptFilepath}} || exit $ERR_FILE_WATCH_TIMEOUT

diff --git a/parts/k8s/cloud-init/artifacts/cse_main.sh b/parts/k8s/cloud-init/artifacts/cse_main.sh
@@ -187,8 +187,8 @@ if [[ -n "${MASTER_NODE}" && "${KMS_PROVIDER_VAULT_NAME}" != "" ]]; then
 fi
 {{end}}
 
-{{/* configure and enable dhcpv6 for dual stack feature */}}
-{{- if IsIPv6DualStackFeatureEnabled}}
+{{/* configure and enable dhcpv6 for ipv6 features */}}
+{{- if IsIPv6Enabled}}
 ensureDHCPv6
 {{end}}
 

diff --git a/parts/k8s/cloud-init/masternodecustomdata.yml b/parts/k8s/cloud-init/masternodecustomdata.yml
@@ -127,7 +127,7 @@ write_files:
     {{CloudInitData "aptPreferences"}}
 {{end}}
 
-{{if IsIPv6DualStackFeatureEnabled}}
+{{if IsIPv6Enabled}}
 - path: {{GetDHCPv6ServiceCSEScriptFilepath}}
   permissions: "0644"
   encoding: gzip

diff --git a/parts/k8s/cloud-init/nodecustomdata.yml b/parts/k8s/cloud-init/nodecustomdata.yml
@@ -113,7 +113,7 @@ write_files:
     {{CloudInitData "aptPreferences"}}
 {{end}}
 
-{{if IsIPv6DualStackFeatureEnabled}}
+{{if IsIPv6Enabled}}
 - path: {{GetDHCPv6ServiceCSEScriptFilepath}}
   permissions: "0644"
   encoding: gzip

diff --git a/pkg/api/addons.go b/pkg/api/addons.go
@@ -297,7 +297,8 @@ func (cs *ContainerService) setAddonsConfig(isUpgrade bool) {
 			"non-masquerade-cidr":           cs.Properties.GetNonMasqueradeCIDR(),
 			"non-masq-cni-cidr":             cs.Properties.GetAzureCNICidr(),
 			"secondary-non-masquerade-cidr": cs.Properties.GetSecondaryNonMasqueradeCIDR(),
-			"enable-ipv6":                   strconv.FormatBool(cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6DualStack")),
+			"enable-ipv6": strconv.FormatBool(cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6DualStack") ||
+				cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6Only")),
 		},
 	}
 
@@ -673,6 +674,13 @@ func (cs *ContainerService) setAddonsConfig(isUpgrade bool) {
 		},
 	}
 
+	// set host network to true for single stack IPv6 as the the nameserver is currently
+	// IPv4 only. By setting it to host network, we can leverage the host routes to successfully
+	// resolve dns.
+	if cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6Only") {
+		defaultCorednsAddonsConfig.Config["use-host-network"] = "true"
+	}
+
 	// If we have any explicit coredns or kube-dns configuration in the addons array
 	if getAddonsIndexByName(o.KubernetesConfig.Addons, common.KubeDNSAddonName) != -1 || getAddonsIndexByName(o.KubernetesConfig.Addons, common.CoreDNSAddonName) != -1 {
 		// Ensure we don't we don't prepare an addons spec w/ both kube-dns and coredns enabled
@@ -697,6 +705,14 @@ func (cs *ContainerService) setAddonsConfig(isUpgrade bool) {
 		},
 	}
 
+	// set bind address, healthz and metric bind address to :: explicitly for
+	// single stack IPv6 cluster as it is single stack IPv6 on dual stack host
+	if cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6Only") {
+		defaultKubeProxyAddonsConfig.Config["bind-address"] = "::"
+		defaultKubeProxyAddonsConfig.Config["healthz-bind-address"] = "::"
+		defaultKubeProxyAddonsConfig.Config["metrics-bind-address"] = "::1"
+	}
+
 	defaultPodSecurityPolicyAddonsConfig := KubernetesAddon{
 		Name:    common.PodSecurityPolicyAddonName,
 		Enabled: to.BoolPtr(common.IsKubernetesVersionGe(o.OrchestratorVersion, "1.15.0") || to.Bool(o.KubernetesConfig.EnablePodSecurityPolicy)),

diff --git a/pkg/api/addons_test.go b/pkg/api/addons_test.go
@@ -3310,6 +3310,162 @@ func TestSetAddonsConfig(t *testing.T) {
 				},
 			}, "1.15.4"),
 		},
+		{
+			name: "addons with IPv6 single stack",
+			cs: &ContainerService{
+				Properties: &Properties{
+					FeatureFlags: &FeatureFlags{
+						EnableIPv6Only: true,
+					},
+					OrchestratorProfile: &OrchestratorProfile{
+						OrchestratorVersion: "1.18.0",
+						KubernetesConfig: &KubernetesConfig{
+							DNSServiceIP:  DefaultKubernetesDNSServiceIPv6,
+							NetworkPlugin: NetworkPluginKubenet,
+							KubeletConfig: map[string]string{
+								"--cluster-domain": "cluster.local",
+								"--node-ip":        "::",
+							},
+							ClusterSubnet: DefaultKubernetesClusterSubnetIPv6,
+							ProxyMode:     KubeProxyModeIPTables,
+							APIServerConfig: map[string]string{
+								"--bind-address": "::",
+							},
+							ControllerManagerConfig: map[string]string{
+								"--bind-address": "::",
+							},
+							SchedulerConfig: map[string]string{
+								"--bind-address": "::",
+							},
+						},
+					},
+				},
+			},
+			isUpgrade: false,
+			expectedAddons: omitFromAddons([]string{common.AzureCNINetworkMonitorAddonName}, overwriteDefaultAddons([]KubernetesAddon{
+				{
+					Name:    common.CoreDNSAddonName,
+					Enabled: to.BoolPtr(DefaultCoreDNSAddonEnabled),
+					Config: map[string]string{
+						"domain":           "cluster.local",
+						"clusterIP":        DefaultKubernetesDNSServiceIPv6,
+						"use-host-network": "true",
+					},
+					Containers: []KubernetesContainerSpec{
+						{
+							Name:  common.CoreDNSAddonName,
+							Image: specConfig.KubernetesImageBase + K8sComponentsByVersionMap["1.18.0"][common.CoreDNSAddonName],
+						},
+					},
+				},
+				{
+					Name:    common.IPMASQAgentAddonName,
+					Enabled: to.BoolPtr(true),
+					Containers: []KubernetesContainerSpec{
+						{
+							Name:           common.IPMASQAgentAddonName,
+							CPURequests:    "50m",
+							MemoryRequests: "50Mi",
+							CPULimits:      "50m",
+							MemoryLimits:   "250Mi",
+							Image:          specConfig.KubernetesImageBase + K8sComponentsByVersionMap["1.18.0"][common.IPMASQAgentAddonName],
+						},
+					},
+					Config: map[string]string{
+						"non-masquerade-cidr":           DefaultKubernetesClusterSubnetIPv6,
+						"enable-ipv6":                   "true",
+						"non-masq-cni-cidr":             "",
+						"secondary-non-masquerade-cidr": "",
+					},
+				},
+				{
+					Name:    common.KubeProxyAddonName,
+					Enabled: to.BoolPtr(DefaultKubeProxyAddonEnabled),
+					Config: map[string]string{
+						"cluster-cidr":         DefaultKubernetesClusterSubnetIPv6,
+						"proxy-mode":           string(KubeProxyModeIPTables),
+						"featureGates":         "{}",
+						"bind-address":         "::",
+						"healthz-bind-address": "::",
+						"metrics-bind-address": "::1",
+					},
+					Containers: []KubernetesContainerSpec{
+						{
+							Name:  common.KubeProxyAddonName,
+							Image: specConfig.KubernetesImageBase + K8sComponentsByVersionMap["1.18.0"][common.KubeProxyAddonName],
+						},
+					},
+				},
+			}, "1.18.0")),
+		},
+		{
+			name: "addons with dual stack",
+			cs: &ContainerService{
+				Properties: &Properties{
+					FeatureFlags: &FeatureFlags{
+						EnableIPv6DualStack: true,
+					},
+					OrchestratorProfile: &OrchestratorProfile{
+						OrchestratorVersion: "1.18.0",
+						KubernetesConfig: &KubernetesConfig{
+							DNSServiceIP:  DefaultKubernetesDNSServiceIP,
+							NetworkPlugin: NetworkPluginKubenet,
+							KubeletConfig: map[string]string{
+								"--cluster-domain": "cluster.local",
+								"--feature-gates":  "IPv6DualStack=true",
+							},
+							ClusterSubnet: DefaultKubernetesClusterSubnet + "," + DefaultKubernetesClusterSubnetIPv6,
+							ServiceCIDR:   DefaultKubernetesServiceCIDR + "," + DefaultKubernetesServiceCIDRIPv6,
+							ProxyMode:     KubeProxyModeIPVS,
+							APIServerConfig: map[string]string{
+								"--feature-gates": "IPv6DualStack=true",
+							},
+							ControllerManagerConfig: map[string]string{
+								"--feature-gates": "IPv6DualStack=true",
+							},
+						},
+					},
+				},
+			},
+			isUpgrade: false,
+			expectedAddons: omitFromAddons([]string{common.AzureCNINetworkMonitorAddonName}, overwriteDefaultAddons([]KubernetesAddon{
+				{
+					Name:    common.IPMASQAgentAddonName,
+					Enabled: to.BoolPtr(true),
+					Containers: []KubernetesContainerSpec{
+						{
+							Name:           common.IPMASQAgentAddonName,
+							CPURequests:    "50m",
+							MemoryRequests: "50Mi",
+							CPULimits:      "50m",
+							MemoryLimits:   "250Mi",
+							Image:          specConfig.KubernetesImageBase + K8sComponentsByVersionMap["1.18.0"][common.IPMASQAgentAddonName],
+						},
+					},
+					Config: map[string]string{
+						"non-masquerade-cidr":           DefaultKubernetesClusterSubnet,
+						"enable-ipv6":                   "true",
+						"non-masq-cni-cidr":             "",
+						"secondary-non-masquerade-cidr": DefaultKubernetesClusterSubnetIPv6,
+					},
+				},
+				{
+					Name:    common.KubeProxyAddonName,
+					Enabled: to.BoolPtr(DefaultKubeProxyAddonEnabled),
+					Config: map[string]string{
+						"cluster-cidr": DefaultKubernetesClusterSubnet + "," + DefaultKubernetesClusterSubnetIPv6,
+						"proxy-mode":   string(KubeProxyModeIPVS),
+						"featureGates": "IPv6DualStack: true",
+					},
+					Containers: []KubernetesContainerSpec{
+						{
+							Name:  common.KubeProxyAddonName,
+							Image: specConfig.KubernetesImageBase + K8sComponentsByVersionMap["1.18.0"][common.KubeProxyAddonName],
+						},
+					},
+				},
+			}, "1.18.0")),
+		},
 	}
 
 	for _, test := range tests {
@@ -3764,17 +3920,22 @@ func concatenateDefaultAddons(addons []KubernetesAddon, version string) []Kubern
 }
 
 func overwriteDefaultAddons(addons []KubernetesAddon, version string) []KubernetesAddon {
+	overrideAddons := make(map[string]KubernetesAddon)
+	for _, addonOverride := range addons {
+		overrideAddons[addonOverride.Name] = addonOverride
+	}
+
 	var ret []KubernetesAddon
 	defaults := getDefaultAddons(version)
-	for _, addonOverride := range addons {
-		for _, addon := range defaults {
-			if addon.Name == addonOverride.Name {
-				ret = append(ret, addonOverride)
-			} else {
-				ret = append(ret, addon)
-			}
+
+	for _, addon := range defaults {
+		if _, exists := overrideAddons[addon.Name]; exists {
+			ret = append(ret, overrideAddons[addon.Name])
+			continue
 		}
+		ret = append(ret, addon)
 	}
+
 	return ret
 }
 

diff --git a/pkg/api/const.go b/pkg/api/const.go
@@ -439,11 +439,15 @@ const (
 	// DefaultKubernetesClusterSubnet specifies the default subnet for pods.
 	DefaultKubernetesClusterSubnet = "10.244.0.0/16"
 	// DefaultKubernetesClusterSubnetIPv6 specifies the IPv6 default subnet for pods.
-	DefaultKubernetesClusterSubnetIPv6 = "fc00::/8"
+	DefaultKubernetesClusterSubnetIPv6 = "fc00::/48"
 	// DefaultKubernetesServiceCIDR specifies the IP subnet that kubernetes will create Service IPs within.
 	DefaultKubernetesServiceCIDR = "10.0.0.0/16"
 	// DefaultKubernetesDNSServiceIP specifies the IP address that kube-dns listens on by default. must by in the default Service CIDR range.
 	DefaultKubernetesDNSServiceIP = "10.0.0.10"
+	// DefaultKubernetesServiceCIDRIPv6 specifies the IPv6 subnet that kubernetes will create Service IPs within.
+	DefaultKubernetesServiceCIDRIPv6 = "fd00::/108"
+	// DefaultKubernetesDNSServiceIPv6 specifies the IPv6 address that kube-dns listens on by default. must by in the default Service CIDR range.
+	DefaultKubernetesDNSServiceIPv6 = "fd00::10"
 	// DefaultMobyVersion specifies the default Azure build version of Moby to install.
 	DefaultMobyVersion = "3.0.10"
 	// DefaultContainerdVersion specifies the default containerd version to install.

diff --git a/pkg/api/converterfromapi.go b/pkg/api/converterfromapi.go
@@ -610,6 +610,7 @@ func convertFeatureFlagsToVLabs(api *FeatureFlags, vlabs *vlabs.FeatureFlags) {
 	vlabs.BlockOutboundInternet = api.BlockOutboundInternet
 	vlabs.EnableIPv6DualStack = api.EnableIPv6DualStack
 	vlabs.EnableTelemetry = api.EnableTelemetry
+	vlabs.EnableIPv6Only = api.EnableIPv6Only
 }
 
 func convertCloudProfileToVLabs(api *CustomCloudProfile, vlabsccp *vlabs.CustomCloudProfile) {

diff --git a/pkg/api/convertertoapi.go b/pkg/api/convertertoapi.go
@@ -123,6 +123,7 @@ func convertVLabsFeatureFlags(vlabs *vlabs.FeatureFlags, api *FeatureFlags) {
 	api.BlockOutboundInternet = vlabs.BlockOutboundInternet
 	api.EnableIPv6DualStack = vlabs.EnableIPv6DualStack
 	api.EnableTelemetry = vlabs.EnableTelemetry
+	api.EnableIPv6Only = vlabs.EnableIPv6Only
 }
 
 func convertVLabsExtensionProfile(vlabs *vlabs.ExtensionProfile, api *ExtensionProfile) {

diff --git a/pkg/api/defaults-apiserver.go b/pkg/api/defaults-apiserver.go
@@ -162,6 +162,14 @@ func (cs *ContainerService) setAPIServerConfig() {
 			delete(o.KubernetesConfig.APIServerConfig, key)
 		}
 	}
+	// Set bind address to prefer IPv6 address for single stack IPv6 cluster
+	// Remove --advertise-address so that --bind-address will be used
+	if cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6Only") {
+		o.KubernetesConfig.APIServerConfig["--bind-address"] = "::"
+		for _, key := range []string{"--advertise-address"} {
+			delete(o.KubernetesConfig.APIServerConfig, key)
+		}
+	}
 }
 
 func getDefaultAdmissionControls(cs *ContainerService) (string, string) {

diff --git a/pkg/api/defaults-apiserver_test.go b/pkg/api/defaults-apiserver_test.go
@@ -506,3 +506,22 @@ func TestAPIServerFeatureGates(t *testing.T) {
 			"1.17.0", a["--feature-gates"])
 	}
 }
+
+func TestAPIServerIPv6Only(t *testing.T) {
+	cs := CreateMockContainerService("testcluster", "1.18.0", 3, 2, false)
+	cs.Properties.FeatureFlags = &FeatureFlags{EnableIPv6Only: true}
+	cs.setAPIServerConfig()
+
+	a := cs.Properties.OrchestratorProfile.KubernetesConfig.APIServerConfig
+	// bind address should be :: for single stack IPv6 cluster
+	if a["--bind-address"] != "::" {
+		t.Fatalf("got unexpected default value for '--bind-address' API server config: %s",
+			a["--bind-address"])
+	}
+	for _, key := range []string{"--advertise-address"} {
+		if _, ok := a[key]; ok {
+			t.Fatalf("got unexpected '%s' API server config value for '--advertise-address' %s",
+				key, a[key])
+		}
+	}
+}