Add support for Kubernetes 1.12.0 (#3918)

packer/install-dependencies.sh

@@ -65,7 +65,7 @@ for KUBE_DNS_VERSION in ${KUBE_DNS_VERSIONS}; do
     pullContainerImage "docker" "k8s.gcr.io/k8s-dns-kube-dns-amd64:${KUBE_DNS_VERSION}"
 done
 
-KUBE_ADDON_MANAGER_VERSIONS="8.6"
+KUBE_ADDON_MANAGER_VERSIONS="8.7 8.6"
 for KUBE_ADDON_MANAGER_VERSION in ${KUBE_ADDON_MANAGER_VERSIONS}; do
     pullContainerImage "docker" "k8s.gcr.io/kube-addon-manager-amd64:v${KUBE_ADDON_MANAGER_VERSION}"
 done
@@ -100,7 +100,7 @@ for AZURE_CNI_NETWORKMONITOR_VERSION in ${AZURE_CNI_NETWORKMONITOR_VERSIONS}; do
     pullContainerImage "docker" "containernetworking/networkmonitor:v${AZURE_CNI_NETWORKMONITOR_VERSION}"
 done
 
-CLUSTER_AUTOSCALER_VERSIONS="1.3.1 1.3.0 1.2.2 1.1.2"
+CLUSTER_AUTOSCALER_VERSIONS="1.3.3 1.3.1 1.3.0 1.2.2 1.1.2"
 for CLUSTER_AUTOSCALER_VERSION in ${CLUSTER_AUTOSCALER_VERSIONS}; do
     pullContainerImage "docker" "k8s.gcr.io/cluster-autoscaler:v${CLUSTER_AUTOSCALER_VERSION}"
 done

parts/k8s/addons/kubernetesmasteraddons-cluster-autoscaler-deployment.yaml

@@ -60,7 +60,7 @@ metadata:
     k8s-addon: cluster-autoscaler.addons.k8s.io
     k8s-app: cluster-autoscaler
     kubernetes.io/cluster-service: "true"
-    addonmanager.kubernetes.io/mode: "EnsureExists"
+    addonmanager.kubernetes.io/mode: Reconcile
 rules:
 - apiGroups: [""]
   resources: ["configmaps"]

parts/k8s/addons/kubernetesmasteraddons-heapster-deployment.yaml

@@ -92,7 +92,6 @@ metadata:
     k8s-app: heapster
     kubernetes.io/cluster-service: "true"
     addonmanager.kubernetes.io/mode: Reconcile
-    addonmanager.kubernetes.io/mode: EnsureExists
 spec:
   replicas: 1
   selector:
@@ -168,4 +167,4 @@ spec:
         - key: "CriticalAddonsOnly"
           operator: "Exists"
       nodeSelector:
-        beta.kubernetes.io/os: linux    
+        beta.kubernetes.io/os: linux

parts/k8s/addons/kubernetesmasteraddons-metrics-server-deployment.yaml

@@ -108,7 +108,7 @@ metadata:
   labels:
     k8s-app: metrics-server
     kubernetes.io/cluster-service: "true"
-    addonmanager.kubernetes.io/mode: EnsureExists
+    addonmanager.kubernetes.io/mode: Reconcile
 spec:
   selector:
     matchLabels:

pkg/acsengine/defaults-kubelet.go

@@ -93,34 +93,15 @@ func setKubeletConfig(cs *api.ContainerService) {
 		o.KubernetesConfig.KubeletConfig[key] = val
 	}
 
-	// Get rid of values not supported in v1.5 clusters
-	if !common.IsKubernetesVersionGe(o.OrchestratorVersion, "1.6.0") {
-		for _, key := range []string{"--non-masquerade-cidr", "--cgroups-per-qos", "--enforce-node-allocatable"} {
-			delete(o.KubernetesConfig.KubeletConfig, key)
-		}
-	}
-
-	// Get rid of values not supported in v1.10 clusters
-	if !common.IsKubernetesVersionGe(o.OrchestratorVersion, "1.10.0") {
-		for _, key := range []string{"--pod-max-pids"} {
-			delete(o.KubernetesConfig.KubeletConfig, key)
-		}
-	}
-
-	// Get rid of values not supported in v1.12 and up
-	if common.IsKubernetesVersionGe(o.OrchestratorVersion, "1.12.0-alpha.1") {
-		for _, key := range []string{"--cadvisor-port"} {
-			delete(o.KubernetesConfig.KubeletConfig, key)
-		}
-	}
-
 	// Remove secure kubelet flags, if configured
 	if !helpers.IsTrueBoolPointer(o.KubernetesConfig.EnableSecureKubelet) {
 		for _, key := range []string{"--anonymous-auth", "--client-ca-file"} {
 			delete(o.KubernetesConfig.KubeletConfig, key)
 		}
 	}
 
+	removeKubeletFlags(o.KubernetesConfig.KubeletConfig, o.OrchestratorVersion)
+
 	// Master-specific kubelet config changes go here
 	if cs.Properties.MasterProfile != nil {
 		if cs.Properties.MasterProfile.KubernetesConfig == nil {
@@ -129,6 +110,8 @@ func setKubeletConfig(cs *api.ContainerService) {
 		}
 		setMissingKubeletValues(cs.Properties.MasterProfile.KubernetesConfig, o.KubernetesConfig.KubeletConfig)
 		addDefaultFeatureGates(cs.Properties.MasterProfile.KubernetesConfig.KubeletConfig, o.OrchestratorVersion, "", "")
+
+		removeKubeletFlags(cs.Properties.MasterProfile.KubernetesConfig.KubeletConfig, o.OrchestratorVersion)
 	}
 
 	// Agent-specific kubelet config changes go here
@@ -156,6 +139,24 @@ func setKubeletConfig(cs *api.ContainerService) {
 				addDefaultFeatureGates(profile.KubernetesConfig.KubeletConfig, o.OrchestratorVersion, "1.6.0", "Accelerators=true")
 			}
 		}
+
+		removeKubeletFlags(profile.KubernetesConfig.KubeletConfig, o.OrchestratorVersion)
+	}
+}
+
+func removeKubeletFlags(k map[string]string, v string) {
+	// Get rid of values not supported until v1.10
+	if !common.IsKubernetesVersionGe(v, "1.10.0") {
+		for _, key := range []string{"--pod-max-pids"} {
+			delete(k, key)
+		}
+	}
+
+	// Get rid of values not supported in v1.12 and up
+	if common.IsKubernetesVersionGe(v, "1.12.0") {
+		for _, key := range []string{"--cadvisor-port"} {
+			delete(k, key)
+		}
 	}
 }
 

pkg/acsengine/defaults_test.go

@@ -714,7 +714,7 @@ func TestIsAzureCNINetworkmonitorAddon(t *testing.T) {
 // TestSetVMSSDefaultsAndZones covers tests for setVMSSDefaultsForAgents and masters
 func TestSetVMSSDefaultsAndZones(t *testing.T) {
 	// masters with vmss and no zones
-	mockCS := getMockBaseContainerService("1.12.0-beta.0")
+	mockCS := getMockBaseContainerService("1.12.0")
 	properties := mockCS.Properties
 	properties.OrchestratorProfile.OrchestratorType = "Kubernetes"
 	properties.MasterProfile.AvailabilityProfile = api.VirtualMachineScaleSets
@@ -728,7 +728,7 @@ func TestSetVMSSDefaultsAndZones(t *testing.T) {
 			properties.OrchestratorProfile.KubernetesConfig.LoadBalancerSku, api.DefaultLoadBalancerSku)
 	}
 	// masters with vmss and zones
-	mockCS = getMockBaseContainerService("1.12.0-beta.0")
+	mockCS = getMockBaseContainerService("1.12.0")
 	properties = mockCS.Properties
 	properties.OrchestratorProfile.OrchestratorType = "Kubernetes"
 	properties.MasterProfile.AvailabilityProfile = api.VirtualMachineScaleSets
@@ -751,7 +751,7 @@ func TestSetVMSSDefaultsAndZones(t *testing.T) {
 			*properties.OrchestratorProfile.KubernetesConfig.ExcludeMasterFromStandardLB, api.DefaultExcludeMasterFromStandardLB)
 	}
 	// agents with vmss and no zones
-	mockCS = getMockBaseContainerService("1.12.0-beta.0")
+	mockCS = getMockBaseContainerService("1.12.0")
 	properties = mockCS.Properties
 	properties.OrchestratorProfile.OrchestratorType = "Kubernetes"
 	properties.AgentPoolProfiles[0].Count = 4
@@ -765,7 +765,7 @@ func TestSetVMSSDefaultsAndZones(t *testing.T) {
 			properties.OrchestratorProfile.KubernetesConfig.LoadBalancerSku, api.DefaultLoadBalancerSku)
 	}
 	// agents with vmss and zones
-	mockCS = getMockBaseContainerService("1.12.0-beta.0")
+	mockCS = getMockBaseContainerService("1.12.0")
 	properties = mockCS.Properties
 	properties.OrchestratorProfile.OrchestratorType = "Kubernetes"
 	properties.AgentPoolProfiles[0].Count = 4

pkg/acsengine/k8s_versions.go

@@ -17,15 +17,15 @@ var k8sComponentVersions = map[string]map[string]string{
 		"heapster":                         "heapster-amd64:v1.5.3",
 		"metrics-server":                   "metrics-server-amd64:v0.2.1",
 		"kube-dns":                         "k8s-dns-kube-dns-amd64:1.14.10",
-		"addon-manager":                    "kube-addon-manager-amd64:v8.6",
+		"addon-manager":                    "kube-addon-manager-amd64:v8.7",
 		"dnsmasq":                          "k8s-dns-dnsmasq-nanny-amd64:1.14.10",
 		"pause":                            "pause-amd64:3.1",
 		"tiller":                           "tiller:v2.8.1",
 		"rescheduler":                      "rescheduler:v0.4.0",
 		"aci-connector":                    "virtual-kubelet:latest",
 		ContainerMonitoringAddonName:       "oms:ciprod05082018",
 		AzureCNINetworkMonitoringAddonName: "networkmonitor:v0.0.4",
-		"cluster-autoscaler":               "cluster-autoscaler:v1.3.1",
+		"cluster-autoscaler":               "cluster-autoscaler:v1.3.3",
 		NVIDIADevicePluginAddonName:        "k8s-device-plugin:1.11",
 		"k8s-dns-sidecar":                  "k8s-dns-sidecar-amd64:1.14.10",
 		"nodestatusfreq":                   DefaultKubernetesNodeStatusUpdateFrequency,

pkg/acsengine/k8s_versions_test.go

@@ -6,11 +6,57 @@ import (
 
 func TestGetK8sVersionComponents(t *testing.T) {
 
+	oneDotTwelveDotZero := getK8sVersionComponents("1.12.0", nil)
+	if oneDotTwelveDotZero == nil {
+		t.Fatalf("getK8sVersionComponents() should not return nil for valid version")
+	}
+	expected := map[string]string{
+		"hyperkube":                        "hyperkube-amd64:v1.12.0",
+		"ccm":                              "cloud-controller-manager-amd64:v1.12.0",
+		"windowszip":                       "v1.12.0-1int.zip",
+		"dockerEngineVersion":              k8sComponentVersions["1.12"]["dockerEngine"],
+		DefaultDashboardAddonName:          k8sComponentVersions["1.12"]["dashboard"],
+		"exechealthz":                      k8sComponentVersions["1.12"]["exechealthz"],
+		"addonresizer":                     k8sComponentVersions["1.12"]["addon-resizer"],
+		"heapster":                         k8sComponentVersions["1.12"]["heapster"],
+		DefaultMetricsServerAddonName:      k8sComponentVersions["1.12"]["metrics-server"],
+		"dns":                              k8sComponentVersions["1.12"]["kube-dns"],
+		"addonmanager":                     k8sComponentVersions["1.12"]["addon-manager"],
+		"dnsmasq":                          k8sComponentVersions["1.12"]["dnsmasq"],
+		"pause":                            k8sComponentVersions["1.12"]["pause"],
+		DefaultTillerAddonName:             k8sComponentVersions["1.12"]["tiller"],
+		DefaultReschedulerAddonName:        k8sComponentVersions["1.12"]["rescheduler"],
+		DefaultACIConnectorAddonName:       k8sComponentVersions["1.12"]["aci-connector"],
+		ContainerMonitoringAddonName:       k8sComponentVersions["1.12"][ContainerMonitoringAddonName],
+		AzureCNINetworkMonitoringAddonName: k8sComponentVersions["1.12"][AzureCNINetworkMonitoringAddonName],
+		DefaultClusterAutoscalerAddonName:  k8sComponentVersions["1.12"]["cluster-autoscaler"],
+		NVIDIADevicePluginAddonName:        k8sComponentVersions["1.12"][NVIDIADevicePluginAddonName],
+		"k8s-dns-sidecar":                  k8sComponentVersions["1.12"]["k8s-dns-sidecar"],
+		"nodestatusfreq":                   k8sComponentVersions["1.12"]["nodestatusfreq"],
+		"nodegraceperiod":                  k8sComponentVersions["1.12"]["nodegraceperiod"],
+		"podeviction":                      k8sComponentVersions["1.12"]["podeviction"],
+		"routeperiod":                      k8sComponentVersions["1.12"]["routeperiod"],
+		"backoffretries":                   k8sComponentVersions["1.12"]["backoffretries"],
+		"backoffjitter":                    k8sComponentVersions["1.12"]["backoffjitter"],
+		"backoffduration":                  k8sComponentVersions["1.12"]["backoffduration"],
+		"backoffexponent":                  k8sComponentVersions["1.12"]["backoffexponent"],
+		"ratelimitqps":                     k8sComponentVersions["1.12"]["ratelimitqps"],
+		"ratelimitbucket":                  k8sComponentVersions["1.12"]["ratelimitbucket"],
+		"gchighthreshold":                  k8sComponentVersions["1.12"]["gchighthreshold"],
+		"gclowthreshold":                   k8sComponentVersions["1.12"]["gclowthreshold"],
+	}
+
+	for k, v := range oneDotTwelveDotZero {
+		if expected[k] != v {
+			t.Fatalf("getK8sVersionComponents() returned an unexpected map[string]string value for k8s 1.12.0: %s = %s", k, oneDotTwelveDotZero[k])
+		}
+	}
+
 	oneDotElevenDotZero := getK8sVersionComponents("1.11.0-alpha.1", nil)
 	if oneDotElevenDotZero == nil {
 		t.Fatalf("getK8sVersionComponents() should not return nil for valid version")
 	}
-	expected := map[string]string{
+	expected = map[string]string{
 		"hyperkube":                        "hyperkube-amd64:v1.11.0-alpha.1",
 		"ccm":                              "cloud-controller-manager-amd64:v1.11.0-alpha.1",
 		"windowszip":                       "v1.11.0-alpha.1-1int.zip",

pkg/api/common/versions.go

@@ -78,11 +78,12 @@ var AllKubernetesSupportedVersions = map[string]bool{
 	"1.11.1":         false,
 	"1.11.2":         true,
 	"1.11.3":         true,
-	"1.12.0-alpha.1": true,
-	"1.12.0-beta.0":  true,
-	"1.12.0-beta.1":  true,
-	"1.12.0-rc.1":    true,
-	"1.12.0-rc.2":    true,
+	"1.12.0-alpha.1": false,
+	"1.12.0-beta.0":  false,
+	"1.12.0-beta.1":  false,
+	"1.12.0-rc.1":    false,
+	"1.12.0-rc.2":    false,
+	"1.12.0":         true,
 }
 
 // GetDefaultKubernetesVersion returns the default Kubernetes version, that is the latest patch of the default release

pkg/api/vlabs/validate.go

@@ -197,8 +197,7 @@ func (a *Properties) validateOrchestratorProfile(isUpdate bool) error {
 			}
 
 			if a.HasAvailabilityZones() {
-				// TODO: update this to 1.12 after it's released
-				minVersion, err := semver.Make("1.12.0-beta.0")
+				minVersion, err := semver.Make("1.12.0")
 				if err != nil {
 					return errors.New("could not validate version")
 				}
@@ -690,17 +689,17 @@ func (a *Properties) validateManagedIdentity() error {
 			if err != nil {
 				return errors.Errorf("could not validate version %s", version)
 			}
-			minVersion, err := semver.Make("1.12.0-beta.0")
+			minVersion, err := semver.Make("1.12.0")
 			if err != nil {
 				return errors.New("could not validate version")
 			}
 
 			if a.MasterProfile.IsVirtualMachineScaleSets() {
 				if sv.LT(minVersion) {
-					return errors.New("managed identity and VMSS masters can only be used with Kubernetes 1.12.0-beta.0 or above. Please specify \"orchestratorRelease\": \"1.12\"")
+					return errors.New("managed identity and VMSS masters can only be used with Kubernetes 1.12.0 or above. Please specify \"orchestratorRelease\": \"1.12\"")
 				}
 			} else if a.OrchestratorProfile.KubernetesConfig.UserAssignedID != "" && sv.LT(minVersion) {
-				return errors.New("user assigned identity can only be used with Kubernetes 1.12.0-beta.0 or above. Please specify \"orchestratorRelease\": \"1.12\"")
+				return errors.New("user assigned identity can only be used with Kubernetes 1.12.0 or above. Please specify \"orchestratorRelease\": \"1.12\"")
 			}
 
 		}

pkg/api/vlabs/validate_test.go

@@ -1437,7 +1437,7 @@ func TestProperties_ValidateManagedIdentity(t *testing.T) {
 					AvailabilityProfile: VirtualMachineScaleSets,
 				},
 			},
-			expectedErr: "managed identity and VMSS masters can only be used with Kubernetes 1.12.0-beta.0 or above. Please specify \"orchestratorRelease\": \"1.12\"",
+			expectedErr: "managed identity and VMSS masters can only be used with Kubernetes 1.12.0 or above. Please specify \"orchestratorRelease\": \"1.12\"",
 		},
 		{
 			name:                "use managed identity with master vmas",
@@ -1471,7 +1471,7 @@ func TestProperties_ValidateManagedIdentity(t *testing.T) {
 					Count:  1,
 				},
 			},
-			expectedErr: "user assigned identity can only be used with Kubernetes 1.12.0-beta.0 or above. Please specify \"orchestratorRelease\": \"1.12\"",
+			expectedErr: "user assigned identity can only be used with Kubernetes 1.12.0 or above. Please specify \"orchestratorRelease\": \"1.12\"",
 		},
 	}
 	for _, test := range tests {

pkg/operations/kubernetesupgrade/upgradecluster.go

@@ -116,7 +116,8 @@ func (uc *UpgradeCluster) UpgradeCluster(subscriptionID uuid.UUID, kubeConfig, r
 
 	case strings.HasPrefix(upgradeVersion, "1.9."),
 		strings.HasPrefix(upgradeVersion, "1.10."),
-		strings.HasPrefix(upgradeVersion, "1.11."):
+		strings.HasPrefix(upgradeVersion, "1.11."),
+		strings.HasPrefix(upgradeVersion, "1.12."):
 		u := &Upgrader{}
 		u.Init(uc.Translator, uc.Logger, uc.ClusterTopology, uc.Client, kubeConfig, uc.StepTimeout, acsengineVersion)
 		upgrader = u