refactor: user-configurable flannel and scheduled maintenance addons

docs/topics/clusterdefinitions.md

@@ -115,6 +115,7 @@ $ aks-engine get-versions
 | azure-cloud-provider                            | true                                                                                | 0    | Delivers required ClusterRole, ClusterRoleBinding, and StorageClass resources required for running the Azure cloudprovider runtime. May not be disabled.                                                                                                                                                                         |
 | aad                            | true if adminGroupID is specified in the aadProfile configuration                                                                                | 0     | ClusterRoleBinding specification that adds an admin group matching the adminGroupID                                                                                                                                                                   |
 | [cilium](https://docs.cilium.io/en/v1.4/kubernetes/policy/#ciliumnetworkpolicy)                            | true if networkPolicy is "cilium"; currently validated against Kubernetes v1.13, v1.14, and v1.15                                                                                | 0     | A NetworkPolicy CRD implementation by the Cilium project (currently supports v1.4)                                                                                                                                                                  |
+| [flannel](https://coreos.com/flannel/docs/0.8.0/index.html)                            | false                                                                                | 0     | An addon that delivers flannel: a virtual network that gives a subnet to each host for use with container runtimes. If `networkPlugin` is set to `"flannel"` this addon will be enabled automatically. Not compatible with any other `networkPlugin` or `networkPolicy`.                                                                                                                                                                 |
 
 To give a bit more info on the `addons` property: We've tried to expose the basic bits of data that allow useful configuration of these cluster features. Here are some example usage patterns that will unpack what `addons` provide:
 

parts/k8s/cloud-init/artifacts/cse_config.sh

@@ -317,6 +317,11 @@ ensureKubelet() {
         sleep 3
     done
     {{end}}
+    {{if HasFlannelNetworkPlugin}}
+    while [ ! -f /etc/cni/net.d/10-flannel.conf ]; do
+        sleep 3
+    done
+    {{end}}
 }
 
 ensureLabelNodes() {

parts/k8s/cloud-init/masternodecustomdata.yml

@@ -342,8 +342,6 @@ write_files:
 
 MASTER_MANIFESTS_CONFIG_PLACEHOLDER
 
-MASTER_ADDONS_CONFIG_PLACEHOLDER
-
 MASTER_CUSTOM_FILES_PLACEHOLDER
 
 MASTER_CONTAINER_ADDONS_PLACEHOLDER

parts/k8s/addons/1.16/kubernetesmasteraddons-flannel-daemonset.yaml → parts/k8s/containeraddons/1.16/kubernetesmasteraddons-flannel-daemonset.yaml

@@ -72,7 +72,7 @@ spec:
       serviceAccountName: flannel
       containers:
       - name: kube-flannel
-        image: quay.io/coreos/flannel:v0.8.0-amd64
+        image: {{ContainerImage "kube-flannel"}}
         command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr" ]
         securityContext:
           privileged: true
@@ -91,7 +91,7 @@ spec:
         - name: flannel-cfg
           mountPath: /etc/kube-flannel/
       - name: install-cni
-        image: quay.io/coreos/flannel:v0.10.0-amd64
+        image: {{ContainerImage "install-cni"}}
         command: [ "/bin/sh", "-c", "set -e -x; cp -f /etc/kube-flannel/cni-conf.json /etc/cni/net.d/10-flannel.conf; while true; do sleep 3600; done" ]
         volumeMounts:
         - name: cni

parts/k8s/addons/1.17/kubernetesmasteraddons-flannel-daemonset.yaml → parts/k8s/containeraddons/1.17/kubernetesmasteraddons-flannel-daemonset.yaml

@@ -74,7 +74,7 @@ spec:
       serviceAccountName: flannel
       containers:
       - name: kube-flannel
-        image: quay.io/coreos/flannel:v0.8.0-amd64
+        image: {{ContainerImage "kube-flannel"}}
         command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr" ]
         securityContext:
           privileged: true
@@ -93,7 +93,7 @@ spec:
         - name: flannel-cfg
           mountPath: /etc/kube-flannel/
       - name: install-cni
-        image: quay.io/coreos/flannel:v0.10.0-amd64
+        image: {{ContainerImage "install-cni"}}
         command: [ "/bin/sh", "-c", "set -e -x; cp -f /etc/kube-flannel/cni-conf.json /etc/cni/net.d/10-flannel.conf; while true; do sleep 3600; done" ]
         volumeMounts:
         - name: cni

parts/k8s/addons/kubernetesmasteraddons-flannel-daemonset.yaml → parts/k8s/containeraddons/kubernetesmasteraddons-flannel-daemonset.yaml

@@ -70,7 +70,7 @@ spec:
       serviceAccountName: flannel
       containers:
       - name: kube-flannel
-        image: quay.io/coreos/flannel:v0.8.0-amd64
+        image: {{ContainerImage "kube-flannel"}}
         command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr" ]
         securityContext:
           privileged: true
@@ -89,7 +89,7 @@ spec:
         - name: flannel-cfg
           mountPath: /etc/kube-flannel/
       - name: install-cni
-        image: quay.io/coreos/flannel:v0.10.0-amd64
+        image: {{ContainerImage "install-cni"}}
         command: [ "/bin/sh", "-c", "set -e -x; cp -f /etc/kube-flannel/cni-conf.json /etc/cni/net.d/10-flannel.conf; while true; do sleep 3600; done" ]
         volumeMounts:
         - name: cni

parts/k8s/addons/kubernetesmasteraddons-scheduled-maintenance-deployment.yaml → parts/k8s/containeraddons/kubernetesmasteraddons-scheduled-maintenance-deployment.yaml

@@ -245,7 +245,7 @@ spec:
         - --upstream=http://127.0.0.1:8080/
         - --logtostderr=true
         - --v=10
-        image: gcr.io/kubebuilder/kube-rbac-proxy:v0.4.0
+        image: {{ContainerImage "kube-rbac-proxy"}}
         name: kube-rbac-proxy
         ports:
         - containerPort: 8443
@@ -267,7 +267,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
-        image: quay.io/awesomenix/drainsafe-manager:latest
+        image: {{ContainerImage "manager"}}
         name: manager
         resources:
           limits:
@@ -314,7 +314,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
-        image: quay.io/awesomenix/drainsafe-manager:latest
+        image: {{ContainerImage "manager"}}
         name: manager
         resources:
           limits:

pkg/api/addons.go

@@ -421,7 +421,7 @@ func (cs *ContainerService) setAddonsConfig(isUpgrade bool) {
 			},
 			{
 				Name:  common.AntreaInstallCNIContainerName,
-				Image: k8sComponents[common.AntreaInstallCNIContainerName],
+				Image: k8sComponents["antrea"+common.AntreaInstallCNIContainerName],
 			},
 		},
 	}
@@ -654,6 +654,36 @@ func (cs *ContainerService) setAddonsConfig(isUpgrade bool) {
 		},
 	}
 
+	defaultFlannelAddonsConfig := KubernetesAddon{
+		Name:    common.FlannelAddonName,
+		Enabled: to.BoolPtr(o.KubernetesConfig.NetworkPlugin == NetworkPluginFlannel),
+		Containers: []KubernetesContainerSpec{
+			{
+				Name:  common.KubeFlannelContainerName,
+				Image: k8sComponents[common.KubeFlannelContainerName],
+			},
+			{
+				Name:  common.FlannelInstallCNIContainerName,
+				Image: k8sComponents["flannel"+common.FlannelInstallCNIContainerName],
+			},
+		},
+	}
+
+	defaultScheduledMaintenanceAddonsConfig := KubernetesAddon{
+		Name:    common.ScheduledMaintenanceAddonName,
+		Enabled: to.BoolPtr(false),
+		Containers: []KubernetesContainerSpec{
+			{
+				Name:  common.KubeRBACProxyContainerName,
+				Image: k8sComponents[common.KubeRBACProxyContainerName],
+			},
+			{
+				Name:  common.ScheduledMaintenanceManagerContainerName,
+				Image: k8sComponents[common.ScheduledMaintenanceManagerContainerName],
+			},
+		},
+	}
+
 	// Allow folks to simply enable kube-dns at cluster creation time without also requiring that coredns be explicitly disabled
 	if !isUpgrade && o.KubernetesConfig.IsAddonEnabled(common.KubeDNSAddonName) {
 		defaultCorednsAddonsConfig.Enabled = to.BoolPtr(false)
@@ -693,6 +723,8 @@ func (cs *ContainerService) setAddonsConfig(isUpgrade bool) {
 		defaultAzureCloudProviderAddonsConfig,
 		defaultAADDefaultAdminGroupAddonsConfig,
 		defaultsAntreaDaemonSetAddonsConfig,
+		defaultFlannelAddonsConfig,
+		defaultScheduledMaintenanceAddonsConfig,
 	}
 	// Add default addons specification, if no user-provided spec exists
 	if o.KubernetesConfig.Addons == nil {

pkg/api/common/const.go

@@ -217,7 +217,15 @@ const (
 	// AntreaAddonName is the name of antrea daemonset addon
 	AntreaAddonName = "antrea"
 	// FlannelAddonName is the name of flannel plugin daemonset addon
-	FlannelAddonName = "flannel-daemonset"
+	FlannelAddonName = "flannel"
+	// KubeFlannelContainerName is the name of the kube-flannel container in the flannel addon
+	KubeFlannelContainerName = "kube-flannel"
+	// FlannelInstallCNIContainerName is the name of the install-cni container in the flannel addon
+	FlannelInstallCNIContainerName = "install-cni"
+	// KubeRBACProxyContainerName is the name of the kube-rbac-proxy container in the scheduled-maintenance addon
+	KubeRBACProxyContainerName = "kube-rbac-proxy"
+	// ScheduledMaintenanceManagerContainerName is the name of the manager container in the scheduled-maintenance addon
+	ScheduledMaintenanceManagerContainerName = "manager"
 	// AADAdminGroupAddonName is the name of the default admin group RBAC addon
 	AADAdminGroupAddonName = "aad"
 	// AzureCloudProviderAddonName is the name of the azure-cloud-provider addon

pkg/api/convertertoapi.go

@@ -365,7 +365,11 @@ func setVlabsKubernetesDefaults(vp *vlabs.Properties, api *OrchestratorProfile)
 		if vp.HasWindows() {
 			api.KubernetesConfig.NetworkPlugin = vlabs.DefaultNetworkPluginWindows
 		} else {
-			api.KubernetesConfig.NetworkPlugin = vlabs.DefaultNetworkPlugin
+			if vp.OrchestratorProfile.KubernetesConfig.IsAddonEnabled(common.FlannelAddonName) {
+				api.KubernetesConfig.NetworkPlugin = NetworkPluginFlannel
+			} else {
+				api.KubernetesConfig.NetworkPlugin = vlabs.DefaultNetworkPlugin
+			}
 		}
 	}
 }

pkg/api/convertertoapi_test.go

@@ -6,6 +6,7 @@ package api
 import (
 	"testing"
 
+	"github.com/Azure/aks-engine/pkg/api/common"
 	"github.com/Azure/go-autorest/autorest/to"
 
 	"github.com/davecgh/go-spew/spew"
@@ -1060,6 +1061,24 @@ func TestSetVlabsKubernetesDefaults(t *testing.T) {
 			expectedNetworkPlugin: "",
 			expectedNetworkPolicy: "antrea",
 		},
+		{
+			name: "flannel addon",
+			p: &vlabs.Properties{
+				OrchestratorProfile: &vlabs.OrchestratorProfile{
+					KubernetesConfig: &vlabs.KubernetesConfig{
+						NetworkPlugin: "",
+						Addons: []vlabs.KubernetesAddon{
+							{
+								Name:    common.FlannelAddonName,
+								Enabled: to.BoolPtr(true),
+							},
+						},
+					},
+				},
+			},
+			expectedNetworkPlugin: NetworkPluginFlannel,
+			expectedNetworkPolicy: "",
+		},
 	}
 
 	for _, test := range tests {

pkg/api/defaults.go

@@ -149,7 +149,11 @@ func (cs *ContainerService) setOrchestratorDefaults(isUpgrade, isScale bool) {
 			}
 		} else {
 			if o.KubernetesConfig.NetworkPlugin == "" {
-				o.KubernetesConfig.NetworkPlugin = DefaultNetworkPlugin
+				if o.KubernetesConfig.IsAddonEnabled(common.FlannelAddonName) {
+					o.KubernetesConfig.NetworkPlugin = NetworkPluginFlannel
+				} else {
+					o.KubernetesConfig.NetworkPlugin = DefaultNetworkPlugin
+				}
 			}
 		}
 		if o.KubernetesConfig.ContainerRuntime == "" {

pkg/api/defaults_test.go

@@ -223,6 +223,7 @@ func TestAssignDefaultAddonImages(t *testing.T) {
 		common.CoreDNSAddonName:                specConfig.KubernetesImageBase + k8sComponents[common.CoreDNSAddonName],
 		common.KubeProxyAddonName:              specConfig.KubernetesImageBase + k8sComponents[common.KubeProxyAddonName],
 		common.AntreaAddonName:                 k8sComponents[common.AntreaControllerContainerName],
+		common.FlannelAddonName:                k8sComponents[common.KubeFlannelContainerName],
 	}
 
 	customAddonImages := make(map[string]string)
@@ -294,6 +295,9 @@ func getFakeAddons(defaultAddonMap map[string]string, customImage string) []Kube
 		if addonName == common.AntreaAddonName {
 			containerName = common.AntreaControllerContainerName
 		}
+		if addonName == common.FlannelAddonName {
+			containerName = common.KubeFlannelContainerName
+		}
 		customAddon := KubernetesAddon{
 			Name:    addonName,
 			Enabled: to.BoolPtr(true),
@@ -878,6 +882,32 @@ func TestNetworkPolicyDefaults(t *testing.T) {
 	}
 }
 
+func TestNetworkPluginDefaults(t *testing.T) {
+	mockCS := getMockBaseContainerService("1.15.7")
+	properties := mockCS.Properties
+	properties.OrchestratorProfile.OrchestratorType = Kubernetes
+	mockCS.setOrchestratorDefaults(true, true)
+	if properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin != DefaultNetworkPlugin {
+		t.Fatalf("NetworkPlugin did not have the expected value, got %s, expected %s",
+			properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin, DefaultNetworkPlugin)
+	}
+
+	mockCS = getMockBaseContainerService("1.15.7")
+	properties = mockCS.Properties
+	properties.OrchestratorProfile.OrchestratorType = Kubernetes
+	properties.OrchestratorProfile.KubernetesConfig.Addons = []KubernetesAddon{
+		{
+			Name:    common.FlannelAddonName,
+			Enabled: to.BoolPtr(true),
+		},
+	}
+	mockCS.setOrchestratorDefaults(true, true)
+	if properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin != NetworkPluginFlannel {
+		t.Fatalf("NetworkPlugin did not have the expected value, got %s, expected %s",
+			properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin, NetworkPluginFlannel)
+	}
+}
+
 func TestContainerRuntime(t *testing.T) {
 
 	for _, mobyVersion := range []string{"3.0.1", "3.0.3", "3.0.4", "3.0.5", "3.0.6", "3.0.7", "3.0.8"} {