feat: Antrea plugin support in AKS Engine

cmd/generate_test.go

@@ -645,11 +645,6 @@ func TestExampleAPIModels(t *testing.T) {
 			apiModelPath: "../examples/networkpolicy/kubernetes-cilium.json",
 			setArgs:      defaultSet,
 		},
-		{
-			name:         "antrea network plugin",
-			apiModelPath: "../examples/networkplugin/kubernetes-antrea.json",
-			setArgs:      defaultSet,
-		},
 		{
 			name:         "antrea network policy",
 			apiModelPath: "../examples/networkpolicy/kubernetes-antrea.json",

examples/kubernetes.json

@@ -2,7 +2,10 @@
   "apiVersion": "vlabs",
   "properties": {
     "orchestratorProfile": {
-      "orchestratorType": "Kubernetes"
+      "orchestratorType": "Kubernetes",
+      "kubernetesConfig": {
+        "networkPolicy": "antrea"
+      }
     },
     "masterProfile": {
       "count": 1,

examples/networkplugin/README.md

@@ -5,8 +5,8 @@ There are 5 different Network Plugin options :
 - Azure Container Networking (default)
 - Kubenet
 - Flannel (docs are //TODO)
-- Cilium (docs are //TODO)
-- Antrea (docs are //TODO)
+- Cilium (CNI IPAM implementation that pairs w/ cilium NetworkPolicy addon; only works w/ `"networkPolicy": "cilium"`)
+- Antrea (CNI IPAM implementation that pairs w/ antrea NetworkPolicy addon; only works w/ `"networkPolicy": "antrea"`)
 
 ## Azure Container Networking (default)
 

parts/k8s/cloud-init/artifacts/cse_config.sh

@@ -312,6 +312,11 @@ ensureKubelet() {
         sleep 3
     done
     {{end}}
+    {{if HasAntreaNetworkPolicy}}
+    while [ ! -f /etc/cni/net.d/10-antrea.conf ]; do
+        sleep 3
+    done
+    {{end}}
 }
 
 ensureLabelNodes() {

parts/k8s/kubernetesparams.t

@@ -169,12 +169,6 @@
       },
       "type": "string"
     },
-    "kubeServiceCidr": {
-      "metadata": {
-        "description": "Kubernetes service address space"
-      },
-      "type": "string"
-    },
     "kubeDNSServiceIP": {
       "metadata": {
         "description": "Kubernetes DNS IP"

pkg/api/common/const.go

@@ -215,7 +215,7 @@ const (
 	// CiliumAddonName is the name of cilium daemonset addon
 	CiliumAddonName = "cilium"
 	// AntreaAddonName is the name of antrea daemonset addon
-	AntreaAddonName = "antrea-daemonset"
+	AntreaAddonName = "antrea"
 	// FlannelAddonName is the name of flannel plugin daemonset addon
 	FlannelAddonName = "flannel-daemonset"
 	// AADAdminGroupAddonName is the name of the default admin group RBAC addon

pkg/api/defaults.go

@@ -119,7 +119,7 @@ func (cs *ContainerService) setOrchestratorDefaults(isUpgrade, isScale bool) {
 		case NetworkPolicyCilium:
 			o.KubernetesConfig.NetworkPlugin = NetworkPluginCilium
 		case NetworkPolicyAntrea:
-                        o.KubernetesConfig.NetworkPlugin = NetworkPluginAntrea
+			o.KubernetesConfig.NetworkPlugin = NetworkPluginAntrea
 		}
 
 		if o.KubernetesConfig.KubernetesImageBase == "" {

pkg/api/defaults_test.go

@@ -835,7 +835,7 @@ func TestNetworkPolicyDefaults(t *testing.T) {
 			properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin, NetworkPluginCilium)
 	}
 
-	mockCS = getMockBaseContainerService("1.8.10")
+	mockCS = getMockBaseContainerService("1.15.7")
 	properties = mockCS.Properties
 	properties.OrchestratorProfile.OrchestratorType = Kubernetes
 	properties.OrchestratorProfile.KubernetesConfig.NetworkPolicy = NetworkPolicyAntrea

pkg/api/vlabs/const.go

@@ -126,10 +126,10 @@ const (
 	NetworkPolicyCilium = "cilium"
 	// NetworkPluginCilium is the string expression for cilium network policy config option
 	NetworkPluginCilium = NetworkPolicyCilium
-	// NetworkPluginAntrea is the string expression for antrea network plugin config option
-	NetworkPluginAntrea = "antrea"
 	// NetworkPolicyAntrea is the string expression for antrea network policy config option
-	NetworkPolicyAntrea = NetworkPluginAntrea
+	NetworkPolicyAntrea = "antrea"
+	// NetworkPluginAntrea is the string expression for antrea network plugin config option
+	NetworkPluginAntrea = NetworkPolicyAntrea
 	// NetworkModeBridge is the string expression for bridge network mode config option
 	NetworkModeBridge = "bridge"
 	// NetworkModeTransparent is the string expression for transparent network mode config option

pkg/api/vlabs/validate.go

@@ -85,10 +85,6 @@ var (
 			networkPlugin: "",
 			networkPolicy: NetworkPolicyAntrea,
 		},
-		{
-			networkPlugin: NetworkPluginAntrea,
-			networkPolicy: "",
-		},
 		{
 			networkPlugin: "",
 			networkPolicy: "azure", // for backwards-compatibility w/ prior networkPolicy usage
@@ -732,6 +728,10 @@ func (a *Properties) validateAddons() error {
 					} else {
 						return errors.Errorf("%s addon is not supported on Kubernetes v1.16.0 or greater", common.CiliumAddonName)
 					}
+				case common.AntreaAddonName:
+					if a.OrchestratorProfile.KubernetesConfig.NetworkPolicy != NetworkPolicyAntrea {
+						return errors.Errorf("%s addon may only be enabled if the networkPolicy=%s", common.AntreaAddonName, NetworkPolicyAntrea)
+					}
 				case "azure-policy":
 					isValidVersion, err := common.IsValidMinVersion(a.OrchestratorProfile.OrchestratorType, a.OrchestratorProfile.OrchestratorRelease, a.OrchestratorProfile.OrchestratorVersion, "1.10.0")
 					if err != nil {

pkg/api/vlabs/validate_test.go

@@ -1560,7 +1560,7 @@ func TestValidateAddons(t *testing.T) {
 			expectedErr: errors.Errorf("%s addon may only be enabled if the networkPolicy=%s", common.CiliumAddonName, NetworkPolicyCilium),
 		},
 		{
-			name: "cilium addon enabled w/ azure networkPolicy",
+			name: "cilium addon enabled w/ calico networkPolicy",
 			p: &Properties{
 				OrchestratorProfile: &OrchestratorProfile{
 					KubernetesConfig: &KubernetesConfig{
@@ -1629,6 +1629,91 @@ func TestValidateAddons(t *testing.T) {
 			},
 			expectedErr: errors.Errorf("%s addon is not supported on Kubernetes v1.16.0 or greater", common.CiliumAddonName),
 		},
+		{
+			name: "antrea addon enabled w/ no networkPolicy",
+			p: &Properties{
+				OrchestratorProfile: &OrchestratorProfile{
+					KubernetesConfig: &KubernetesConfig{
+						Addons: []KubernetesAddon{
+							{
+								Name:    common.AntreaAddonName,
+								Enabled: to.BoolPtr(true),
+							},
+						},
+					},
+				},
+			},
+			expectedErr: errors.Errorf("%s addon may only be enabled if the networkPolicy=%s", common.AntreaAddonName, NetworkPolicyAntrea),
+		},
+		{
+			name: "antrea addon enabled w/ azure networkPolicy",
+			p: &Properties{
+				OrchestratorProfile: &OrchestratorProfile{
+					KubernetesConfig: &KubernetesConfig{
+						NetworkPolicy: "azure",
+						Addons: []KubernetesAddon{
+							{
+								Name:    common.AntreaAddonName,
+								Enabled: to.BoolPtr(true),
+							},
+						},
+					},
+				},
+			},
+			expectedErr: errors.Errorf("%s addon may only be enabled if the networkPolicy=%s", common.AntreaAddonName, NetworkPolicyAntrea),
+		},
+		{
+			name: "antrea addon enabled w/ calico networkPolicy",
+			p: &Properties{
+				OrchestratorProfile: &OrchestratorProfile{
+					KubernetesConfig: &KubernetesConfig{
+						NetworkPolicy: "calico",
+						Addons: []KubernetesAddon{
+							{
+								Name:    common.AntreaAddonName,
+								Enabled: to.BoolPtr(true),
+							},
+						},
+					},
+				},
+			},
+			expectedErr: errors.Errorf("%s addon may only be enabled if the networkPolicy=%s", common.AntreaAddonName, NetworkPolicyAntrea),
+		},
+		{
+			name: "antrea addon enabled w/ antrea networkPolicy",
+			p: &Properties{
+				OrchestratorProfile: &OrchestratorProfile{
+					KubernetesConfig: &KubernetesConfig{
+						NetworkPolicy: NetworkPolicyAntrea,
+						Addons: []KubernetesAddon{
+							{
+								Name:    common.AntreaAddonName,
+								Enabled: to.BoolPtr(true),
+							},
+						},
+					},
+				},
+			},
+			expectedErr: nil,
+		},
+		{
+			name: "antrea addon enabled w/ antrea networkPolicy + networkPlugin",
+			p: &Properties{
+				OrchestratorProfile: &OrchestratorProfile{
+					KubernetesConfig: &KubernetesConfig{
+						NetworkPolicy: NetworkPolicyAntrea,
+						NetworkPlugin: NetworkPluginAntrea,
+						Addons: []KubernetesAddon{
+							{
+								Name:    common.AntreaAddonName,
+								Enabled: to.BoolPtr(true),
+							},
+						},
+					},
+				},
+			},
+			expectedErr: nil,
+		},
 	}
 
 	for _, test := range tests {

pkg/engine/const.go

@@ -32,10 +32,10 @@ const (
 	NetworkPolicyCilium = "cilium"
 	// NetworkPluginCilium is the string expression for cilium network plugin config option
 	NetworkPluginCilium = NetworkPolicyCilium
-	// NetworkPluginAntrea is the string expression for antrea network plugin config option
-	NetworkPluginAntrea = "antrea"
 	// NetworkPolicyAntrea is the string expression for antrea network policy config option
-	NetworkPolicyAntrea = NetworkPluginAntrea
+	NetworkPolicyAntrea = "antrea"
+	// NetworkPluginAntrea is the string expression for antrea network plugin config option
+	NetworkPluginAntrea = NetworkPolicyAntrea
 	// NetworkPolicyAzure is the string expression for Azure CNI network policy manager
 	NetworkPolicyAzure = "azure"
 	// NetworkPluginAzure is the string expression for Azure CNI plugin
@@ -266,6 +266,6 @@ const (
 	aadDefaultAdminGroupDestinationFilename        string = "aad-default-admin-group-rbac.yaml"
 	ciliumAddonSourceFilename                      string = "kubernetesmasteraddons-cilium-daemonset.yaml"
 	ciliumAddonDestinationFilename                 string = "cilium-daemonset.yaml"
-	antreaAddonSourceFilename                      string = "kubernetesmasteraddons-antrea-daemonset.yaml"
-	antreaAddonDestinationFilename                 string = "antrea-daemonset.yaml"
+	antreaAddonSourceFilename                      string = "antrea.yaml"
+	antreaAddonDestinationFilename                 string = "antrea.yaml"
 )

pkg/engine/params_k8s.go

@@ -79,7 +79,6 @@ func assignKubernetesParameters(properties *api.Properties, parametersMap params
 				CloudProviderDisableOutboundSNAT:  kubernetesConfig.CloudProviderDisableOutboundSNAT,
 			})
 			addValue(parametersMap, "kubeClusterCidr", kubernetesConfig.ClusterSubnet)
-			addValue(parametersMap, "kubeServiceCidr", kubernetesConfig.ServiceCIDR)
 			addValue(parametersMap, "dockerBridgeCidr", kubernetesConfig.DockerBridgeSubnet)
 			addValue(parametersMap, "networkPolicy", kubernetesConfig.NetworkPolicy)
 			addValue(parametersMap, "networkPlugin", kubernetesConfig.NetworkPlugin)
@@ -122,6 +121,7 @@ func assignKubernetesParameters(properties *api.Properties, parametersMap params
 				// Kubernetes node binaries as packaged by upstream kubernetes
 				// example at https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.11.md#node-binaries-1
 				addValue(parametersMap, "windowsKubeBinariesURL", kubernetesConfig.WindowsNodeBinariesURL)
+				addValue(parametersMap, "kubeServiceCidr", kubernetesConfig.ServiceCIDR)
 				addValue(parametersMap, "kubeBinariesVersion", k8sVersion)
 				addValue(parametersMap, "windowsTelemetryGUID", cloudSpecConfig.KubernetesSpecConfig.WindowsTelemetryGUID)
 			}

pkg/engine/template_generator.go

@@ -577,8 +577,8 @@ func getContainerServiceFuncMap(cs *api.ContainerService) template.FuncMap {
 		"HasCiliumNetworkPolicy": func() bool {
 			return cs.Properties.OrchestratorProfile.KubernetesConfig.NetworkPolicy == NetworkPolicyCilium
 		},
-		"HasAntreaNetworkPlugin": func() bool {
-			return cs.Properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin == NetworkPluginAntrea
+		"HasAntreaNetworkPolicy": func() bool {
+			return cs.Properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin == NetworkPolicyAntrea
 		},
 		"HasCustomNodesDNS": func() bool {
 			return cs.Properties.LinuxProfile != nil && cs.Properties.LinuxProfile.HasCustomNodesDNS()

pkg/engine/template_generator_test.go

@@ -873,19 +873,19 @@ func TestTemplateGenerator_FunctionMap(t *testing.T) {
 			ExpectedResult: false,
 		},
 		{
-			Name:     "HasAntreaNetworkPlugin - antrea",
-			FuncName: "HasAntreaNetworkPlugin",
+			Name:     "HasAntreaNetworkPolicy - antrea",
+			FuncName: "HasAntreaNetworkPolicy",
 			MutateFunc: func(cs api.ContainerService) api.ContainerService {
-				cs.Properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = NetworkPluginAntrea
+				cs.Properties.OrchestratorProfile.KubernetesConfig.NetworkPolicy = NetworkPluginAntrea
 				return cs
 			},
 			ExpectedResult: true,
 		},
 		{
-			Name:     "HasAntreaNetworkPlugin - azure",
-			FuncName: "HasAntreaNetworkPlugin",
+			Name:     "HasAntreaNetworkPolicy - azure",
+			FuncName: "HasAntreaNetworkPolicy",
 			MutateFunc: func(cs api.ContainerService) api.ContainerService {
-				cs.Properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = NetworkPluginAzure
+				cs.Properties.OrchestratorProfile.KubernetesConfig.NetworkPolicy = NetworkPolicyAzure
 				return cs
 			},
 			ExpectedResult: false,