feat: Antrea plugin support in AKS Engine (#2407)

* Antrea plugin support in AKS Engine

* chore: clean up

* chore: use ContainerImage

* chore: generated code

* refactor: Updating antrea yaml to 0.2.0

Co-authored-by: Jack Francis <jackfrancis@gmail.com>

cmd/generate_test.go

@@ -645,6 +645,11 @@ func TestExampleAPIModels(t *testing.T) {
 			apiModelPath: "../examples/networkpolicy/kubernetes-cilium.json",
 			setArgs:      defaultSet,
 		},
+		{
+			name:         "antrea network policy",
+			apiModelPath: "../examples/networkpolicy/kubernetes-antrea.json",
+			setArgs:      defaultSet,
+		},
 		{
 			name:         "istio",
 			apiModelPath: "../examples/service-mesh/istio.json",

docs/topics/clusterdefinitions.md

@@ -61,8 +61,8 @@ $ aks-engine get-versions
 | kubeletConfig                    | no                        | Configure various runtime configuration for kubelet. See `kubeletConfig` [below](#feat-kubelet-config)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
 | kubernetesImageBase              | no                        | Specifies the default image base URL (everything preceding the actual image filename) to be used for all kubernetes-related containers such as hyperkube, cloud-controller-manager, pause, addon-manager, heapster, exechealthz etc. e.g., `k8s.gcr.io/`                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |
 | loadBalancerSku                  | no                        | Sku of Load Balancer and Public IP. Candidate values are: `basic` and `standard`. If not set, it will be default to basic. Requires Kubernetes 1.11 or newer. NOTE: VMs behind standard SKU load balancer will not be able to access the internet without an outbound rule configured with at least one frontend IP. We have created a loadbalancer with an outbound rule and with agent nodes added to the backend pool, as described in the [Outbound NAT for internal Standard Load Balancer scenarios doc](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-outbound-rules-overview#outbound-nat-for-internal-standard-load-balancer-scenarios)                                                                     |
-| networkPlugin                    | no                        | Specifies the network plugin implementation for the cluster. Valid values are:<br>`"azure"` (default), which provides an Azure native networking experience <br>`"kubenet"` for k8s software networking implementation. <br> `"flannel"` for using CoreOS Flannel <br> `"cilium"` for using the default Cilium CNI IPAM (requires the `"cilium"` networkPolicy as well)                                                                                                                                                                                                                                                                                                                                                                                                                  |
-| networkPolicy                    | no                        | Specifies the network policy enforcement tool for the cluster (currently Linux-only). Valid values are:<br>`"calico"` for Calico network policy.<br>`"cilium"` for cilium network policy (uses the `"cilium"` networkPlugin exclusively), and `"azure"` (experimental) for Azure CNI-compliant network policy (note: Azure CNI-compliant network policy requires explicit `"networkPlugin": "azure"` configuration as well).<br>See [network policy examples](../../examples/networkpolicy) for more information.                                                                                                                                                                                                                                                                   |
+| networkPlugin                    | no                        | Specifies the network plugin implementation for the cluster. Valid values are:<br>`"azure"` (default), which provides an Azure native networking experience <br>`"kubenet"` for k8s software networking implementation. <br> `"flannel"` for using CoreOS Flannel <br> `"cilium"` for using the default Cilium CNI IPAM (requires the `"cilium"` networkPolicy as well)<br> `"antrea"` for using the Antrea network plugin (requires the `"antrea"` networkPolicy as well)                                                                                                                                                                                                                                                                                                                                                                                                                   |
+| networkPolicy                    | no                        | Specifies the network policy enforcement tool for the cluster (currently Linux-only). Valid values are:<br>`"calico"` for Calico network policy.<br>`"cilium"` for cilium network policy (uses the `"cilium"` networkPlugin exclusively).<br> `"antrea"` for Antrea network policy (uses the `"antrea"` networkPlugin exclusively).<br> `"azure"` (experimental) for Azure CNI-compliant network policy (note: Azure CNI-compliant network policy requires explicit `"networkPlugin": "azure"` configuration as well).<br>See [network policy examples](../../examples/networkpolicy) for more information.                                                                                                                                                                                                                                                                   |
 | privateCluster                   | no                        | Build a cluster without public addresses assigned. See `privateClusters` [below](#feat-private-cluster).                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |
 | schedulerConfig                  | no                        | Configure various runtime configuration for scheduler. See `schedulerConfig` [below](#feat-scheduler-config)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |
 | serviceCidr                      | no                        | IP range for Service IPs, Default is "10.0.0.0/16". This range is never routed outside of a node so does not need to lie within clusterSubnet or the VNET                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |

docs/topics/features.md

@@ -5,6 +5,7 @@
 |Managed Disks|Beta|`vlabs`|[kubernetes-vmas.json](../../examples/disks-managed/kubernetes-vmas.json)|[Description](#feat-managed-disks)|
 |Calico Network Policy|Alpha|`vlabs`|[kubernetes-calico.json](../../examples/networkpolicy/kubernetes-calico-azure.json)|[Description](#feat-calico)|
 |Cilium Network Policy|Alpha|`vlabs`|[kubernetes-cilium.json](../../examples/networkpolicy/kubernetes-cilium.json)|[Description](#feat-cilium)|
+|Antrea Network Policy|Alpha|`vlabs`|[kubernetes-antrea.json](../../examples/networkpolicy/kubernetes-antrea.json)|[Description](#feat-antrea)|
 |Custom VNET|Beta|`vlabs`|[kubernetesvnet-azure-cni.json](../../examples/vnet/kubernetesvnet-azure-cni.json)|[Description](#feat-custom-vnet)|
 |Kata Containers Runtime|Alpha|`vlabs`|[kubernetes-kata-containers.json](../../examples/kubernetes-kata-containers.json)|[Description](#feat-kata-containers)|
 |Private Cluster|Alpha|`vlabs`|[kubernetes-private-cluster.json](../../examples/kubernetes-config/kubernetes-private-cluster.json)|[Description](#feat-private-cluster)|
@@ -224,6 +225,39 @@ you can define stricter policies. Good resources to get information about that a
 - [NetworkPolicy Example Walkthrough](https://kubernetes.io/docs/getting-started-guides/network-policy/walkthrough/)
 - [Cilium Kubernetes](https://github.com/Azure/aks-engine/blob/master/examples/networkpolicy)
 
+<a name="feat-antrea"></a>
+
+## Network Policy Enforcement with Antrea
+
+Using the default configuration, Kubernetes allows communication between all
+Pods within a cluster. To ensure that Pods can only be accessed by authorized
+Pods, a policy enforcement is needed. To enable policy enforcement using Antrea refer to the
+[cluster definitions](clusterdefinitions.md#kubernetesconfig)
+document under networkPolicy. There is also a reference cluster definition available
+[here](https://github.com/Azure/aks-engine/blob/master/examples/networkpolicy/kubernetes-antrea.json).
+
+This will deploy single replica of Antrea controller and Antrea agent to every
+instance of the cluster using a Kubernetes DaemonSet. After a successful deployment
+you should be able to see these Pods running in your cluster:
+
+```console
+kubectl get pods --namespace kube-system  -l app=antrea -o wide
+NAME                                 READY   STATUS    RESTARTS   AGE     IP             NODE
+antrea-agent-67t9z                   2/2     Running   1          7m38s   10.240.0.5     k8s-agentpool1-14956401-vmss000001
+antrea-agent-87nm2                   2/2     Running   0          11m     10.240.0.4     k8s-agentpool1-14956401-vmss000000
+antrea-agent-fhbsg                   2/2     Running   0          11m     10.240.0.6     k8s-agentpool1-14956401-vmss000002
+antrea-agent-jjhxt                   2/2     Running   0          11m     10.240.255.5   k8s-master-14956401-0
+antrea-controller-685c8c6f64-zk4jh   1/1     Running   0          11m     10.240.0.4     k8s-agentpool1-14956401-vmss000000
+```
+
+Per default Antrea still allows all communication within the cluster. Using Kubernetes' NetworkPolicy API,
+you can define stricter policies. Good resources to get information about that are:
+
+- [Antrea Architecture Docs](https://github.com/vmware-tanzu/antrea/blob/master/docs/architecture.md)
+- [NetworkPolicy User Guide](https://kubernetes.io/docs/user-guide/networkpolicies/)
+- [NetworkPolicy Example Walkthrough](https://kubernetes.io/docs/getting-started-guides/network-policy/walkthrough/)
+- [Antrea Kubernetes](https://github.com/Azure/aks-engine/blob/master/examples/networkpolicy)
+
 <a name="feat-custom-vnet"></a>
 
 ## Custom VNET

docs/tutorials/deploy.md

@@ -141,6 +141,7 @@ aks-engine generate --set agentPoolProfiles[0].count=5,agentPoolProfiles[1].name
 
 * To enable the optional network policy enforcement using calico, you have to set the parameter during this step according to this [guide](../topics/features.md#optional-enable-network-policy-enforcement-using-calico)
 * To enable the optional network policy enforcement using cilium, you have to set the parameter during this step according to this [guide](../topics/features.md#optional-enable-network-policy-enforcement-using-cilium)
+* To enable the optional network policy enforcement using antrea, you have to set the parameter during this step according to this [guide](../topics/features.md#optional-enable-network-policy-enforcement-using-antrea)
 
 **Note**: If the cluster is using an existing VNET please see the [Custom VNET](custom-vnet.md) feature documentation for additional steps that must be completed after cluster provisioning.
 

docs/tutorials/quickstart.md

@@ -228,6 +228,7 @@ aks-engine generate --set agentPoolProfiles[0].count=5,agentPoolProfiles[1].name
 
 * To enable the optional network policy enforcement using calico, you have to set the parameter during this step according to this [guide](../topics/features.md#optional-enable-network-policy-enforcement-using-calico)
 * To enable the optional network policy enforcement using cilium, you have to set the parameter during this step according to this [guide](../topics/features.md#optional-enable-network-policy-enforcement-using-cilium)
+* To enable the optional network policy enforcement using antrea, you have to set the parameter during this step according to this [guide](../topics/features.md#optional-enable-network-policy-enforcement-using-antrea)
 
 Now we can deploy the files `azuredeploy.json` and `azuredeploy.parameters.json` using either the Azure CLI or PowerShell.
 

examples/kubernetes.json

@@ -2,7 +2,10 @@
   "apiVersion": "vlabs",
   "properties": {
     "orchestratorProfile": {
-      "orchestratorType": "Kubernetes"
+      "orchestratorType": "Kubernetes",
+      "kubernetesConfig": {
+        "networkPolicy": "antrea"
+      }
     },
     "masterProfile": {
       "count": 1,

examples/networkplugin/README.md

@@ -1,11 +1,12 @@
 # AKS Engine - Network Plugin
 
-There are 2 different Network Plugin options :
+There are 5 different Network Plugin options :
 
 - Azure Container Networking (default)
 - Kubenet
 - Flannel (docs are //TODO)
-- Cilium (docs are //TODO)
+- Cilium (CNI IPAM implementation that pairs w/ cilium NetworkPolicy addon; only works w/ `"networkPolicy": "cilium"`)
+- Antrea (CNI IPAM implementation that pairs w/ antrea NetworkPolicy addon; only works w/ `"networkPolicy": "antrea"`)
 
 ## Azure Container Networking (default)
 

examples/networkpolicy/README.md

@@ -1,9 +1,10 @@
 # AKS Engine - Network Policy
 
-There are 2 different Network Policy options :
+There are 3 different Network Policy options :
 
 - Calico
 - Cilium
+- Antrea
 
 ## Calico
 
@@ -97,3 +98,20 @@ The kubernetes-cilium deployment template enables Cilium networking and policies
 Once the template has been successfully deployed, following the [deploy the demo application](http://cilium.readthedocs.io/en/latest/gettingstarted/minikube/#step-2-deploy-the-demo-application) tutorial will provide a good foundation for how to do L3/4 policy as well as more advanced Layer 7 inspection and routing. If you have [Istio](https://istio.io) you can try this [tutorial](http://cilium.readthedocs.io/en/latest/gettingstarted/istio/) where cilium is used to side by side with Istio to enforce security policies in a Kubernetes deployment.
 
 For the latest documentation on Cilium (including BPF and XDP reference guides), please refer to [this](http://cilium.readthedocs.io/en/latest/)
+
+## Antrea
+
+The kubernetes-antrea deployment template enables Antrea networking and policies for the AKS Engine cluster via `"networkPolicy": "antrea"` or `"networkPlugin": "antrea"` being present inside the `kubernetesConfig`.
+
+```json
+  "properties": {
+    "orchestratorProfile": {
+      "orchestratorType": "Kubernetes",
+      "kubernetesConfig": {
+        "networkPolicy": "antrea"
+      }
+```
+
+### Post installation
+
+For the latest documentation on Antrea, please refer to [this](https://github.com/vmware-tanzu/antrea).

examples/networkpolicy/kubernetes-antrea.json

@@ -0,0 +1,39 @@
+{
+  "apiVersion": "vlabs",
+  "properties": {
+    "orchestratorProfile": {
+      "orchestratorType": "Kubernetes",
+      "orchestratorRelease": "1.13",
+      "kubernetesConfig": {
+        "networkPolicy": "antrea"
+      }
+    },
+    "masterProfile": {
+      "count": 1,
+      "dnsPrefix": "",
+      "vmSize": "Standard_D2_v3"
+    },
+    "agentPoolProfiles": [
+      {
+        "name": "agentpool1",
+        "count": 3,
+        "vmSize": "Standard_D2_v3",
+        "availabilityProfile": "AvailabilitySet"
+      }
+    ],
+    "linuxProfile": {
+      "adminUsername": "azureuser",
+      "ssh": {
+        "publicKeys": [
+          {
+            "keyData": ""
+          }
+        ]
+      }
+    },
+    "servicePrincipalProfile": {
+      "clientId": "",
+      "secret": ""
+    }
+  }
+}

parts/k8s/cloud-init/artifacts/cse_config.sh

@@ -312,6 +312,11 @@ ensureKubelet() {
         sleep 3
     done
     {{end}}
+    {{if HasAntreaNetworkPolicy}}
+    while [ ! -f /etc/cni/net.d/10-antrea.conf ]; do
+        sleep 3
+    done
+    {{end}}
 }
 
 ensureLabelNodes() {