add containerd and fix flannel and cilium (#2967)

* add containerd to container runtimes Signed-off-by: Jess Frazelle <acidburn@microsoft.com> * fix flannel and cilium closes #2966 Signed-off-by: Jess Frazelle <acidburn@microsoft.com>
Azure · May 15, 2018 · 244d661 · 244d661
1 parent 976390c
commit 244d661
Show file tree

Hide file tree

Showing 17 changed files with 196 additions and 66 deletions.
diff --git a/docs/clusterdefinition.md b/docs/clusterdefinition.md
@@ -32,9 +32,9 @@ Here are the valid values for the orchestrator types:
 |---|---|---|
 |kubernetesImageBase|no|Specifies the base URL (everything preceding the actual image filename) of the kubernetes hyperkube image to use for cluster deployment, e.g., `k8s-gcrio.azureedge.net/`|
 |dockerEngineVersion|no|Which version of docker-engine to use in your cluster, e.g. "17.03.*"|
-|networkPlugin|no|Specifies the network plugin implementation for the cluster. Valid values are:<br>`"azure"` (default), which provides an Azure native networking experience, or <br>`"kubenet"` for k8s software networking implementation.|
+|networkPlugin|no|Specifies the network plugin implementation for the cluster. Valid values are:<br>`"azure"` (default), which provides an Azure native networking experience <br>`"kubenet"` for k8s software networking implementation. <br> `"flannel"` for using CoreOS Flannel <br> `"cilium"` for using the default Cilium CNI IPAM |
 |networkPolicy|no|Specifies the network policy enforcement tool for the cluster (currently Linux-only). Valid values are:<br>`calico` for Calico network policy.<br>`cilium` for cilium network policy (Lin).<br>See [network policy examples](../examples/networkpolicy) for more information|
-|containerRuntime|no|The container runtime to use as a backend. The default is `docker`. The only other option is `clear-containers`|
+|containerRuntime|no|The container runtime to use as a backend. The default is `docker`. The other options are `clear-containers` and `containerd`|
 |clusterSubnet|no|The IP subnet used for allocating IP addresses for pod network interfaces. The subnet must be in the VNET address space. Default value is 10.244.0.0/16|
 |dnsServiceIP|no|IP address for kube-dns to listen on. If specified must be in the range of `serviceCidr`|
 |dockerBridgeSubnet|no|The specific IP and subnet used for allocating IP addresses for the docker bridge network created on the kubernetes master and agents. Default value is 172.17.0.1/16. This value is used to configure the docker daemon using the [--bip flag](https://docs.docker.com/engine/userguide/networking/default_network/custom-docker0)|

diff --git a/examples/e2e-tests/kubernetes/kubernetes-config/clear-containers.json b/examples/e2e-tests/kubernetes/kubernetes-config/clear-containers.json
@@ -5,7 +5,7 @@
       "orchestratorType": "Kubernetes",
       "orchestratorRelease": "1.10",
       "kubernetesConfig": {
-        "networkPlugin": "azure",
+        "networkPlugin": "flannel",
         "containerRuntime": "clear-containers"
       }
     },

diff --git a/examples/e2e-tests/kubernetes/kubernetes-config/containerd.json b/examples/e2e-tests/kubernetes/kubernetes-config/containerd.json
@@ -0,0 +1,40 @@
+{
+  "apiVersion": "vlabs",
+  "properties": {
+    "orchestratorProfile": {
+      "orchestratorType": "Kubernetes",
+      "orchestratorRelease": "1.10",
+      "kubernetesConfig": {
+        "networkPlugin": "flannel",
+        "containerRuntime": "containerd"
+      }
+    },
+    "masterProfile": {
+      "count": 1,
+      "dnsPrefix": "",
+      "vmSize": "Standard_D2_v2"
+    },
+    "agentPoolProfiles": [
+      {
+        "name": "agentpool1",
+        "count": 3,
+        "vmSize": "Standard_D2_v2",
+        "availabilityProfile": "AvailabilitySet"
+      }
+    ],
+    "linuxProfile": {
+      "adminUsername": "azureuser",
+      "ssh": {
+        "publicKeys": [
+          {
+            "keyData": ""
+          }
+        ]
+      }
+    },
+    "servicePrincipalProfile": {
+      "clientId": "",
+      "secret": ""
+    }
+  }
+}
diff --git a/examples/e2e-tests/kubernetes/kubernetes-config/flannel.json b/examples/e2e-tests/kubernetes/kubernetes-config/flannel.json
@@ -0,0 +1,39 @@
+{
+  "apiVersion": "vlabs",
+  "properties": {
+    "orchestratorProfile": {
+      "orchestratorType": "Kubernetes",
+      "kubernetesConfig": {
+        "networkPlugin":"flannel",
+      }
+    },
+    "masterProfile": {
+      "count": 1,
+      "dnsPrefix": "",
+      "vmSize": "Standard_D2_v2"
+    },
+    "agentPoolProfiles": [
+      {
+        "name": "linuxpool1",
+        "count": 3,
+        "vmSize": "Standard_D2_v2",
+        "availabilityProfile": "AvailabilitySet"
+      }
+    ],
+    "linuxProfile": {
+      "adminUsername": "azureuser",
+      "ssh": {
+        "publicKeys": [
+          {
+            "keyData": ""
+          }
+        ]
+      }
+    },
+    "servicePrincipalProfile": {
+      "clientId": "",
+      "secret": ""
+    },
+    "certificateProfile": {}
+  }
+}
diff --git a/examples/kubernetes-clear-containers.json b/examples/kubernetes-clear-containers.json
@@ -5,8 +5,7 @@
       "orchestratorType": "Kubernetes",
       "orchestratorRelease": "1.10",
       "kubernetesConfig": {
-        "networkPlugin": "kubenet",
-        "networkPolicy": "flannel",
+        "networkPlugin": "flannel",
         "containerRuntime": "clear-containers",
         "addons": [
           {

diff --git a/examples/kubernetes-containerd.json b/examples/kubernetes-containerd.json
@@ -0,0 +1,50 @@
+{
+  "apiVersion": "vlabs",
+  "properties": {
+    "orchestratorProfile": {
+      "orchestratorType": "Kubernetes",
+      "orchestratorRelease": "1.10",
+      "kubernetesConfig": {
+        "networkPlugin": "flannel",
+        "containerRuntime": "containerd",
+        "addons": [
+          {
+            "name": "tiller",
+            "enabled" : false
+          },
+          {
+            "name": "kubernetes-dashboard",
+            "enabled" : false
+          }
+        ]
+      }
+    },
+    "masterProfile": {
+      "count": 1,
+      "dnsPrefix": "",
+      "vmSize": "Standard_D2_v2"
+    },
+    "agentPoolProfiles": [
+      {
+        "name": "agentpool1",
+        "count": 3,
+        "vmSize": "Standard_D4s_v3",
+        "availabilityProfile": "AvailabilitySet"
+      }
+    ],
+    "linuxProfile": {
+      "adminUsername": "azureuser",
+      "ssh": {
+        "publicKeys": [
+          {
+            "keyData": ""
+          }
+        ]
+      }
+    },
+    "servicePrincipalProfile": {
+      "clientId": "",
+      "secret": ""
+    }
+  }
+}
diff --git a/examples/networkplugin/README.md b/examples/networkplugin/README.md
@@ -4,6 +4,8 @@ There are 2 different Network Plugin options :
 
 - Azure Container Networking (default)
 - Kubenet
+- Flannel (docs are //TODO) 
+- Cilium (docs are //TODO)
 
 ## Azure Container Networking (default)
 

diff --git a/examples/networkpolicy/README.md b/examples/networkpolicy/README.md
@@ -30,4 +30,4 @@ To understand how to deploy this template, please read the baseline [Kubernetes]
 
 Once the template has been successfully deployed, following the [simple policy tutorial](https://docs.projectcalico.org/v3.0/getting-started/kubernetes/tutorials/simple-policy) or the [advanced policy tutorial](https://docs.projectcalico.org/v3.0/getting-started/kubernetes/tutorials/advanced-policy) will help to understand calico networking.
 
-> Note: `ping` (ICMP) traffic is blocked on the cluster by default.  Wherever `ping` is used in any tutorial substitute testing access with something like `wget -q --timeout=5 google.com -O -` instead.
+> Note: `ping` (ICMP) traffic is blocked on the cluster by default.  Wherever `ping` is used in any tutorial substitute testing access with something like `wget -q --timeout=5 google.com -O -` instead.
diff --git a/parts/k8s/kubernetescustomscript.sh b/parts/k8s/kubernetescustomscript.sh
@@ -260,22 +260,13 @@ function configAzureCNI() {
     /sbin/ebtables -t nat --list
 }
 
-function configKubenet() {
-    installCNI
-}
-
-function configFlannel() {
-    installCNI
-    setDockerOpts " --volume=/etc/cni/:/etc/cni:ro --volume=/opt/cni/:/opt/cni:ro"
-}
-
 function configNetworkPlugin() {
     if [[ "${NETWORK_PLUGIN}" = "azure" ]]; then
         configAzureCNI
-    elif [[ "${NETWORK_PLUGIN}" = "kubenet" ]] ; then
+    elif [[ "${NETWORK_PLUGIN}" = "kubenet" ]]; then
+		installCNI
+	elif [[ "${NETWORK_PLUGIN}" = "flannel" ]]; then
         installCNI
-    elif [[ "${NETWORK_POLICY}" = "flannel" ]] ; then
-        configCNINetworkPolicy
     fi
 }
 
@@ -301,8 +292,6 @@ function installClearContainersRuntime() {
 	# Enable and start Clear Containers proxy service
 	echo "Enabling and starting Clear Containers proxy service..."
 	systemctlEnableAndStart cc-proxy
-
-	setKubeletOpts " --container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock"
 }
 
 function installContainerd() {
@@ -327,21 +316,24 @@ function setupContainerd() {
 	echo "oom_score = 0" >> "$CRI_CONTAINERD_CONFIG"
 	echo "[plugins.cri.containerd.untrusted_workload_runtime]" >> "$CRI_CONTAINERD_CONFIG"
 	echo "runtime_type = 'io.containerd.runtime.v1.linux'" >> "$CRI_CONTAINERD_CONFIG"
-	echo "runtime_engine = '/usr/bin/cc-runtime'" >> "$CRI_CONTAINERD_CONFIG"
+	if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]]; then
+		echo "runtime_engine = '/usr/bin/cc-runtime'" >> "$CRI_CONTAINERD_CONFIG"
+	else
+		echo "runtime_engine = '/usr/local/sbin/runc'" >> "$CRI_CONTAINERD_CONFIG"
+	fi
 	echo "[plugins.cri.containerd.default_runtime]" >> "$CRI_CONTAINERD_CONFIG"
 	echo "runtime_type = 'io.containerd.runtime.v1.linux'" >> "$CRI_CONTAINERD_CONFIG"
 	echo "runtime_engine = '/usr/local/sbin/runc'" >> "$CRI_CONTAINERD_CONFIG"
+
+	setKubeletOpts " --container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock"
 }
 
 function ensureContainerd() {
-	if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]]; then
-		# Make sure we can nest virtualization
-		if grep -q vmx /proc/cpuinfo; then
-			# Enable and start cri-containerd service
-			# Make sure this is done after networking plugins are installed
-			echo "Enabling and starting cri-containerd service..."
-			systemctlEnableAndStart containerd
-		fi
+	if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]] || [[ "$CONTAINER_RUNTIME" == "containerd" ]]; then
+		# Enable and start cri-containerd service
+		# Make sure this is done after networking plugins are installed
+		echo "Enabling and starting cri-containerd service..."
+		systemctlEnableAndStart containerd
 	fi
 }
 
@@ -542,10 +534,12 @@ if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]]; then
 	if grep -q vmx /proc/cpuinfo; then
 		echo `date`,`hostname`, installClearContainersRuntimeStart>>/opt/m
 		installClearContainersRuntime
-		echo `date`,`hostname`, installContainerdStart>>/opt/m
-		installContainerd
 	fi
 fi
+if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]] || [[ "$CONTAINER_RUNTIME" == "containerd" ]]; then
+	echo `date`,`hostname`, installContainerdStart>>/opt/m
+	installContainerd
+fi
 echo `date`,`hostname`, ensureContainerdStart>>/opt/m
 ensureContainerd
 echo `date`,`hostname`, extractHyperkubeStart>>/opt/m

diff --git a/parts/k8s/kubernetesparams.t b/parts/k8s/kubernetesparams.t
@@ -646,37 +646,39 @@
     "networkPolicy": {
       "defaultValue": "{{.OrchestratorProfile.KubernetesConfig.NetworkPolicy}}",
       "metadata": {
-        "description": "The network policy enforcement to use (calico|cilium|flannel); 'none' and 'azure' here for backwards compatibility"
+        "description": "The network policy enforcement to use (calico|cilium); 'none' and 'azure' here for backwards compatibility"
       },
       "allowedValues": [
         "",
         "none",
         "azure",
         "calico",
-        "cilium",
-        "flannel"
+        "cilium"
       ],
       "type": "string"
     },
     "networkPlugin": {
       "defaultValue": "{{.OrchestratorProfile.KubernetesConfig.NetworkPlugin}}",
       "metadata": {
-        "description": "The network plugin to use for Kubernetes (kubenet|azure)"
+        "description": "The network plugin to use for Kubernetes (kubenet|azure|flannel|cilium)"
       },
       "allowedValues": [
         "kubenet",
-        "azure"
+        "azure",
+        "flannel",
+        "cilium"
       ],
       "type": "string"
     },
     "containerRuntime": {
       "defaultValue": "{{.OrchestratorProfile.KubernetesConfig.ContainerRuntime}}",
       "metadata": {
-        "description": "The container runtime to use (docker|clear-containers)"
+        "description": "The container runtime to use (docker|clear-containers|containerd)"
       },
       "allowedValues": [
         "docker",
-        "clear-containers"
+        "clear-containers",
+        "containerd"
       ],
       "type": "string"
     },

diff --git a/pkg/acsengine/addons.go b/pkg/acsengine/addons.go
@@ -80,7 +80,7 @@ func kubernetesAddonSettingsInit(profile *api.Properties) []kubernetesFeatureSet
 		{
 			"kubernetesmasteraddons-flannel-daemonset.yaml",
 			"flannel-daemonset.yaml",
-			profile.OrchestratorProfile.KubernetesConfig.NetworkPolicy == NetworkPolicyFlannel,
+			profile.OrchestratorProfile.KubernetesConfig.NetworkPlugin == NetworkPluginFlannel,
 		},
 		{
 			"kubernetesmasteraddons-aad-default-admin-group-rbac.yaml",

diff --git a/pkg/acsengine/const.go b/pkg/acsengine/const.go
@@ -48,14 +48,14 @@ const (
 	NetworkPolicyNone = "none"
 	// NetworkPolicyCalico is the string expression for calico network policy config option
 	NetworkPolicyCalico = "calico"
-	// NetworkPolicyFlannel is the string expression for flannel network policy config option
-	NetworkPolicyFlannel = "flannel"
 	// NetworkPolicyCilium is the string expression for cilium network policy config option
 	NetworkPolicyCilium = "cilium"
 	// NetworkPluginAzure is the string expression for Azure CNI network policy
 	NetworkPluginAzure = "azure"
 	// NetworkPluginKubenet is the string expression for kubenet network plugin
 	NetworkPluginKubenet = "kubenet"
+	// NetworkPluginFlannel is the string expression for flannel network policy config option
+	NetworkPluginFlannel = "flannel"
 	// DefaultNetworkPlugin defines the network plugin to use by default
 	DefaultNetworkPlugin = NetworkPluginKubenet
 	// DefaultNetworkPolicy defines the network policy implementation to use by default

diff --git a/pkg/acsengine/defaults.go b/pkg/acsengine/defaults.go
@@ -360,9 +360,7 @@ func setOrchestratorDefaults(cs *api.ContainerService) {
 		case NetworkPolicyCalico:
 			o.KubernetesConfig.NetworkPlugin = NetworkPluginKubenet
 		case NetworkPolicyCilium:
-			o.KubernetesConfig.NetworkPlugin = NetworkPluginKubenet
-		case NetworkPolicyFlannel:
-			o.KubernetesConfig.NetworkPlugin = NetworkPluginKubenet
+			o.KubernetesConfig.NetworkPlugin = NetworkPolicyCilium
 		}
 
 		// Add default addons specification, if no user-provided spec exists

diff --git a/pkg/acsengine/defaults_test.go b/pkg/acsengine/defaults_test.go
@@ -414,19 +414,9 @@ func TestNetworkPolicyDefaults(t *testing.T) {
 	properties.OrchestratorProfile.OrchestratorType = "Kubernetes"
 	properties.OrchestratorProfile.KubernetesConfig.NetworkPolicy = "cilium"
 	setOrchestratorDefaults(&mockCS)
-	if properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin != "kubenet" {
+	if properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin != "cilium" {
 		t.Fatalf("NetworkPlugin did not have the expected value, got %s, expected %s",
-			properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin, "kubenet")
-	}
-
-	mockCS = getMockBaseContainerService("1.8.10")
-	properties = mockCS.Properties
-	properties.OrchestratorProfile.OrchestratorType = "Kubernetes"
-	properties.OrchestratorProfile.KubernetesConfig.NetworkPolicy = "flannel"
-	setOrchestratorDefaults(&mockCS)
-	if properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin != "kubenet" {
-		t.Fatalf("NetworkPlugin did not have the expected value, got %s, expected %s",
-			properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin, "kubenet")
+			properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin, "cilium")
 	}
 
 	mockCS = getMockBaseContainerService("1.8.10")

diff --git a/pkg/api/vlabs/const.go b/pkg/api/vlabs/const.go
@@ -72,14 +72,14 @@ const (
 
 var (
 	// NetworkPluginValues holds the valid values for network plugin implementation
-	NetworkPluginValues = [...]string{"", "kubenet", "azure"}
+	NetworkPluginValues = [...]string{"", "kubenet", "azure", "cilium", "flannel"}
 
 	// NetworkPolicyValues holds the valid values for a network policy
 	// "azure" and "none" are there for backwards-compatibility
-	NetworkPolicyValues = [...]string{"", "calico", "cilium", "azure", "none", "flannel"}
+	NetworkPolicyValues = [...]string{"", "calico", "cilium", "azure", "none"}
 
 	// ContainerRuntimeValues holds the valid values for container runtimes
-	ContainerRuntimeValues = [...]string{"", "docker", "clear-containers"}
+	ContainerRuntimeValues = [...]string{"", "docker", "clear-containers", "containerd"}
 )
 
 // Kubernetes configuration