Skip to content
This repository has been archived by the owner on Jan 11, 2023. It is now read-only.

Commit

Permalink
Introduce kata container support (#3465)
Browse files Browse the repository at this point in the history 
Signed-off-by: Eric Ernst <eric.ernst@intel.com>
  • Loading branch information
Eric Ernst authored and Cecile Robert-Michon committed Jul 13, 2018
1 parent 111226b commit 81cf27c
Show file tree
Hide file tree
Showing 8 changed files with 122 additions and 6 deletions.
2 changes: 1 addition & 1 deletion docs/clusterdefinition.md
View file
Expand Up @@ -36,7 +36,7 @@ Here are the valid values for the orchestrator types:
| apiServerConfig | no | Configure various runtime configuration for apiserver. See `apiServerConfig` [below](#feat-apiserver-config) |
| cloudControllerManagerConfig | no | Configure various runtime configuration for cloud-controller-manager. See `cloudControllerManagerConfig` [below](#feat-cloud-controller-manager-config) |
| clusterSubnet | no | The IP subnet used for allocating IP addresses for pod network interfaces. The subnet must be in the VNET address space. Default value is 10.244.0.0/16 |
| containerRuntime | no | The container runtime to use as a backend. The default is `docker`. The other options are `clear-containers` and `containerd` |
| containerRuntime | no | The container runtime to use as a backend. The default is `docker`. The other options are `clear-containers`, `kata-containers`, and `containerd` |
| controllerManagerConfig | no | Configure various runtime configuration for controller-manager. See `controllerManagerConfig` [below](#feat-controller-manager-config) |
| customWindowsPackageURL | no | Configure custom windows Kubernetes release package URL for deployment on Windows |
| dnsServiceIP | no | IP address for kube-dns to listen on. If specified must be in the range of `serviceCidr` |
Expand Down
32 changes: 32 additions & 0 deletions docs/kubernetes/features.md
View file
Expand Up @@ -7,6 +7,7 @@
|Cilium Network Policy|Alpha|`vlabs`|[kubernetes-cilium.json](../../examples/networkpolicy/kubernetes-cilium.json)|[Description](#feat-cilium)|
|Custom VNET|Beta|`vlabs`|[kubernetesvnet-azure-cni.json](../../examples/vnet/kubernetesvnet-azure-cni.json)|[Description](#feat-custom-vnet)|
|Clear Containers Runtime|Alpha|`vlabs`|[kubernetes-clear-containers.json](../../examples/kubernetes-clear-containers.json)|[Description](#feat-clear-containers)|
|Kata Containers Runtime|Alpha|`vlabs`|[kubernetes-kata-containers.json](../../examples/kubernetes-kata-containers.json)|[Description](#feat-kata-containers)|
|Private Cluster|Alpha|`vlabs`|[kubernetes-private-cluster.json](../../examples/kubernetes-config/kubernetes-private-cluster.json)|[Description](#feat-private-cluster)|
|Azure Key Vault Encryption|Alpha|`vlabs`|[kubernetes-keyvault-encryption.json](../../examples/kubernetes-config/kubernetes-keyvault-encryption.json)|[Description](#feat-keyvault-encryption)|

Expand Down Expand Up @@ -306,6 +307,37 @@ This should look like:
],
```
<a name="feat-kata-containers"></a>
## Kata Containers
You can designate kubernetes agents to use Kata Containers as the
container runtime by setting:
```
"kubernetesConfig": {
"containerRuntime": "kata-containers"
}
```
You will need to make sure your agents are using a `vmSize` that [supports
nested virtualization](https://azure.microsoft.com/en-us/blog/nested-virtualization-in-azure/).
These are the `Dv3` or `Ev3` series nodes.
This should look like:
```
"agentPoolProfiles": [
{
"name": "agentpool1",
"count": 3,
"vmSize": "Standard_D4s_v3",
"availabilityProfile": "AvailabilitySet",
"diskSizesGB": [1023]
}
],
```
<a name="feat-private-cluster"></a>
## Private Cluster
Expand Down
40 changes: 40 additions & 0 deletions examples/kubernetes-kata-containers.json
View file
@@ -0,0 +1,40 @@
{
"apiVersion": "vlabs",
"properties": {
"orchestratorProfile": {
"orchestratorType": "Kubernetes",
"orchestratorRelease": "1.10",
"kubernetesConfig": {
"networkPlugin": "flannel",
"containerRuntime": "kata-containers"
}
},
"masterProfile": {
"count": 1,
"dnsPrefix": "",
"vmSize": "Standard_D2_v2"
},
"agentPoolProfiles": [
{
"name": "agentpool1",
"count": 3,
"vmSize": "Standard_D4s_v3",
"availabilityProfile": "AvailabilitySet"
}
],
"linuxProfile": {
"adminUsername": "azureuser",
"ssh": {
"publicKeys": [
{
"keyData": ""
}
]
}
},
"servicePrincipalProfile": {
"clientId": "",
"secret": ""
}
}
}
35 changes: 33 additions & 2 deletions parts/k8s/kubernetescustomscript.sh
View file
Expand Up @@ -27,6 +27,9 @@ ERR_CNI_DOWNLOAD_TIMEOUT=41 # Timeout waiting for CNI download(s)
ERR_MS_PROD_DEB_DOWNLOAD_TIMEOUT=42 # Timeout waiting for https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb
ERR_MS_PROD_DEB_PKG_ADD_FAIL=43 # Failed to add repo pkg file
ERR_OUTBOUND_CONN_FAIL=50 # Unable to establish outbound connection
ERR_KATA_KEY_DOWNLOAD_TIMEOUT=60 # Timeout waiting to download kata repo key
ERR_KATA_APT_KEY_TIMEOUT=61 # Timeout waiting for kata apt-key
ERR_KATA_INSTALL_TIMEOUT=62 # Timeout waiting for kata install
ERR_CUSTOM_SEARCH_DOMAINS_FAIL=80 # Unable to configure custom search domains
ERR_APT_DAILY_TIMEOUT=98 # Timeout waiting for apt daily updates
ERR_APT_UPDATE_TIMEOUT=99 # Timeout waiting for apt-get update to complete
Expand Down Expand Up @@ -281,6 +284,24 @@ function configNetworkPlugin() {
fi
}

function installKataContainersRuntime() {
# Add Kata Containers repository key
echo "Adding Kata Containers repository key..."
KATA_RELEASE_KEY_TMP=/tmp/kata-containers-release.key
KATA_URL=http://download.opensuse.org/repositories/home:/katacontainers:/release/xUbuntu_16.04/Release.key
retrycmd_if_failure_no_stats 20 1 5 curl -fsSL $KATA_URL > $KATA_RELEASE_KEY_TMP || exit $ERR_KATA_KEY_DOWNLOAD_TIMEOUT
retrycmd_if_failure 10 5 10 apt-key add $KATA_RELEASE_KEY_TMP || exit $ERR_KATA_APT_KEY_TIMEOUT

# Add Kata Container repository
echo "Adding Kata Containers repository..."
echo 'deb http://download.opensuse.org/repositories/home:/katacontainers:/release/xUbuntu_16.04/ /' > /etc/apt/sources.list.d/kata-containers.list

# Install Kata Containers runtime
echo "Installing Kata Containers runtime..."
apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT
apt_get_install 20 30 120 kata-runtime || exit $ERR_KATA_INSTALL_TIMEOUT
}

function installClearContainersRuntime() {
# Add Clear Containers repository key
echo "Adding Clear Containers repository key..."
Expand Down Expand Up @@ -323,6 +344,8 @@ function setupContainerd() {
echo "runtime_type = 'io.containerd.runtime.v1.linux'" >> "$CRI_CONTAINERD_CONFIG"
if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]]; then
echo "runtime_engine = '/usr/bin/cc-runtime'" >> "$CRI_CONTAINERD_CONFIG"
elif [[ "$CONTAINER_RUNTIME" == "kata-containers" ]]; then
echo "runtime_engine = '/usr/bin/kata-runtime'" >> "$CRI_CONTAINERD_CONFIG"
else
echo "runtime_engine = '/usr/local/sbin/runc'" >> "$CRI_CONTAINERD_CONFIG"
fi
Expand All @@ -344,13 +367,13 @@ function installContainerd() {
sed -i '/\[Service\]/a ExecStartPost=\/sbin\/iptables -P FORWARD ACCEPT' /etc/systemd/system/containerd.service

echo "Successfully installed cri-containerd..."
if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]] || [[ "$CONTAINER_RUNTIME" == "containerd" ]]; then
if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]] || [[ "$CONTAINER_RUNTIME" == "kata-containers" ]] || [[ "$CONTAINER_RUNTIME" == "containerd" ]]; then
setupContainerd
fi
}

function ensureContainerd() {
if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]] || [[ "$CONTAINER_RUNTIME" == "containerd" ]]; then
if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]] || [[ "$CONTAINER_RUNTIME" == "kata-containers" ]] || [[ "$CONTAINER_RUNTIME" == "containerd" ]]; then
# Enable and start cri-containerd service
# Make sure this is done after networking plugins are installed
echo "Enabling and starting cri-containerd service..."
Expand Down Expand Up @@ -546,6 +569,14 @@ if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]]; then
installClearContainersRuntime
fi
fi

if [[ "$CONTAINER_RUNTIME" == "kata-containers" ]]; then
# Ensure we can nest virtualization
if grep -q vmx /proc/cpuinfo; then
installKataContainersRuntime
fi
fi

echo `date`,`hostname`, ensureContainerdStart>>/opt/m
ensureContainerd

Expand Down
3 changes: 2 additions & 1 deletion parts/k8s/kubernetesparams.t
View file
Expand Up @@ -659,11 +659,12 @@
"containerRuntime": {
"defaultValue": "{{.OrchestratorProfile.KubernetesConfig.ContainerRuntime}}",
"metadata": {
"description": "The container runtime to use (docker|clear-containers|containerd)"
"description": "The container runtime to use (docker|clear-containers|kata-containers|containerd)"
},
"allowedValues": [
"docker",
"clear-containers",
"kata-containers",
"containerd"
],
"type": "string"
Expand Down
2 changes: 1 addition & 1 deletion pkg/api/vlabs/const.go
View file
Expand Up @@ -79,7 +79,7 @@ var (
NetworkPolicyValues = [...]string{"", "calico", "cilium", "azure", "none"}

// ContainerRuntimeValues holds the valid values for container runtimes
ContainerRuntimeValues = [...]string{"", "docker", "clear-containers", "containerd"}
ContainerRuntimeValues = [...]string{"", "docker", "clear-containers", "kata-containers", "containerd"}
)

// Kubernetes configuration
Expand Down
2 changes: 1 addition & 1 deletion pkg/api/vlabs/validate.go
View file
Expand Up @@ -1092,7 +1092,7 @@ func (a *Properties) validateContainerRuntime() error {
}

// Make sure we don't use clear containers on windows.
if (containerRuntime == "clear-containers" || containerRuntime == "containerd") && a.HasWindows() {
if (containerRuntime == "clear-containers" || containerRuntime == "kata-containers" || containerRuntime == "containerd") && a.HasWindows() {
return fmt.Errorf("containerRuntime %q is not supporting windows agents", containerRuntime)
}

Expand Down
12 changes: 12 additions & 0 deletions pkg/api/vlabs/validate_test.go
View file
Expand Up @@ -1083,6 +1083,18 @@ func Test_Properties_ValidateContainerRuntime(t *testing.T) {
)
}

p.OrchestratorProfile.KubernetesConfig.ContainerRuntime = "kata-containers"
p.AgentPoolProfiles = []*AgentPoolProfile{
{
OSType: Windows,
},
}
if err := p.validateContainerRuntime(); err == nil {
t.Errorf(
"should error on kata-containers for windows clusters",
)
}

p.OrchestratorProfile.KubernetesConfig.ContainerRuntime = "containerd"
p.AgentPoolProfiles = []*AgentPoolProfile{
{
Expand Down

0 comments on commit 81cf27c

Please sign in to comment.