Skip to content
This repository has been archived by the owner on Oct 24, 2023. It is now read-only.

Commit

Permalink
feat: modify container runtime data dir (#3072)
Browse files Browse the repository at this point in the history
  • Loading branch information
alexeldeib committed Apr 21, 2020
1 parent 22c5fb6 commit b6ea0dc
Show file tree
Hide file tree
Showing 51 changed files with 4,564 additions and 206 deletions.
10 changes: 10 additions & 0 deletions cmd/generate_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -780,6 +780,16 @@ func TestExampleAPIModels(t *testing.T) {
apiModelPath: "../examples/kubernetes-non-vhd-distros.json",
setArgs: defaultSet,
},
{
name: "docker tmp dir",
apiModelPath: "../examples/kubernetes-config/kubernetes-docker-tmpdir.json",
setArgs: defaultSet,
},
{
name: "containerd tmp dir",
apiModelPath: "../examples/kubernetes-config/kubernetes-containerd-tmpdir.json",
setArgs: defaultSet,
},
{
name: "e2e gpu",
apiModelPath: "../examples/e2e-tests/kubernetes/gpu-enabled/definition.json",
Expand Down
2 changes: 1 addition & 1 deletion docs/topics/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ Once the input is validated, the template generator is invoked which will conver
{{range $index, $agent := .AgentPoolProfiles}}
"{{.Name}}Index": {{$index}},
{{template "k8s/kubernetesagentvars.t" .}}
{{if IsNSeriesSKU .}}
{{if IsNSeriesSKU .VMSize}}
{{if IsNVIDIADevicePluginEnabled}}
"registerWithGpuTaints": "nvidia.com/gpu=true:NoSchedule",
{{end}}
Expand Down
1 change: 1 addition & 0 deletions docs/topics/clusterdefinitions.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ $ aks-engine get-versions
| cloudControllerManagerConfig | no | Configure various runtime configuration for cloud-controller-manager. See `cloudControllerManagerConfig` [below](#feat-cloud-controller-manager-config) |
| clusterSubnet | no | The IP subnet used for allocating IP addresses for pod network interfaces. The subnet must be in the VNET address space. With Azure CNI enabled, the default value is 10.240.0.0/12. Without Azure CNI, the default value is 10.244.0.0/16. |
| containerRuntime | no | The container runtime to use as a backend. The default is `docker`. Also supported is `containerd`. Windows support for `containerd` is **Experimental** - see [Windows ContainerD](features.md#windows-containerd) |
| containerRuntimeConfig | no | A map of key-value pairs to drive configuration of the container runtime. Currently accepts a single key, "dataDir", which configures the root data directory for the container runtime. dataDir must be an absolute path. This is only implemented on Linux. See an [example](../../examples/kubernetes-config/kubernetes-docker-tmpdir.json) which places docker on the tmp disk of a Linux VM. |
| controllerManagerConfig | no | Configure various runtime configuration for controller-manager. See `controllerManagerConfig` [below](#feat-controller-manager-config) |
| customWindowsPackageURL | no | Configure custom windows Kubernetes release package URL for deployment on Windows. The format of this file is a zip file with multiple items (binaries, cni, infra container) in it. This setting will be deprecated in a future release of aks-engine where the binaries will be pulled in the format of Kubernetes releases that only contain the kubernetes binaries. |
| WindowsNodeBinariesURL | no | Windows Kubernetes Node binaries can be provided in the format of Kubernetes release (example: https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.11.md#node-binaries-1). This setting allows overriding the binaries for custom builds. |
Expand Down
47 changes: 47 additions & 0 deletions examples/kubernetes-config/kubernetes-containerd-tmpdir.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
{
"apiVersion": "vlabs",
"properties": {
"orchestratorProfile": {
"orchestratorType": "Kubernetes",
"orchestratorRelease": "1.17",
"kubernetesConfig": {
"networkPlugin": "kubenet",
"containerRuntime": "containerd",
"containerRuntimeConfig": {
"dataDir": "/mnt/containerd"
}
}
},
"masterProfile": {
"count": 1,
"dnsPrefix": "",
"vmSize": "Standard_D8s_v3",
"osDiskSizeGb": 1024,
"distro": "ubuntu-18.04"
},
"agentPoolProfiles": [
{
"name": "agentpool1",
"count": 1,
"vmSize": "Standard_D8s_v3",
"availabilityProfile": "VirtualMachineScaleSets",
"osDiskSizeGb": 1024,
"distro": "ubuntu-18.04"
}
],
"linuxProfile": {
"adminUsername": "azureuser",
"ssh": {
"publicKeys": [
{
"keyData": ""
}
]
}
},
"servicePrincipalProfile": {
"clientId": "",
"secret": ""
}
}
}
46 changes: 46 additions & 0 deletions examples/kubernetes-config/kubernetes-docker-tmpdir.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
{
"apiVersion": "vlabs",
"properties": {
"orchestratorProfile": {
"orchestratorType": "Kubernetes",
"orchestratorRelease": "1.17",
"kubernetesConfig": {
"containerRuntime": "docker",
"containerRuntimeConfig": {
"dataDir": "/mnt/docker"
}
}
},
"masterProfile": {
"count": 1,
"dnsPrefix": "",
"vmSize": "Standard_D8s_v3",
"osDiskSizeGb": 1024,
"distro": "ubuntu-18.04"
},
"agentPoolProfiles": [
{
"name": "agentpool1",
"count": 1,
"vmSize": "Standard_NC12s_v3",
"availabilityProfile": "VirtualMachineScaleSets",
"osDiskSizeGb": 1024,
"distro": "ubuntu-18.04"
}
],
"linuxProfile": {
"adminUsername": "azureuser",
"ssh": {
"publicKeys": [
{
"keyData": ""
}
]
}
},
"servicePrincipalProfile": {
"clientId": "",
"secret": ""
}
}
}
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ require (
github.com/Azure/go-autorest/autorest/date v0.2.0
github.com/Azure/go-autorest/autorest/to v0.3.0
github.com/Azure/go-autorest/autorest/validation v0.2.0 // indirect
github.com/BurntSushi/toml v0.3.1
github.com/Jeffail/gabs v1.1.1
github.com/blang/semver v3.5.1+incompatible
github.com/davecgh/go-spew v1.1.1
Expand Down Expand Up @@ -57,7 +58,7 @@ require (
gopkg.in/go-playground/validator.v9 v9.25.0
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.41.0
gopkg.in/yaml.v2 v2.2.2 // indirect
gopkg.in/yaml.v2 v2.2.8 // indirect
k8s.io/api v0.0.0-20190222213804-5cb15d344471
k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628
k8s.io/client-go v10.0.0+incompatible
Expand Down
6 changes: 4 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1Gn
github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k=
github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/Jeffail/gabs v1.1.1 h1:V0uzR08Hj22EX8+8QMhyI9sX2hwRu+/RJhJUmnwda/E=
github.com/Jeffail/gabs v1.1.1/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc=
github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
Expand Down Expand Up @@ -186,8 +188,8 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
k8s.io/api v0.0.0-20190222213804-5cb15d344471 h1:MzQGt8qWQCR+39kbYRd0uQqsvSidpYqJLFeWiJ9l4OE=
k8s.io/api v0.0.0-20190222213804-5cb15d344471/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA=
k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628 h1:UYfHH+KEF88OTg+GojQUwFTNxbxwmoktLwutUzR0GPg=
Expand Down
31 changes: 3 additions & 28 deletions parts/k8s/cloud-init/masternodecustomdata.yml
Original file line number Diff line number Diff line change
Expand Up @@ -182,14 +182,7 @@ write_files:
permissions: "0644"
owner: root
content: |
{
"live-restore": true,
"log-driver": "json-file",
"log-opts": {
"max-size": "50m",
"max-file": "5"
}
}
{{IndentString (GetDockerConfig false) 4}}
{{end}}

{{if HasCiliumNetworkPlugin}}
Expand Down Expand Up @@ -221,27 +214,9 @@ write_files:
permissions: "0644"
owner: root
content: |
version = 2
subreaper = false
oom_score = 0
[plugins]
[plugins."io.containerd.grpc.v1.cri"]
sandbox_image = "{{GetPodInfraContainerSpec}}"
[plugins."io.containerd.grpc.v1.cri".cni]
{{if IsKubenet}}
conf_template = "/etc/containerd/kubenet_template.conf"
{{end}}
[plugins."io.containerd.grpc.v1.cri".containerd]
default_runtime_name = "runc"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.untrusted]
{{/* note: runc really should not be used for untrusted workloads... should we remove this? This is here because it was here before */}}
runtime_type = "io.containerd.runc.v2"
{{IndentString GetContainerdConfig 4}}
#EOF
{{if IsKubenet}}
- path: /etc/containerd/kubenet_template.conf
permissions: "0644"
Expand Down
38 changes: 3 additions & 35 deletions parts/k8s/cloud-init/nodecustomdata.yml
Original file line number Diff line number Diff line change
Expand Up @@ -176,21 +176,7 @@ write_files:
permissions: "0644"
owner: root
content: |
{
"live-restore": true,
"log-driver": "json-file",
"log-opts": {
"max-size": "50m",
"max-file": "5"
}{{if IsNSeriesSKU .}}
,"default-runtime": "nvidia",
"runtimes": {
"nvidia": {
"path": "/usr/bin/nvidia-container-runtime",
"runtimeArgs": []
}
}{{end}}
}
{{IndentString (GetDockerConfig (IsNSeriesSKU .VMSize)) 4}}
{{end}}

{{if HasCiliumNetworkPlugin}}
Expand Down Expand Up @@ -222,25 +208,7 @@ write_files:
permissions: "0644"
owner: root
content: |
version = 2
subreaper = false
oom_score = 0
[plugins]
[plugins."io.containerd.grpc.v1.cri"]
sandbox_image = "{{GetPodInfraContainerSpec}}"
[plugins."io.containerd.grpc.v1.cri".cni]
{{if IsKubenet}}
conf_template = "/etc/containerd/kubenet_template.conf"
{{end}}
[plugins."io.containerd.grpc.v1.cri".containerd]
default_runtime_name = "runc"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.untrusted]
{{/* note: runc really should not be used for untrusted workloads... should we remove this? This is here because it was here before */}}
runtime_type = "io.containerd.runc.v2"
{{IndentString GetContainerdConfig 4}}
#EOF

{{if IsKubenet }}
Expand Down Expand Up @@ -269,7 +237,7 @@ write_files:
{{end}}
{{end}}

{{if IsNSeriesSKU .}}
{{if IsNSeriesSKU .VMSize}}
- path: /etc/systemd/system/nvidia-modprobe.service
permissions: "0644"
owner: root
Expand Down
50 changes: 50 additions & 0 deletions pkg/api/common/const.go
Original file line number Diff line number Diff line change
Expand Up @@ -313,3 +313,53 @@ const (
KubernetesImageBaseTypeGCR = "gcr"
KubernetesImageBaseTypeMCR = "mcr"
)

var (
// DefaultDockerConfig describes the default configuration of the docker daemon.
DefaultDockerConfig = DockerConfig{
LiveRestore: true,
LogDriver: "json-file",
LogOpts: LogOpts{
MaxSize: "50m",
MaxFile: "5",
},
}

// DefaultContainerdConfig describes the default configuration of the containerd daemon.
DefaultContainerdConfig = ContainerdConfig{
Version: 2,
OomScore: 0,
Plugins: Plugins{
IoContainerdGrpcV1Cri: IoContainerdGrpcV1Cri{
CNI: ContainerdCNIPlugin{},
Containerd: ContainerdPlugin{
DefaultRuntimeName: "runc",
Runtimes: map[string]ContainerdRuntime{
"runc": {
RuntimeType: "io.containerd.runc.v2",
},
// note: runc really should not be used for untrusted workloads... should we remove this? This is here because it was here before
"untrusted": {
RuntimeType: "io.containerd.runc.v2",
},
},
},
},
},
}
)

// GetDefaultDockerConfig returns the default docker config for processing.
func GetDefaultDockerConfig() DockerConfig {
return DefaultDockerConfig
}

// GetDefaultContainerdConfig returns the default containerd config for processing.
func GetDefaultContainerdConfig() ContainerdConfig {
return DefaultContainerdConfig
}

// Known container runtime configuration keys
const (
ContainerDataDirKey = "dataDir"
)

0 comments on commit b6ea0dc

Please sign in to comment.