Skip to content
This repository has been archived by the owner on Oct 24, 2023. It is now read-only.

feat: modify container runtime data dir #3072

Merged
merged 1 commit into from
Apr 21, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
10 changes: 10 additions & 0 deletions cmd/generate_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -780,6 +780,16 @@ func TestExampleAPIModels(t *testing.T) {
apiModelPath: "../examples/kubernetes-non-vhd-distros.json",
setArgs: defaultSet,
},
{
name: "docker tmp dir",
apiModelPath: "../examples/kubernetes-config/kubernetes-docker-tmpdir.json",
setArgs: defaultSet,
},
{
name: "containerd tmp dir",
apiModelPath: "../examples/kubernetes-config/kubernetes-containerd-tmpdir.json",
setArgs: defaultSet,
},
{
name: "e2e gpu",
apiModelPath: "../examples/e2e-tests/kubernetes/gpu-enabled/definition.json",
Expand Down
2 changes: 1 addition & 1 deletion docs/topics/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ Once the input is validated, the template generator is invoked which will conver
{{range $index, $agent := .AgentPoolProfiles}}
"{{.Name}}Index": {{$index}},
{{template "k8s/kubernetesagentvars.t" .}}
{{if IsNSeriesSKU .}}
{{if IsNSeriesSKU .VMSize}}
{{if IsNVIDIADevicePluginEnabled}}
"registerWithGpuTaints": "nvidia.com/gpu=true:NoSchedule",
{{end}}
Expand Down
1 change: 1 addition & 0 deletions docs/topics/clusterdefinitions.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ $ aks-engine get-versions
| cloudControllerManagerConfig | no | Configure various runtime configuration for cloud-controller-manager. See `cloudControllerManagerConfig` [below](#feat-cloud-controller-manager-config) |
| clusterSubnet | no | The IP subnet used for allocating IP addresses for pod network interfaces. The subnet must be in the VNET address space. With Azure CNI enabled, the default value is 10.240.0.0/12. Without Azure CNI, the default value is 10.244.0.0/16. |
| containerRuntime | no | The container runtime to use as a backend. The default is `docker`. Also supported is `containerd`. Windows support for `containerd` is **Experimental** - see [Windows ContainerD](features.md#windows-containerd) |
| containerRuntimeConfig | no | A map of key-value pairs to drive configuration of the container runtime. Currently accepts a single key, "dataDir", which configures the root data directory for the container runtime. dataDir must be an absolute path. This is only implemented on Linux. See an [example](../../examples/kubernetes-config/kubernetes-docker-tmpdir.json) which places docker on the tmp disk of a Linux VM. |
| controllerManagerConfig | no | Configure various runtime configuration for controller-manager. See `controllerManagerConfig` [below](#feat-controller-manager-config) |
| customWindowsPackageURL | no | Configure custom windows Kubernetes release package URL for deployment on Windows. The format of this file is a zip file with multiple items (binaries, cni, infra container) in it. This setting will be deprecated in a future release of aks-engine where the binaries will be pulled in the format of Kubernetes releases that only contain the kubernetes binaries. |
| WindowsNodeBinariesURL | no | Windows Kubernetes Node binaries can be provided in the format of Kubernetes release (example: https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.11.md#node-binaries-1). This setting allows overriding the binaries for custom builds. |
Expand Down
47 changes: 47 additions & 0 deletions examples/kubernetes-config/kubernetes-containerd-tmpdir.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
{
"apiVersion": "vlabs",
"properties": {
"orchestratorProfile": {
"orchestratorType": "Kubernetes",
"orchestratorRelease": "1.17",
"kubernetesConfig": {
"networkPlugin": "kubenet",
"containerRuntime": "containerd",
"containerRuntimeConfig": {
"dataDir": "/mnt/containerd"
}
}
},
"masterProfile": {
"count": 1,
"dnsPrefix": "",
"vmSize": "Standard_D8s_v3",
"osDiskSizeGb": 1024,
"distro": "ubuntu-18.04"
},
"agentPoolProfiles": [
{
"name": "agentpool1",
"count": 1,
"vmSize": "Standard_D8s_v3",
"availabilityProfile": "VirtualMachineScaleSets",
"osDiskSizeGb": 1024,
"distro": "ubuntu-18.04"
}
],
"linuxProfile": {
"adminUsername": "azureuser",
"ssh": {
"publicKeys": [
{
"keyData": ""
}
]
}
},
"servicePrincipalProfile": {
"clientId": "",
"secret": ""
}
}
}
46 changes: 46 additions & 0 deletions examples/kubernetes-config/kubernetes-docker-tmpdir.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
{
"apiVersion": "vlabs",
"properties": {
"orchestratorProfile": {
"orchestratorType": "Kubernetes",
"orchestratorRelease": "1.17",
"kubernetesConfig": {
"containerRuntime": "docker",
"containerRuntimeConfig": {
"dataDir": "/mnt/docker"
}
}
},
"masterProfile": {
"count": 1,
"dnsPrefix": "",
"vmSize": "Standard_D8s_v3",
"osDiskSizeGb": 1024,
"distro": "ubuntu-18.04"
},
"agentPoolProfiles": [
{
"name": "agentpool1",
"count": 1,
"vmSize": "Standard_NC12s_v3",
"availabilityProfile": "VirtualMachineScaleSets",
"osDiskSizeGb": 1024,
"distro": "ubuntu-18.04"
}
],
"linuxProfile": {
"adminUsername": "azureuser",
"ssh": {
"publicKeys": [
{
"keyData": ""
}
]
}
},
"servicePrincipalProfile": {
"clientId": "",
"secret": ""
}
}
}
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ require (
github.com/Azure/go-autorest/autorest/date v0.2.0
github.com/Azure/go-autorest/autorest/to v0.3.0
github.com/Azure/go-autorest/autorest/validation v0.2.0 // indirect
github.com/BurntSushi/toml v0.3.1
github.com/Jeffail/gabs v1.1.1
github.com/blang/semver v3.5.1+incompatible
github.com/davecgh/go-spew v1.1.1
Expand Down Expand Up @@ -57,7 +58,7 @@ require (
gopkg.in/go-playground/validator.v9 v9.25.0
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.41.0
gopkg.in/yaml.v2 v2.2.2 // indirect
gopkg.in/yaml.v2 v2.2.8 // indirect
k8s.io/api v0.0.0-20190222213804-5cb15d344471
k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628
k8s.io/client-go v10.0.0+incompatible
Expand Down
6 changes: 4 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1Gn
github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k=
github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/Jeffail/gabs v1.1.1 h1:V0uzR08Hj22EX8+8QMhyI9sX2hwRu+/RJhJUmnwda/E=
github.com/Jeffail/gabs v1.1.1/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc=
github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
Expand Down Expand Up @@ -186,8 +188,8 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
k8s.io/api v0.0.0-20190222213804-5cb15d344471 h1:MzQGt8qWQCR+39kbYRd0uQqsvSidpYqJLFeWiJ9l4OE=
k8s.io/api v0.0.0-20190222213804-5cb15d344471/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA=
k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628 h1:UYfHH+KEF88OTg+GojQUwFTNxbxwmoktLwutUzR0GPg=
Expand Down
31 changes: 3 additions & 28 deletions parts/k8s/cloud-init/masternodecustomdata.yml
Original file line number Diff line number Diff line change
Expand Up @@ -182,14 +182,7 @@ write_files:
permissions: "0644"
owner: root
content: |
{
"live-restore": true,
"log-driver": "json-file",
"log-opts": {
"max-size": "50m",
"max-file": "5"
}
}
{{IndentString (GetDockerConfig false) 4}}
{{end}}

{{if HasCiliumNetworkPlugin}}
Expand Down Expand Up @@ -221,27 +214,9 @@ write_files:
permissions: "0644"
owner: root
content: |
version = 2
subreaper = false
oom_score = 0

[plugins]
[plugins."io.containerd.grpc.v1.cri"]
sandbox_image = "{{GetPodInfraContainerSpec}}"
[plugins."io.containerd.grpc.v1.cri".cni]
{{if IsKubenet}}
conf_template = "/etc/containerd/kubenet_template.conf"
{{end}}
[plugins."io.containerd.grpc.v1.cri".containerd]
default_runtime_name = "runc"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.untrusted]
{{/* note: runc really should not be used for untrusted workloads... should we remove this? This is here because it was here before */}}
runtime_type = "io.containerd.runc.v2"
{{IndentString GetContainerdConfig 4}}
#EOF

{{if IsKubenet}}
- path: /etc/containerd/kubenet_template.conf
permissions: "0644"
Expand Down
38 changes: 3 additions & 35 deletions parts/k8s/cloud-init/nodecustomdata.yml
Original file line number Diff line number Diff line change
Expand Up @@ -176,21 +176,7 @@ write_files:
permissions: "0644"
owner: root
content: |
{
"live-restore": true,
"log-driver": "json-file",
"log-opts": {
"max-size": "50m",
"max-file": "5"
}{{if IsNSeriesSKU .}}
,"default-runtime": "nvidia",
"runtimes": {
"nvidia": {
"path": "/usr/bin/nvidia-container-runtime",
"runtimeArgs": []
}
}{{end}}
}
{{IndentString (GetDockerConfig (IsNSeriesSKU .VMSize)) 4}}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You're teaching me new things about go template syntax

{{end}}

{{if HasCiliumNetworkPlugin}}
Expand Down Expand Up @@ -222,25 +208,7 @@ write_files:
permissions: "0644"
owner: root
content: |
version = 2
subreaper = false
oom_score = 0

[plugins]
[plugins."io.containerd.grpc.v1.cri"]
sandbox_image = "{{GetPodInfraContainerSpec}}"
[plugins."io.containerd.grpc.v1.cri".cni]
{{if IsKubenet}}
conf_template = "/etc/containerd/kubenet_template.conf"
{{end}}
[plugins."io.containerd.grpc.v1.cri".containerd]
default_runtime_name = "runc"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.untrusted]
{{/* note: runc really should not be used for untrusted workloads... should we remove this? This is here because it was here before */}}
runtime_type = "io.containerd.runc.v2"
{{IndentString GetContainerdConfig 4}}
#EOF
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We need to make sure we keep these #EOF sentinel chars at the end of the data stream, which the CSE scripts use to determine that cloud-init has paved the entire file.


{{if IsKubenet }}
Expand Down Expand Up @@ -269,7 +237,7 @@ write_files:
{{end}}
{{end}}

{{if IsNSeriesSKU .}}
{{if IsNSeriesSKU .VMSize}}
- path: /etc/systemd/system/nvidia-modprobe.service
permissions: "0644"
owner: root
Expand Down
50 changes: 50 additions & 0 deletions pkg/api/common/const.go
Original file line number Diff line number Diff line change
Expand Up @@ -313,3 +313,53 @@ const (
KubernetesImageBaseTypeGCR = "gcr"
KubernetesImageBaseTypeMCR = "mcr"
)

var (
// DefaultDockerConfig describes the default configuration of the docker daemon.
DefaultDockerConfig = DockerConfig{
LiveRestore: true,
LogDriver: "json-file",
LogOpts: LogOpts{
MaxSize: "50m",
MaxFile: "5",
},
}

// DefaultContainerdConfig describes the default configuration of the containerd daemon.
DefaultContainerdConfig = ContainerdConfig{
Version: 2,
OomScore: 0,
Plugins: Plugins{
IoContainerdGrpcV1Cri: IoContainerdGrpcV1Cri{
CNI: ContainerdCNIPlugin{},
Containerd: ContainerdPlugin{
DefaultRuntimeName: "runc",
Runtimes: map[string]ContainerdRuntime{
"runc": {
RuntimeType: "io.containerd.runc.v2",
},
// note: runc really should not be used for untrusted workloads... should we remove this? This is here because it was here before
"untrusted": {
RuntimeType: "io.containerd.runc.v2",
},
},
},
},
},
}
)

// GetDefaultDockerConfig returns the default docker config for processing.
func GetDefaultDockerConfig() DockerConfig {
return DefaultDockerConfig
}

// GetDefaultContainerdConfig returns the default containerd config for processing.
func GetDefaultContainerdConfig() ContainerdConfig {
return DefaultContainerdConfig
}

// Known container runtime configuration keys
const (
ContainerDataDirKey = "dataDir"
)