refactor: more flexible CRI config

cmd/generate_test.go

@@ -780,6 +780,16 @@ func TestExampleAPIModels(t *testing.T) {
 			apiModelPath: "../examples/kubernetes-non-vhd-distros.json",
 			setArgs:      defaultSet,
 		},
+		{
+			name:         "docker tmp dir",
+			apiModelPath: "../examples/kubernetes-config/kubernetes-docker-tmpdir.json",
+			setArgs:      defaultSet,
+		},
+		{
+			name:         "containerd tmp dir",
+			apiModelPath: "../examples/kubernetes-config/kubernetes-containerd-tmpdir.json",
+			setArgs:      defaultSet,
+		},
 		{
 			name:         "e2e gpu",
 			apiModelPath: "../examples/e2e-tests/kubernetes/gpu-enabled/definition.json",

examples/kubernetes-config/kubernetes-containerd-tmpdir.json

@@ -0,0 +1,47 @@
+{
+  "apiVersion": "vlabs",
+  "properties": {
+    "orchestratorProfile": {
+      "orchestratorType": "Kubernetes",
+      "orchestratorRelease": "1.17",
+      "kubernetesConfig": {
+        "networkPlugin": "kubenet",
+        "containerRuntime": "containerd",
+        "containerRuntimeConfig": {
+          "dataDir": "/mnt/containerd"
+        }
+      }
+    },
+    "masterProfile": {
+      "count": 1,
+      "dnsPrefix": "",
+      "vmSize": "Standard_D8s_v3",
+      "osDiskSizeGb": 1024,
+      "distro": "ubuntu-18.04"
+    },
+    "agentPoolProfiles": [
+      {
+        "name": "agentpool1",
+        "count": 1,
+        "vmSize": "Standard_D8s_v3",
+        "availabilityProfile": "VirtualMachineScaleSets",
+        "osDiskSizeGb": 1024,
+        "distro": "ubuntu-18.04"
+      }
+    ],
+    "linuxProfile": {
+      "adminUsername": "azureuser",
+      "ssh": {
+        "publicKeys": [
+          {
+            "keyData": ""
+          }
+        ]
+      }
+    },
+    "servicePrincipalProfile": {
+      "clientId": "",
+      "secret": ""
+    }
+  }
+}

examples/kubernetes-config/kubernetes-docker-tmpdir.json

@@ -22,7 +22,7 @@
       {
         "name": "agentpool1",
         "count": 1,
-        "vmSize": "Standard_D8s_v3",
+        "vmSize": "Standard_NC12s_v3",
         "availabilityProfile": "VirtualMachineScaleSets",
         "osDiskSizeGb": 1024,
         "distro": "ubuntu-18.04"

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/Azure/go-autorest/autorest/date v0.2.0
 	github.com/Azure/go-autorest/autorest/to v0.3.0
 	github.com/Azure/go-autorest/autorest/validation v0.2.0 // indirect
+	github.com/BurntSushi/toml v0.3.1
 	github.com/Jeffail/gabs v1.1.1
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/davecgh/go-spew v1.1.1
@@ -57,7 +58,7 @@ require (
 	gopkg.in/go-playground/validator.v9 v9.25.0
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.41.0
-	gopkg.in/yaml.v2 v2.2.2 // indirect
+	gopkg.in/yaml.v2 v2.2.8 // indirect
 	k8s.io/api v0.0.0-20190222213804-5cb15d344471
 	k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628
 	k8s.io/client-go v10.0.0+incompatible

go.sum

@@ -25,6 +25,8 @@ github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1Gn
 github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
 github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k=
 github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/Jeffail/gabs v1.1.1 h1:V0uzR08Hj22EX8+8QMhyI9sX2hwRu+/RJhJUmnwda/E=
 github.com/Jeffail/gabs v1.1.1/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
@@ -182,8 +184,8 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 k8s.io/api v0.0.0-20190222213804-5cb15d344471 h1:MzQGt8qWQCR+39kbYRd0uQqsvSidpYqJLFeWiJ9l4OE=
 k8s.io/api v0.0.0-20190222213804-5cb15d344471/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA=
 k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628 h1:UYfHH+KEF88OTg+GojQUwFTNxbxwmoktLwutUzR0GPg=

parts/k8s/cloud-init/artifacts/cse_config.sh

@@ -282,9 +282,6 @@ installContainerd() {
 ensureContainerd() {
   wait_for_file 1200 1 /etc/systemd/system/containerd.service.d/exec_start.conf || exit {{GetCSEErrorCode "ERR_FILE_WATCH_TIMEOUT"}}
   wait_for_file 1200 1 /etc/containerd/config.toml || exit {{GetCSEErrorCode "ERR_FILE_WATCH_TIMEOUT"}}
-  {{- if HasContainerDataDir}}
-  echo -e "root = \"{{GetContainerDataDir}}\"\n$(cat /etc/containerd/config.toml)" > /etc/containerd/config.toml
-  {{- end}}
   systemctlEnableAndStart containerd || exit {{GetCSEErrorCode "ERR_SYSTEMCTL_START_FAIL"}}
 }
 {{end}}
@@ -297,12 +294,7 @@ ensureDocker() {
   DOCKER_JSON_FILE=/etc/docker/daemon.json
   for i in $(seq 1 1200); do
     if [ -s $DOCKER_JSON_FILE ]; then
-      {{- if HasContainerDataDir}}
-      TMP=$(jq '.["data-root"]="{{GetContainerDataDir}}"' <$DOCKER_JSON_FILE)
-      echo "$TMP" > "$DOCKER_JSON_FILE" && break
-      {{- else}}
       jq '.' <$DOCKER_JSON_FILE && break
-      {{- end}}
     fi
     if [ $i -eq 1200 ]; then
       exit {{GetCSEErrorCode "ERR_FILE_WATCH_TIMEOUT"}}

parts/k8s/cloud-init/masternodecustomdata.yml

@@ -182,14 +182,9 @@ write_files:
   permissions: "0644"
   owner: root
   content: |
-    {
-      "live-restore": true,
-      "log-driver": "json-file",
-      "log-opts":  {
-         "max-size": "50m",
-         "max-file": "5"
-      }
-    }
+{{IndentString GetDockerConfig 4}}
+    #EOF
+
 {{end}}
 
 {{if HasCiliumNetworkPlugin}}
@@ -221,27 +216,9 @@ write_files:
   permissions: "0644"
   owner: root
   content: |
-    version = 2
-    subreaper = false
-    oom_score = 0
-
-    [plugins]
-      [plugins."io.containerd.grpc.v1.cri"]
-        sandbox_image = "{{GetPodInfraContainerSpec}}"
-        [plugins."io.containerd.grpc.v1.cri".cni]
-          {{if IsKubenet}}
-          conf_template = "/etc/containerd/kubenet_template.conf"
-          {{end}}
-        [plugins."io.containerd.grpc.v1.cri".containerd]
-          default_runtime_name = "runc"
-         [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
-            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
-              runtime_type = "io.containerd.runc.v2"
-            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.untrusted]
-              {{/* note: runc really should not be used for untrusted workloads... should we remove this? This is here because it was here before */}}
-              runtime_type = "io.containerd.runc.v2"
+{{IndentString GetContainerdConfig 4}}
     #EOF
-
+    
   {{if IsKubenet}}
 - path: /etc/containerd/kubenet_template.conf
   permissions: "0644"

parts/k8s/cloud-init/nodecustomdata.yml

@@ -176,23 +176,10 @@ write_files:
   permissions: "0644"
   owner: root
   content: |
-    {
-      "live-restore": true,
-      "log-driver": "json-file",
-      "log-opts":  {
-         "max-size": "50m",
-         "max-file": "5"
-      }{{if IsNSeriesSKU .}}
-      ,"default-runtime": "nvidia",
-      "runtimes": {
-         "nvidia": {
-             "path": "/usr/bin/nvidia-container-runtime",
-             "runtimeArgs": []
-        }
-      }{{end}}
-    }
-{{end}}
+{{IndentString GetDockerConfig 4}}
+    #EOF
 
+{{end}}
 {{if HasCiliumNetworkPlugin}}
 - path: /etc/systemd/system/sys-fs-bpf.mount
   permissions: "0644"
@@ -222,25 +209,7 @@ write_files:
   permissions: "0644"
   owner: root
   content: |
-    version = 2
-    subreaper = false
-    oom_score = 0
-
-    [plugins]
-      [plugins."io.containerd.grpc.v1.cri"]
-        sandbox_image = "{{GetPodInfraContainerSpec}}"
-        [plugins."io.containerd.grpc.v1.cri".cni]
-          {{if IsKubenet}}
-          conf_template = "/etc/containerd/kubenet_template.conf"
-          {{end}}
-        [plugins."io.containerd.grpc.v1.cri".containerd]
-          default_runtime_name = "runc"
-         [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
-            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
-              runtime_type = "io.containerd.runc.v2"
-            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.untrusted]
-              {{/* note: runc really should not be used for untrusted workloads... should we remove this? This is here because it was here before */}}
-              runtime_type = "io.containerd.runc.v2"
+{{IndentString GetContainerdConfig 4}}
     #EOF
 
   {{if IsKubenet }}

pkg/api/common/const.go

@@ -307,3 +307,53 @@ const (
 	KubernetesImageBaseTypeGCR = "gcr"
 	KubernetesImageBaseTypeMCR = "mcr"
 )
+
+var (
+	// DefaultDockerConfig describes the default configuration of the docker daemon.
+	DefaultDockerConfig = DockerConfig{
+		LiveRestore: true,
+		LogDriver:   "json-file",
+		LogOpts: LogOpts{
+			MaxSize: "50m",
+			MaxFile: "5",
+		},
+	}
+
+	// DefaultContainerdConfig describes the default configuration of the containerd daemon.
+	DefaultContainerdConfig = ContainerdConfig{
+		Version:  2,
+		OomScore: 0,
+		Plugins: Plugins{
+			IoContainerdGrpcV1Cri: IoContainerdGrpcV1Cri{
+				CNI: ContainerdCNIPlugin{},
+				Containerd: ContainerdPlugin{
+					DefaultRuntimeName: "runc",
+					Runtimes: map[string]ContainerdRuntime{
+						"runc": {
+							RuntimeType: "io.containerd.runc.v2",
+						},
+						// note: runc really should not be used for untrusted workloads... should we remove this? This is here because it was here before
+						"untrusted": {
+							RuntimeType: "io.containerd.runc.v2",
+						},
+					},
+				},
+			},
+		},
+	}
+)
+
+// GetDefaultDockerConfig returns the default docker config for processing.
+func GetDefaultDockerConfig() DockerConfig {
+	return DefaultDockerConfig
+}
+
+// GetDefaultContainerdConfig returns the default containerd config for processing.
+func GetDefaultContainerdConfig() ContainerdConfig {
+	return DefaultContainerdConfig
+}
+
+// Known container runtime configuration keys
+const (
+	ContainerDataDirKey = "dataDir"
+)

pkg/api/common/helper.go

@@ -4,12 +4,15 @@
 package common
 
 import (
+	"bufio"
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"regexp"
 	"sort"
 	"strings"
 
+	"github.com/BurntSushi/toml"
 	"github.com/pkg/errors"
 	validator "gopkg.in/go-playground/validator.v9"
 )
@@ -351,3 +354,83 @@ func WrapAsParameter(s string) string {
 func WrapAsVerbatim(s string) string {
 	return fmt.Sprintf("',%s,'", s)
 }
+
+// GetDockerConfig transforms the default docker config with overrides. Overrides may be nil.
+func GetDockerConfig(opts map[string]string, overrides []func(*DockerConfig) error) (string, error) {
+	config := GetDefaultDockerConfig()
+
+	for i := range overrides {
+		if err := overrides[i](&config); err != nil {
+			return "", err
+		}
+	}
+
+	dataDir, ok := opts[ContainerDataDirKey]
+	if ok {
+		config.DataRoot = dataDir
+	}
+
+	b, err := json.MarshalIndent(config, "", "    ")
+	return string(b), err
+}
+
+// GetContainerdConfig transforms the default containerd config with overrides. Overrides may be nil.
+func GetContainerdConfig(opts map[string]string, overrides []func(*ContainerdConfig) error) (string, error) {
+	config := GetDefaultContainerdConfig()
+
+	for i := range overrides {
+		if err := overrides[i](&config); err != nil {
+			return "", err
+		}
+	}
+
+	dataDir, ok := opts[ContainerDataDirKey]
+	if ok {
+		config.Root = dataDir
+	}
+
+	buf := new(bytes.Buffer)
+	err := toml.NewEncoder(buf).Encode(config)
+	return buf.String(), err
+}
+
+// ContainerdKubenetOverride transforms a containerd config to set details required when using kubenet.
+func ContainerdKubenetOverride(config *ContainerdConfig) error {
+	config.Plugins.IoContainerdGrpcV1Cri.CNI.ConfTemplate = "/etc/containerd/kubenet_template.conf"
+	return nil
+}
+
+// ContainerdSandboxImageOverrider produces a function to transform containerd config by setting the SandboxImage.
+func ContainerdSandboxImageOverrider(image string) func(*ContainerdConfig) error {
+	return func(config *ContainerdConfig) error {
+		config.Plugins.IoContainerdGrpcV1Cri.SandboxImage = image
+		return nil
+	}
+}
+
+// DockerNvidiaOverride transforms a docker config to supply nvidia runtime configuration.
+func DockerNvidiaOverride(config *DockerConfig) error {
+	if config.DockerDaemonRuntimes == nil {
+		config.DockerDaemonRuntimes = make(map[string]DockerDaemonRuntime)
+	}
+	config.DefaultRuntime = "nvidia"
+	config.DockerDaemonRuntimes["nvidia"] = DockerDaemonRuntime{
+		Path:        "/usr/bin/nvidia-container-runtime",
+		RuntimeArgs: []string{},
+	}
+	return nil
+}
+
+// IndentString pads each line of an original string with N spaces and returns the new value.
+func IndentString(original string, spaces int) string {
+	out := bytes.NewBuffer(nil)
+	scanner := bufio.NewScanner(strings.NewReader(original))
+	for scanner.Scan() {
+		for i := 0; i < spaces; i++ {
+			out.WriteString(" ")
+		}
+		out.WriteString(scanner.Text())
+		out.WriteString("\n")
+	}
+	return out.String()
+}