feat: modify container runtime data dir

Azure · Apr 15, 2020 · e00c4f0 · e00c4f0
1 parent d6bf733
commit e00c4f0
Show file tree

Hide file tree

Showing 14 changed files with 242 additions and 0 deletions.
diff --git a/docs/topics/clusterdefinitions.md b/docs/topics/clusterdefinitions.md
@@ -44,6 +44,7 @@ $ aks-engine get-versions
 | cloudControllerManagerConfig     | no                        | Configure various runtime configuration for cloud-controller-manager. See `cloudControllerManagerConfig` [below](#feat-cloud-controller-manager-config)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   |
 | clusterSubnet                    | no                        | The IP subnet used for allocating IP addresses for pod network interfaces. The subnet must be in the VNET address space. With Azure CNI enabled, the default value is 10.240.0.0/12. Without Azure CNI, the default value is 10.244.0.0/16.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |
 | containerRuntime                 | no                        | The container runtime to use as a backend. The default is `docker`. Also supported is `containerd`. Windows support for `containerd` is **Experimental** - see [Windows ContainerD](features.md#windows-containerd) |
+| containerRuntimeConfig                | no                        | A map of key-value pairs to drive configuration of the container runtime. Currently a single key is accepted, "dataDir", which configures the root data directory for the container runtime. dataDir must be an absolute path. This is only implemented on Linux. See an [example](../../examples/kubernetes-config/kubernetes-docker-tmpdir.json) which places docker on the tmp disk of a Linux VM. |
 | controllerManagerConfig          | no                        | Configure various runtime configuration for controller-manager. See `controllerManagerConfig` [below](#feat-controller-manager-config)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
 | customWindowsPackageURL          | no                        | Configure custom windows Kubernetes release package URL for deployment on Windows. The format of this file is a zip file with multiple items (binaries, cni, infra container) in it. This setting will be deprecated in a future release of aks-engine where the binaries will be pulled in the format of Kubernetes releases that only contain the kubernetes binaries.                                                                                                                                                                                                                                                                                                                 |
 | WindowsNodeBinariesURL           | no                        | Windows Kubernetes Node binaries can be provided in the format of Kubernetes release (example: https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.11.md#node-binaries-1). This setting allows overriding the binaries for custom builds.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |

diff --git a/examples/kubernetes-config/kubernetes-docker-tmpdir.json b/examples/kubernetes-config/kubernetes-docker-tmpdir.json
@@ -0,0 +1,46 @@
+{
+  "apiVersion": "vlabs",
+  "properties": {
+    "orchestratorProfile": {
+      "orchestratorType": "Kubernetes",
+      "orchestratorRelease": "1.17",
+      "kubernetesConfig": {
+        "containerRuntime": "docker",
+        "containerRuntimeConfig": {
+          "dataDir": "/mnt/docker"
+        }
+      }
+    },
+    "masterProfile": {
+      "count": 1,
+      "dnsPrefix": "",
+      "vmSize": "Standard_D8s_v3",
+      "osDiskSizeGb": 1024,
+      "distro": "ubuntu-18.04"
+    },
+    "agentPoolProfiles": [
+      {
+        "name": "agentpool1",
+        "count": 1,
+        "vmSize": "Standard_D8s_v3",
+        "availabilityProfile": "VirtualMachineScaleSets",
+        "osDiskSizeGb": 1024,
+        "distro": "ubuntu-18.04"
+      }
+    ],
+    "linuxProfile": {
+      "adminUsername": "azureuser",
+      "ssh": {
+        "publicKeys": [
+          {
+            "keyData": ""
+          }
+        ]
+      }
+    },
+    "servicePrincipalProfile": {
+      "clientId": "",
+      "secret": ""
+    }
+  }
+}
diff --git a/parts/k8s/cloud-init/artifacts/cse_config.sh b/parts/k8s/cloud-init/artifacts/cse_config.sh
@@ -282,6 +282,9 @@ installContainerd() {
 ensureContainerd() {
   wait_for_file 1200 1 /etc/systemd/system/containerd.service.d/exec_start.conf || exit {{GetCSEErrorCode "ERR_FILE_WATCH_TIMEOUT"}}
   wait_for_file 1200 1 /etc/containerd/config.toml || exit {{GetCSEErrorCode "ERR_FILE_WATCH_TIMEOUT"}}
+  {{- if HasContainerDataDir}}
+  echo -e "root = \"{{GetContainerDataDir}}\"\n$(cat /etc/containerd/config.toml)" > /etc/containerd/config.toml
+  {{- end}}
   systemctlEnableAndStart containerd || exit {{GetCSEErrorCode "ERR_SYSTEMCTL_START_FAIL"}}
 }
 {{end}}
@@ -294,7 +297,12 @@ ensureDocker() {
   DOCKER_JSON_FILE=/etc/docker/daemon.json
   for i in $(seq 1 1200); do
     if [ -s $DOCKER_JSON_FILE ]; then
+      {{- if HasContainerDataDir}}
+      TMP=$(jq '.["data-root"]="{{GetContainerDataDir}}"' <$DOCKER_JSON_FILE)
+      echo "$TMP" > "$DOCKER_JSON_FILE" && break
+      {{- else}}
       jq '.' <$DOCKER_JSON_FILE && break
+      {{- end}}
     fi
     if [ $i -eq 1200 ]; then
       exit {{GetCSEErrorCode "ERR_FILE_WATCH_TIMEOUT"}}

diff --git a/pkg/api/const.go b/pkg/api/const.go
@@ -114,6 +114,11 @@ const (
 	Containerd     = "containerd"
 )
 
+// Known container runtime configuration keys
+const (
+	ContainerDataDirKey = "dataDir"
+)
+
 // storage profiles
 const (
 	// StorageAccount means that the nodes use raw storage accounts for their os and attached volumes

diff --git a/pkg/api/converterfromapi.go b/pkg/api/converterfromapi.go
@@ -333,6 +333,14 @@ func convertKubernetesConfigToVLabs(apiCfg *KubernetesConfig, vlabsCfg *vlabs.Ku
 	convertSchedulerConfigToVlabs(apiCfg, vlabsCfg)
 	convertPrivateClusterToVlabs(apiCfg, vlabsCfg)
 	convertPodSecurityPolicyConfigToVlabs(apiCfg, vlabsCfg)
+	convertContainerRuntimeConfigToVlabs(apiCfg, vlabsCfg)
+}
+
+func convertContainerRuntimeConfigToVlabs(a *KubernetesConfig, v *vlabs.KubernetesConfig) {
+	v.ContainerRuntimeConfig = map[string]string{}
+	for key, val := range a.ContainerRuntimeConfig {
+		v.ContainerRuntimeConfig[key] = val
+	}
 }
 
 func convertKubeletConfigToVlabs(a *KubernetesConfig, v *vlabs.KubernetesConfig) {

diff --git a/pkg/api/convertertoapi.go b/pkg/api/convertertoapi.go
@@ -347,6 +347,7 @@ func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *Kubernetes
 	convertSchedulerConfigToAPI(vlabs, api)
 	convertPrivateClusterToAPI(vlabs, api)
 	convertPodSecurityPolicyConfigToAPI(vlabs, api)
+	convertContainerRuntimeConfigToAPI(vlabs, api)
 }
 
 func setVlabsKubernetesDefaults(vp *vlabs.Properties, api *OrchestratorProfile) {
@@ -459,6 +460,13 @@ func convertCustomFilesToAPI(v *vlabs.MasterProfile, a *MasterProfile) {
 	}
 }
 
+func convertContainerRuntimeConfigToAPI(v *vlabs.KubernetesConfig, a *KubernetesConfig) {
+	a.ContainerRuntimeConfig = map[string]string{}
+	for key, val := range v.ContainerRuntimeConfig {
+		a.ContainerRuntimeConfig[key] = val
+	}
+}
+
 func convertKubeletConfigToAPI(v *vlabs.KubernetesConfig, a *KubernetesConfig) {
 	a.KubeletConfig = map[string]string{}
 	for key, val := range v.KubeletConfig {

diff --git a/pkg/api/types.go b/pkg/api/types.go
@@ -467,6 +467,7 @@ type KubernetesConfig struct {
 	Addons                            []KubernetesAddon     `json:"addons,omitempty"`
 	Components                        []KubernetesComponent `json:"components,omitempty"`
 	KubeletConfig                     map[string]string     `json:"kubeletConfig,omitempty"`
+	ContainerRuntimeConfig            map[string]string     `json:"containerRuntimeConfig"`
 	ControllerManagerConfig           map[string]string     `json:"controllerManagerConfig,omitempty"`
 	CloudControllerManagerConfig      map[string]string     `json:"cloudControllerManagerConfig,omitempty"`
 	APIServerConfig                   map[string]string     `json:"apiServerConfig,omitempty"`
@@ -1980,6 +1981,17 @@ func (k *KubernetesConfig) ShouldCreateNewUserAssignedIdentity() bool {
 	return !(k.UserAssignedIDEnabled() && strings.Contains(k.UserAssignedID, "/"))
 }
 
+// HasContainerDataDir returns the path to the container runtime data directory, if specified.
+func (k *KubernetesConfig) HasContainerDataDir() bool {
+	_, ok := k.ContainerRuntimeConfig[ContainerDataDirKey]
+	return ok
+}
+
+// GetContainerDataDir returns the path to the container runtime dataDir directory, if specified.
+func (k *KubernetesConfig) GetContainerDataDir() string {
+	return k.ContainerRuntimeConfig[ContainerDataDirKey]
+}
+
 // GetOrderedKubeletConfigString returns an ordered string of key/val pairs
 func (k *KubernetesConfig) GetOrderedKubeletConfigString() string {
 	keys := []string{}

diff --git a/pkg/api/types_test.go b/pkg/api/types_test.go
@@ -7331,6 +7331,70 @@ func TestHasContainerd(t *testing.T) {
 	}
 }
 
+func TestContainerDataDir(t *testing.T) {
+	tests := []struct {
+		name           string
+		k              *KubernetesConfig
+		expectedExists bool
+		expectedPath   string
+	}{
+		{
+			name: "valid if unspecified",
+			k: &KubernetesConfig{
+				ContainerRuntimeConfig: map[string]string{},
+			},
+			expectedExists: false,
+			expectedPath:   "",
+		},
+		{
+			name: "empty string",
+			k: &KubernetesConfig{
+				ContainerRuntimeConfig: map[string]string{
+					ContainerDataDirKey: "",
+				},
+			},
+			expectedExists: true,
+			expectedPath:   "",
+		},
+		{
+			name: "invalid relative path",
+			k: &KubernetesConfig{
+				ContainerRuntimeConfig: map[string]string{
+					ContainerDataDirKey: "mnt/docker",
+				},
+			},
+			expectedExists: true,
+			expectedPath:   "mnt/docker",
+		},
+		{
+			name: "valid name",
+			k: &KubernetesConfig{
+				ContainerRuntimeConfig: map[string]string{
+					ContainerDataDirKey: "/mnt/docker",
+				},
+			},
+			expectedExists: true,
+			expectedPath:   "/mnt/docker",
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+			exists := test.k.HasContainerDataDir()
+			path := test.k.GetContainerDataDir()
+
+			if test.expectedExists != exists {
+				t.Errorf("case: %s -- expected to exist: %t, instead got : %t", test.name, test.expectedExists, exists)
+			}
+			if test.expectedPath != path {
+				t.Errorf("case: %s -- expected value: %s, instead got : %s", test.name, test.expectedPath, path)
+			}
+		})
+	}
+}
+
 func TestGetNonMasqueradeCIDR(t *testing.T) {
 	tests := []struct {
 		name     string

diff --git a/pkg/api/vlabs/const.go b/pkg/api/vlabs/const.go
@@ -88,6 +88,11 @@ const (
 	Containerd     = "containerd"
 )
 
+// Known container runtime configuration keys
+const (
+	ContainerDataDirKey = "dataDir"
+)
+
 var (
 	// NetworkPluginValues holds the valid values for network plugin implementation
 	NetworkPluginValues = [...]string{"", "kubenet", "azure", NetworkPluginCilium, NetworkPluginAntrea, "flannel"}

diff --git a/pkg/api/vlabs/types.go b/pkg/api/vlabs/types.go
@@ -348,6 +348,7 @@ type KubernetesConfig struct {
 	EnablePodSecurityPolicy           *bool                 `json:"enablePodSecurityPolicy,omitempty"`
 	Addons                            []KubernetesAddon     `json:"addons,omitempty"`
 	Components                        []KubernetesComponent `json:"components,omitempty"`
+	ContainerRuntimeConfig            map[string]string     `json:"containerRuntimeConfig,omitempty"`
 	KubeletConfig                     map[string]string     `json:"kubeletConfig,omitempty"`
 	ControllerManagerConfig           map[string]string     `json:"controllerManagerConfig,omitempty"`
 	CloudControllerManagerConfig      map[string]string     `json:"cloudControllerManagerConfig,omitempty"`

diff --git a/pkg/api/vlabs/validate.go b/pkg/api/vlabs/validate.go
@@ -1543,6 +1543,23 @@ func (k *KubernetesConfig) Validate(k8sVersion string, hasWindows, ipv6DualStack
 	if e := k.validateKubernetesImageBaseType(); e != nil {
 		return e
 	}
+	if e := k.validateContainerRuntimeConfig(); e != nil {
+		return e
+	}
+	return nil
+}
+
+func (k *KubernetesConfig) validateContainerRuntimeConfig() error {
+	if k.ContainerRuntimeConfig != nil {
+		if val, ok := k.ContainerRuntimeConfig[ContainerDataDirKey]; ok {
+			if val == "" {
+				return errors.Errorf("OrchestratorProfile.KubernetesConfig.ContainerRuntimeConfig.DataDir '%s' is invalid: must not be empty", val)
+			}
+			if !strings.HasPrefix(val, "/") {
+				return errors.Errorf("OrchestratorProfile.KubernetesConfig.ContainerRuntimeConfig.DataDir '%s' is invalid: must be absolute path", val)
+			}
+		}
+	}
 	return nil
 }
 

diff --git a/pkg/api/vlabs/validate_test.go b/pkg/api/vlabs/validate_test.go
@@ -4745,3 +4745,56 @@ func TestValidateKubernetesImageBaseType(t *testing.T) {
 		})
 	}
 }
+
+func TestValidateContainerRuntimeConfig(t *testing.T) {
+	tests := map[string]struct {
+		k             *KubernetesConfig
+		expectedError error
+	}{
+		"should ignore and validate if unspecified": {
+			k:             &KubernetesConfig{},
+			expectedError: nil,
+		},
+		"should succeed if config is defined but key not present": {
+			k: &KubernetesConfig{
+				ContainerRuntimeConfig: map[string]string{},
+			},
+			expectedError: nil,
+		},
+		"should fail on empty string": {
+			k: &KubernetesConfig{
+				ContainerRuntimeConfig: map[string]string{
+					ContainerDataDirKey: "",
+				},
+			},
+			expectedError: errors.Errorf("OrchestratorProfile.KubernetesConfig.ContainerRuntimeConfig.DataDir '' is invalid: must not be empty"),
+		},
+		"should fail on relative path": {
+			k: &KubernetesConfig{
+				ContainerRuntimeConfig: map[string]string{
+					ContainerDataDirKey: "mnt/docker",
+				},
+			},
+			expectedError: errors.Errorf("OrchestratorProfile.KubernetesConfig.ContainerRuntimeConfig.DataDir 'mnt/docker' is invalid: must be absolute path"),
+		},
+		"should pass with absolute path": {
+			k: &KubernetesConfig{
+				ContainerRuntimeConfig: map[string]string{
+					ContainerDataDirKey: "/mnt/docker",
+				},
+			},
+			expectedError: nil,
+		},
+	}
+
+	for testName, test := range tests {
+		test := test
+		t.Run(testName, func(t *testing.T) {
+			t.Parallel()
+			err := test.k.validateContainerRuntimeConfig()
+			if !helpers.EqualError(err, test.expectedError) {
+				t.Errorf("expected error: %v, got: %v", test.expectedError, err)
+			}
+		})
+	}
+}
diff --git a/pkg/engine/template_generator.go b/pkg/engine/template_generator.go
@@ -692,6 +692,12 @@ func getContainerServiceFuncMap(cs *api.ContainerService) template.FuncMap {
 		"IsDockerContainerRuntime": func() bool {
 			return cs.Properties.OrchestratorProfile.KubernetesConfig.ContainerRuntime == api.Docker
 		},
+		"HasContainerDataDir": func() bool {
+			return cs.Properties.OrchestratorProfile.KubernetesConfig.HasContainerDataDir()
+		},
+		"GetContainerDataDir": func() string {
+			return cs.Properties.OrchestratorProfile.KubernetesConfig.GetContainerDataDir()
+		},
 		"HasNSeriesSKU": func() bool {
 			return cs.Properties.HasNSeriesSKU()
 		},

diff --git a/pkg/engine/templates_generated.go b/pkg/engine/templates_generated.go