fix: tests + validation

Azure · Apr 17, 2020 · e2ae09a · e2ae09a
1 parent 9fbb6ec
commit e2ae09a
Show file tree

Hide file tree

Showing 10 changed files with 255 additions and 45 deletions.
diff --git a/go.mod b/go.mod
@@ -40,7 +40,6 @@ require (
 	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/onsi/ginkgo v1.10.3
 	github.com/onsi/gomega v1.4.3
-	github.com/pelletier/go-toml v1.7.0
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pkg/errors v0.8.1
 	github.com/satori/go.uuid v1.2.0 // indirect
@@ -59,6 +58,7 @@ require (
 	gopkg.in/go-playground/validator.v9 v9.25.0
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.41.0
+	gopkg.in/yaml.v2 v2.2.8 // indirect
 	k8s.io/api v0.0.0-20190222213804-5cb15d344471
 	k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628
 	k8s.io/client-go v10.0.0+incompatible

diff --git a/go.sum b/go.sum
@@ -111,8 +111,6 @@ github.com/onsi/ginkgo v1.10.3 h1:OoxbjfXVZyod1fmWYhI7SEyaD8B00ynP3T+D5GiyHOY=
 github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI=
-github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
 github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=

diff --git a/pkg/api/common/const.go b/pkg/api/common/const.go
@@ -310,7 +310,7 @@ const (
 
 var (
 	// DefaultDockerConfig describes the default configuration of the docker daemon.
-	DefaultDockerConfig = &DockerConfig{
+	DefaultDockerConfig = DockerConfig{
 		LiveRestore: true,
 		LogDriver:   "json-file",
 		LogOpts: LogOpts{
@@ -320,7 +320,7 @@ var (
 	}
 
 	// DefaultContainerdConfig describes the default configuration of the containerd daemon.
-	DefaultContainerdConfig = &ContainerdConfig{
+	DefaultContainerdConfig = ContainerdConfig{
 		Version: 2,
 		// should this be true? seems maybe not https://github.com/containerd/containerd/blob/master/docs/ops.md#base-configuration
 		Subreaper: false,
@@ -346,23 +346,16 @@ var (
 )
 
 // GetDefaultDockerConfig returns the default docker config for processing.
-func GetDefaultDockerConfig() *DockerConfig {
+func GetDefaultDockerConfig() DockerConfig {
 	return DefaultDockerConfig
 }
 
 // GetDefaultContainerdConfig returns the default containerd config for processing.
-func GetDefaultContainerdConfig() *ContainerdConfig {
+func GetDefaultContainerdConfig() ContainerdConfig {
 	return DefaultContainerdConfig
 }
 
 // Known container runtime configuration keys
 const (
 	ContainerDataDirKey = "dataDir"
-	// Key to the name of cgroup driver for container runtime (kubelet must match)
-	// TODO(ace): validation against kubelet matching?
-	CgroupDriverKey = "cgroupDriver"
-	// Systemd is a possible cgroup driver
-	Systemd = "systemd"
-	// Cgroupfs is a possible cgroup driver
-	Cgroupfs = "cgroupfs"
 )
diff --git a/pkg/api/common/helper.go b/pkg/api/common/helper.go
@@ -354,12 +354,12 @@ func WrapAsVerbatim(s string) string {
 	return fmt.Sprintf("',%s,'", s)
 }
 
-// GetDockerConfig transforms the default docker config with overrides forced by tangential settings. Overrides may be nil.
-func GetDockerConfig(opts map[string]string, overrides ...func(*DockerConfig) error) (string, error) {
+// GetDockerConfig transforms the default docker config with overrides. Overrides may be nil.
+func GetDockerConfig(opts map[string]string, overrides []func(*DockerConfig) error) (string, error) {
 	config := GetDefaultDockerConfig()
 
 	for i := range overrides {
-		if err := overrides[i](config); err != nil {
+		if err := overrides[i](&config); err != nil {
 			return "", err
 		}
 	}
@@ -369,21 +369,16 @@ func GetDockerConfig(opts map[string]string, overrides ...func(*DockerConfig) er
 		config.DataRoot = dataDir
 	}
 
-	cgroupDriver, ok := opts[CgroupDriverKey]
-	if ok {
-		config.ExecOpts = []string{fmt.Sprintf("native.cgroupdriver=%s", cgroupDriver)}
-	}
-
 	b, err := json.MarshalIndent(config, "", "    ")
 	return string(b), err
 }
 
-// GetContainerdConfig transforms the default containerd config with overrides forced by tangential settings. Overrides may be nil.
-func GetContainerdConfig(opts map[string]string, overrides ...func(*ContainerdConfig) error) (string, error) {
+// GetContainerdConfig transforms the default containerd config with overrides. Overrides may be nil.
+func GetContainerdConfig(opts map[string]string, overrides []func(*ContainerdConfig) error) (string, error) {
 	config := GetDefaultContainerdConfig()
 
 	for i := range overrides {
-		if err := overrides[i](config); err != nil {
+		if err := overrides[i](&config); err != nil {
 			return "", err
 		}
 	}
@@ -411,3 +406,16 @@ func ContainerdSandboxImageOverrider(image string) func(*ContainerdConfig) error
 		return nil
 	}
 }
+
+// DockerNvidiaOverride transforms a docker config to supply nvidia runtime configuration.
+func DockerNvidiaOverride(config *DockerConfig) error {
+	if config.DockerDaemonRuntimes == nil {
+		config.DockerDaemonRuntimes = make(map[string]DockerDaemonRuntime)
+	}
+	config.DefaultRuntime = "nvidia"
+	config.DockerDaemonRuntimes["nvidia"] = DockerDaemonRuntime{
+		Path:        "/usr/bin/nvidia-container-runtime",
+		RuntimeArgs: []string{},
+	}
+	return nil
+}
diff --git a/pkg/api/common/helper_test.go b/pkg/api/common/helper_test.go
@@ -6,6 +6,7 @@ package common
 import (
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
 	"github.com/pkg/errors"
 )
 
@@ -427,3 +428,198 @@ func TestWrapAsVerbatim(t *testing.T) {
 		})
 	}
 }
+
+func TestGetDockerConfig(t *testing.T) {
+	tests := []struct {
+		name      string
+		options   map[string]string
+		overrides []func(*DockerConfig) error
+		want      string
+	}{
+		{
+			name: "docker default config",
+			want: defaultDockerConfigString,
+		},
+		{
+			name: "docker reroot config",
+			want: dockerRerootConfigString,
+			options: map[string]string{
+				"dataDir": "/mnt/docker",
+			},
+		},
+		{
+			name: "docker nvidia config",
+			want: dockerNvidiaConfigString,
+			overrides: []func(*DockerConfig) error{
+				DockerNvidiaOverride,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+			got, err := GetDockerConfig(test.options, test.overrides)
+			if err != nil {
+				t.Fatalf("failed to get docker config: %v", err)
+			}
+			diff := cmp.Diff(test.want, got)
+			if diff != "" {
+				t.Fatalf(diff)
+			}
+		})
+	}
+}
+
+func TestGetContainerdConfig(t *testing.T) {
+	tests := []struct {
+		name      string
+		options   map[string]string
+		overrides []func(*ContainerdConfig) error
+		want      string
+	}{
+		{
+			name: "container default config",
+			want: defaultContainerdConfigString,
+		},
+		{
+			name: "container reroot config",
+			want: containerdRerootConfigString,
+			options: map[string]string{
+				"dataDir": "/mnt/containerd",
+			},
+		},
+		{
+			name: "container kubenet config",
+			want: containerdKubenetConfigString,
+			overrides: []func(*ContainerdConfig) error{
+				ContainerdKubenetOverride,
+			},
+		},
+		{
+			name: "container sandbox image config",
+			want: containerdImageConfigString,
+			overrides: []func(*ContainerdConfig) error{
+				ContainerdSandboxImageOverrider("foobar"),
+			},
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+			got, err := GetContainerdConfig(test.options, test.overrides)
+			if err != nil {
+				t.Fatalf("failed to get docker config: %v", err)
+			}
+			diff := cmp.Diff(test.want, got)
+			if diff != "" {
+				t.Fatalf(diff)
+			}
+		})
+	}
+}
+
+var defaultContainerdConfigString = `oom_score = 0
+subreaper = false
+version = 2
+
+[plugins]
+  [plugins."io.containerd.grpc.v1.cri"]
+    [plugins."io.containerd.grpc.v1.cri".cni]
+    [plugins."io.containerd.grpc.v1.cri".containerd]
+      default_runtime_name = "runc"
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          runtime_type = "io.containerd.runc.v2"
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.untrusted]
+          runtime_type = "io.containerd.runc.v2"
+`
+
+var containerdRerootConfigString = `oom_score = 0
+root = "/mnt/containerd"
+subreaper = false
+version = 2
+
+[plugins]
+  [plugins."io.containerd.grpc.v1.cri"]
+    [plugins."io.containerd.grpc.v1.cri".cni]
+    [plugins."io.containerd.grpc.v1.cri".containerd]
+      default_runtime_name = "runc"
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          runtime_type = "io.containerd.runc.v2"
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.untrusted]
+          runtime_type = "io.containerd.runc.v2"
+`
+
+var containerdKubenetConfigString = `oom_score = 0
+subreaper = false
+version = 2
+
+[plugins]
+  [plugins."io.containerd.grpc.v1.cri"]
+    [plugins."io.containerd.grpc.v1.cri".cni]
+      conf_template = "/etc/containerd/kubenet_template.conf"
+    [plugins."io.containerd.grpc.v1.cri".containerd]
+      default_runtime_name = "runc"
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          runtime_type = "io.containerd.runc.v2"
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.untrusted]
+          runtime_type = "io.containerd.runc.v2"
+`
+
+var containerdImageConfigString = `oom_score = 0
+subreaper = false
+version = 2
+
+[plugins]
+  [plugins."io.containerd.grpc.v1.cri"]
+    sandbox_image = "foobar"
+    [plugins."io.containerd.grpc.v1.cri".cni]
+    [plugins."io.containerd.grpc.v1.cri".containerd]
+      default_runtime_name = "runc"
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          runtime_type = "io.containerd.runc.v2"
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.untrusted]
+          runtime_type = "io.containerd.runc.v2"
+`
+
+var defaultDockerConfigString = `{
+    "live-restore": true,
+    "log-driver": "json-file",
+    "log-opts": {
+        "max-size": "50m",
+        "max-file": "5"
+    }
+}`
+
+var dockerRerootConfigString = `{
+    "data-root": "/mnt/docker",
+    "live-restore": true,
+    "log-driver": "json-file",
+    "log-opts": {
+        "max-size": "50m",
+        "max-file": "5"
+    }
+}`
+
+var dockerNvidiaConfigString = `{
+    "live-restore": true,
+    "log-driver": "json-file",
+    "log-opts": {
+        "max-size": "50m",
+        "max-file": "5"
+    },
+    "default-runtime": "nvidia",
+    "runtimes": {
+        "nvidia": {
+            "path": "/usr/bin/nvidia-container-runtime",
+            "runtimeArgs": []
+        }
+    }
+}`
diff --git a/pkg/api/common/types.go b/pkg/api/common/types.go
@@ -1,3 +1,6 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
 package common
 
 type ContainerdConfig struct {
@@ -47,6 +50,6 @@ type LogOpts struct {
 }
 
 type DockerDaemonRuntime struct {
-	Path        string        `json:"path,omitempty"`
-	RuntimeArgs []interface{} `json:"runtimeArgs,omitempty"`
+	Path        string   `json:"path,omitempty"`
+	RuntimeArgs []string `json:"runtimeArgs"`
 }
diff --git a/pkg/api/converterfromapi_test.go b/pkg/api/converterfromapi_test.go
@@ -9,6 +9,7 @@ import (
 
 	"github.com/Azure/go-autorest/autorest/to"
 
+	"github.com/Azure/aks-engine/pkg/api/common"
 	"github.com/Azure/aks-engine/pkg/api/vlabs"
 	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/google/go-cmp/cmp"
@@ -507,11 +508,14 @@ func getDefaultContainerService() *ContainerService {
 					ResourceGroup: "FooImageRefResourceGroup",
 				},
 				KubernetesConfig: &KubernetesConfig{
-					KubernetesImageBase:             "quay.io",
-					ClusterSubnet:                   "fooClusterSubnet",
-					NetworkPolicy:                   "calico",
-					NetworkPlugin:                   "azure-cni",
-					ContainerRuntime:                "docker",
+					KubernetesImageBase: "quay.io",
+					ClusterSubnet:       "fooClusterSubnet",
+					NetworkPolicy:       "calico",
+					NetworkPlugin:       "azure-cni",
+					ContainerRuntime:    "docker",
+					ContainerRuntimeConfig: map[string]string{
+						common.ContainerDataDirKey: "/mnt/docker",
+					},
 					MaxPods:                         3,
 					DockerBridgeSubnet:              "sampleDockerSubnet",
 					DNSServiceIP:                    "172.0.0.1",