fix: tests + validation

go.mod

@@ -43,6 +43,7 @@ require (
 	github.com/pelletier/go-toml v1.7.0
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pkg/errors v0.8.1
+	github.com/sanity-io/litter v1.2.0
 	github.com/satori/go.uuid v1.2.0 // indirect
 	github.com/sirupsen/logrus v1.4.1
 	github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect

go.sum

@@ -31,6 +31,7 @@ github.com/Jeffail/gabs v1.1.1 h1:V0uzR08Hj22EX8+8QMhyI9sX2hwRu+/RJhJUmnwda/E=
 github.com/Jeffail/gabs v1.1.1/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -117,8 +118,10 @@ github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+v
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sanity-io/litter v1.2.0/go.mod h1:JF6pZUFgu2Q0sBZ+HSV35P8TVPI1TTzEwyu9FXAw2W4=
 github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k=
@@ -133,6 +136,7 @@ github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=

pkg/api/common/const.go

@@ -310,7 +310,7 @@ const (
 
 var (
 	// DefaultDockerConfig describes the default configuration of the docker daemon.
-	DefaultDockerConfig = &DockerConfig{
+	DefaultDockerConfig = DockerConfig{
 		LiveRestore: true,
 		LogDriver:   "json-file",
 		LogOpts: LogOpts{
@@ -320,7 +320,7 @@ var (
 	}
 
 	// DefaultContainerdConfig describes the default configuration of the containerd daemon.
-	DefaultContainerdConfig = &ContainerdConfig{
+	DefaultContainerdConfig = ContainerdConfig{
 		Version: 2,
 		// should this be true? seems maybe not https://github.com/containerd/containerd/blob/master/docs/ops.md#base-configuration
 		Subreaper: false,
@@ -346,23 +346,16 @@ var (
 )
 
 // GetDefaultDockerConfig returns the default docker config for processing.
-func GetDefaultDockerConfig() *DockerConfig {
+func GetDefaultDockerConfig() DockerConfig {
 	return DefaultDockerConfig
 }
 
 // GetDefaultContainerdConfig returns the default containerd config for processing.
-func GetDefaultContainerdConfig() *ContainerdConfig {
+func GetDefaultContainerdConfig() ContainerdConfig {
 	return DefaultContainerdConfig
 }
 
 // Known container runtime configuration keys
 const (
 	ContainerDataDirKey = "dataDir"
-	// Key to the name of cgroup driver for container runtime (kubelet must match)
-	// TODO(ace): validation against kubelet matching?
-	CgroupDriverKey = "cgroupDriver"
-	// Systemd is a possible cgroup driver
-	Systemd = "systemd"
-	// Cgroupfs is a possible cgroup driver
-	Cgroupfs = "cgroupfs"
 )

pkg/api/common/helper.go

@@ -354,12 +354,12 @@ func WrapAsVerbatim(s string) string {
 	return fmt.Sprintf("',%s,'", s)
 }
 
-// GetDockerConfig transforms the default docker config with overrides forced by tangential settings. Overrides may be nil.
-func GetDockerConfig(opts map[string]string, overrides ...func(*DockerConfig) error) (string, error) {
+// GetDockerConfig transforms the default docker config with overrides. Overrides may be nil.
+func GetDockerConfig(opts map[string]string, overrides []func(*DockerConfig) error) (string, error) {
 	config := GetDefaultDockerConfig()
 
 	for i := range overrides {
-		if err := overrides[i](config); err != nil {
+		if err := overrides[i](&config); err != nil {
 			return "", err
 		}
 	}
@@ -369,21 +369,16 @@ func GetDockerConfig(opts map[string]string, overrides ...func(*DockerConfig) er
 		config.DataRoot = dataDir
 	}
 
-	cgroupDriver, ok := opts[CgroupDriverKey]
-	if ok {
-		config.ExecOpts = []string{fmt.Sprintf("native.cgroupdriver=%s", cgroupDriver)}
-	}
-
 	b, err := json.MarshalIndent(config, "", "    ")
 	return string(b), err
 }
 
-// GetContainerdConfig transforms the default containerd config with overrides forced by tangential settings. Overrides may be nil.
-func GetContainerdConfig(opts map[string]string, overrides ...func(*ContainerdConfig) error) (string, error) {
+// GetContainerdConfig transforms the default containerd config with overrides. Overrides may be nil.
+func GetContainerdConfig(opts map[string]string, overrides []func(*ContainerdConfig) error) (string, error) {
 	config := GetDefaultContainerdConfig()
 
 	for i := range overrides {
-		if err := overrides[i](config); err != nil {
+		if err := overrides[i](&config); err != nil {
 			return "", err
 		}
 	}
@@ -411,3 +406,16 @@ func ContainerdSandboxImageOverrider(image string) func(*ContainerdConfig) error
 		return nil
 	}
 }
+
+// DockerNvidiaOverride transforms a docker config to supply nvidia runtime configuration.
+func DockerNvidiaOverride(config *DockerConfig) error {
+	if config.DockerDaemonRuntimes == nil {
+		config.DockerDaemonRuntimes = make(map[string]DockerDaemonRuntime)
+	}
+	config.DefaultRuntime = "nvidia"
+	config.DockerDaemonRuntimes["nvidia"] = DockerDaemonRuntime{
+		Path:        "/usr/bin/nvidia-container-runtime",
+		RuntimeArgs: []string{},
+	}
+	return nil
+}

pkg/api/common/helper_test.go

@@ -6,6 +6,7 @@ package common
 import (
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
 	"github.com/pkg/errors"
 )
 
@@ -427,3 +428,198 @@ func TestWrapAsVerbatim(t *testing.T) {
 		})
 	}
 }
+
+func TestGetDockerConfig(t *testing.T) {
+	tests := []struct {
+		name      string
+		options   map[string]string
+		overrides []func(*DockerConfig) error
+		want      string
+	}{
+		{
+			name: "docker default config",
+			want: defaultDockerConfigString,
+		},
+		{
+			name: "docker reroot config",
+			want: dockerRerootConfigString,
+			options: map[string]string{
+				"dataDir": "/mnt/docker",
+			},
+		},
+		{
+			name: "docker nvidia config",
+			want: dockerNvidiaConfigString,
+			overrides: []func(*DockerConfig) error{
+				DockerNvidiaOverride,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+			got, err := GetDockerConfig(test.options, test.overrides)
+			if err != nil {
+				t.Fatalf("failed to get docker config: %v", err)
+			}
+			diff := cmp.Diff(test.want, got)
+			if diff != "" {
+				t.Fatalf(diff)
+			}
+		})
+	}
+}
+
+func TestGetContainerdConfig(t *testing.T) {
+	tests := []struct {
+		name      string
+		options   map[string]string
+		overrides []func(*ContainerdConfig) error
+		want      string
+	}{
+		{
+			name: "container default config",
+			want: defaultContainerdConfigString,
+		},
+		{
+			name: "container reroot config",
+			want: containerdRerootConfigString,
+			options: map[string]string{
+				"dataDir": "/mnt/containerd",
+			},
+		},
+		{
+			name: "container kubenet config",
+			want: containerdKubenetConfigString,
+			overrides: []func(*ContainerdConfig) error{
+				ContainerdKubenetOverride,
+			},
+		},
+		{
+			name: "container sandbox image config",
+			want: containerdImageConfigString,
+			overrides: []func(*ContainerdConfig) error{
+				ContainerdSandboxImageOverrider("foobar"),
+			},
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+			got, err := GetContainerdConfig(test.options, test.overrides)
+			if err != nil {
+				t.Fatalf("failed to get docker config: %v", err)
+			}
+			diff := cmp.Diff(test.want, got)
+			if diff != "" {
+				t.Fatalf(diff)
+			}
+		})
+	}
+}
+
+var defaultContainerdConfigString = `oom_score = 0
+subreaper = false
+version = 2
+
+[plugins]
+  [plugins."io.containerd.grpc.v1.cri"]
+    [plugins."io.containerd.grpc.v1.cri".cni]
+    [plugins."io.containerd.grpc.v1.cri".containerd]
+      default_runtime_name = "runc"
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          runtime_type = "io.containerd.runc.v2"
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.untrusted]
+          runtime_type = "io.containerd.runc.v2"
+`
+
+var containerdRerootConfigString = `oom_score = 0
+root = "/mnt/containerd"
+subreaper = false
+version = 2
+
+[plugins]
+  [plugins."io.containerd.grpc.v1.cri"]
+    [plugins."io.containerd.grpc.v1.cri".cni]
+    [plugins."io.containerd.grpc.v1.cri".containerd]
+      default_runtime_name = "runc"
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          runtime_type = "io.containerd.runc.v2"
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.untrusted]
+          runtime_type = "io.containerd.runc.v2"
+`
+
+var containerdKubenetConfigString = `oom_score = 0
+subreaper = false
+version = 2
+
+[plugins]
+  [plugins."io.containerd.grpc.v1.cri"]
+    [plugins."io.containerd.grpc.v1.cri".cni]
+      conf_template = "/etc/containerd/kubenet_template.conf"
+    [plugins."io.containerd.grpc.v1.cri".containerd]
+      default_runtime_name = "runc"
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          runtime_type = "io.containerd.runc.v2"
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.untrusted]
+          runtime_type = "io.containerd.runc.v2"
+`
+
+var containerdImageConfigString = `oom_score = 0
+subreaper = false
+version = 2
+
+[plugins]
+  [plugins."io.containerd.grpc.v1.cri"]
+    sandbox_image = "foobar"
+    [plugins."io.containerd.grpc.v1.cri".cni]
+    [plugins."io.containerd.grpc.v1.cri".containerd]
+      default_runtime_name = "runc"
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          runtime_type = "io.containerd.runc.v2"
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.untrusted]
+          runtime_type = "io.containerd.runc.v2"
+`
+
+var defaultDockerConfigString = `{
+    "live-restore": true,
+    "log-driver": "json-file",
+    "log-opts": {
+        "max-size": "50m",
+        "max-file": "5"
+    }
+}`
+
+var dockerRerootConfigString = `{
+    "data-root": "/mnt/docker",
+    "live-restore": true,
+    "log-driver": "json-file",
+    "log-opts": {
+        "max-size": "50m",
+        "max-file": "5"
+    }
+}`
+
+var dockerNvidiaConfigString = `{
+    "live-restore": true,
+    "log-driver": "json-file",
+    "log-opts": {
+        "max-size": "50m",
+        "max-file": "5"
+    },
+    "default-runtime": "nvidia",
+    "runtimes": {
+        "nvidia": {
+            "path": "/usr/bin/nvidia-container-runtime",
+            "runtimeArgs": []
+        }
+    }
+}`

pkg/api/common/types.go

@@ -1,3 +1,6 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
 package common
 
 type ContainerdConfig struct {
@@ -47,6 +50,6 @@ type LogOpts struct {
 }
 
 type DockerDaemonRuntime struct {
-	Path        string        `json:"path,omitempty"`
-	RuntimeArgs []interface{} `json:"runtimeArgs,omitempty"`
+	Path        string   `json:"path,omitempty"`
+	RuntimeArgs []string `json:"runtimeArgs"`
 }

pkg/api/converterfromapi_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/Azure/go-autorest/autorest/to"
 
+	"github.com/Azure/aks-engine/pkg/api/common"
 	"github.com/Azure/aks-engine/pkg/api/vlabs"
 	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/google/go-cmp/cmp"
@@ -507,11 +508,14 @@ func getDefaultContainerService() *ContainerService {
 					ResourceGroup: "FooImageRefResourceGroup",
 				},
 				KubernetesConfig: &KubernetesConfig{
-					KubernetesImageBase:             "quay.io",
-					ClusterSubnet:                   "fooClusterSubnet",
-					NetworkPolicy:                   "calico",
-					NetworkPlugin:                   "azure-cni",
-					ContainerRuntime:                "docker",
+					KubernetesImageBase: "quay.io",
+					ClusterSubnet:       "fooClusterSubnet",
+					NetworkPolicy:       "calico",
+					NetworkPlugin:       "azure-cni",
+					ContainerRuntime:    "docker",
+					ContainerRuntimeConfig: map[string]string{
+						common.ContainerDataDirKey: "/mnt/docker",
+					},
 					MaxPods:                         3,
 					DockerBridgeSubnet:              "sampleDockerSubnet",
 					DNSServiceIP:                    "172.0.0.1",