Merge pull request #1125 from Random-Liu/add-default-runtime-name

Add default runtime name

cluster/gce/configure.sh

@@ -176,9 +176,11 @@ disabled_plugins = ["restart"]
   conf_template = "${cni_template_path}"
 [plugins.cri.registry.mirrors."docker.io"]
   endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
-[plugins.cri.containerd.default_runtime]
+[plugins.cri.containerd]
+  default_runtime_name = "${CONTAINERD_DEFAULT_RUNTIME:-"runc"}"
+[plugins.cri.containerd.runtimes.runc]
   runtime_type = "io.containerd.runc.v1"
-[plugins.cri.containerd.default_runtime.options]
+[plugins.cri.containerd.runtimes.runc.options]
   BinaryName = "${CONTAINERD_HOME}/usr/local/sbin/runc"
 EOF
 chmod 644 "${config_path}"

cri.go

@@ -17,6 +17,7 @@ limitations under the License.
 package cri
 
 import (
+	"context"
 	"flag"
 	"path/filepath"
 	"time"
@@ -73,7 +74,7 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) {
 	}
 	log.G(ctx).Infof("Start cri plugin with config %+v", c)
 
-	if err := validateConfig(&c); err != nil {
+	if err := validateConfig(ctx, &c); err != nil {
 		return nil, errors.Wrap(err, "invalid config")
 	}
 
@@ -111,14 +112,36 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) {
 }
 
 // validateConfig validates the given configuration.
-func validateConfig(c *criconfig.Config) error {
-	// It is an error to provide both an UntrustedWorkloadRuntime & define an 'untrusted' runtime.
-	if _, ok := c.ContainerdConfig.Runtimes[criconfig.RuntimeUntrusted]; ok {
-		if c.ContainerdConfig.UntrustedWorkloadRuntime.Type != "" {
-			return errors.New("conflicting definitions: configuration includes untrusted_workload_runtime and runtimes['untrusted']")
+func validateConfig(ctx context.Context, c *criconfig.Config) error {
+	if c.ContainerdConfig.Runtimes == nil {
+		c.ContainerdConfig.Runtimes = make(map[string]criconfig.Runtime)
+	}
+
+	// Validation for deprecated untrusted_workload_runtime.
+	if c.ContainerdConfig.UntrustedWorkloadRuntime.Type != "" {
+		log.G(ctx).Warning("`untrusted_workload_runtime` is deprecated, please use `untrusted` runtime in `runtimes` instead")
+		if _, ok := c.ContainerdConfig.Runtimes[criconfig.RuntimeUntrusted]; ok {
+			return errors.Errorf("conflicting definitions: configuration includes both `untrusted_workload_runtime` and `runtimes[%q]`", criconfig.RuntimeUntrusted)
 		}
+		c.ContainerdConfig.Runtimes[criconfig.RuntimeUntrusted] = c.ContainerdConfig.UntrustedWorkloadRuntime
+	}
+
+	// Validation for deprecated default_runtime field.
+	if c.ContainerdConfig.DefaultRuntime.Type != "" {
+		log.G(ctx).Warning("`default_runtime` is deprecated, please use `default_runtime_name` to reference the default configuration you have defined in `runtimes`")
+		c.ContainerdConfig.DefaultRuntimeName = criconfig.RuntimeDefault
+		c.ContainerdConfig.Runtimes[criconfig.RuntimeDefault] = c.ContainerdConfig.DefaultRuntime
+	}
+
+	// Validation for default_runtime_name
+	if c.ContainerdConfig.DefaultRuntimeName == "" {
+		return errors.New("`default_runtime_name` is empty")
+	}
+	if _, ok := c.ContainerdConfig.Runtimes[c.ContainerdConfig.DefaultRuntimeName]; !ok {
+		return errors.New("no corresponding runtime configured in `runtimes` for `default_runtime_name`")
 	}
 
+	// Validation for stream_idle_timeout
 	if c.StreamIdleTimeout != "" {
 		if _, err := time.ParseDuration(c.StreamIdleTimeout); err != nil {
 			return errors.Wrap(err, "invalid stream idle timeout")

docs/config.md

@@ -80,60 +80,22 @@ The explanation and default value of each configuration item are as follows:
     #   For runtime "io.containerd.runc.v1", use the option `NoPivotRoot`.
     no_pivot = false
 
+    # default_runtime_name is the default runtime name to use.
+    default_runtime_name = "runc"
+
     # "plugins.cri.containerd.default_runtime" is the runtime to use in containerd.
+    # DEPRECATED: use `default_runtime_name` and `plugins.cri.runtimes` instead.
+    # Remove in containerd 1.4.
     [plugins.cri.containerd.default_runtime]
-      # runtime_type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux
-      runtime_type = "io.containerd.runtime.v1.linux"
-
-      # runtime_engine is the name of the runtime engine used by containerd.
-      # This only works for runtime type "io.containerd.runtime.v1.linux".
-      # DEPRECATED: use Runtime.Options for runtime specific config for shim v2 runtimes.
-      #   For runtime "io.containerd.runc.v1", use the option `BinaryName`.
-      runtime_engine = ""
-
-      # runtime_root is the directory used by containerd for runtime state.
-      # This only works for runtime type "io.containerd.runtime.v1.linux".
-      # DEPRECATED: use Runtime.Options for runtime specific config for shim v2 runtimes.
-      #   For runtime "io.containerd.runc.v1", use the option `Root`.
-      runtime_root = ""
-
-      # "plugins.cri.containerd.default_runtime.options" is options specific to
-      # the default runtime. The options type for "io.containerd.runtime.v1.linux" is:
-      #   https://github.com/containerd/containerd/blob/v1.2.0-rc.1/runtime/linux/runctypes/runc.pb.go#L40
-      # NOTE: when `options` is specified, all related deprecated options will
-      #   be ignored, including `systemd_cgroup`, `no_pivot`, `runtime_engine`
-      #   and `runtime_root`.
-      [plugins.cri.containerd.default_runtime.options]
-        # Runtime is the binary name of the runtime.
-        Runtime = ""
-
-        # RuntimeRoot is the root directory of the runtime.
-        RuntimeRoot = ""
-
-        # CriuPath is the criu binary path.
-        CriuPath = ""
-
-        # SystemdCgroup enables systemd cgroups.
-        SystemdCgroup = false
 
     # "plugins.cri.containerd.untrusted_workload_runtime" is a runtime to run untrusted workloads on it.
-    # DEPRECATED: use plugins.cri.runtimes instead. If provided, this runtime is mapped to the
-    #   runtime handler named 'untrusted'. It is a configuration error to provide both the (now
-    #   deprecated) UntrustedWorkloadRuntime and a handler in the Runtimes handler map (below) for
-    #   'untrusted' workloads at the same time. Please provide one or the other.
+    # DEPRECATED: use `untrusted` runtime in `plugins.cri.runtimes` instead.
+    # Remove in containerd 1.4.
     [plugins.cri.containerd.untrusted_workload_runtime]
-      # runtime_type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux
-      runtime_type = ""
-
-      # runtime_engine is the name of the runtime engine used by containerd.
-      runtime_engine = ""
-
-      # runtime_root is the directory used by containerd for runtime state.
-      runtime_root = ""
 
     # plugins.cri.containerd.runtimes is a map from CRI RuntimeHandler strings, which specify types
-    # of runtime configurations, to the matching configurations. In this example,
-    # 'runc' is the RuntimeHandler string to match.
+    # of runtime configurations, to the matching configurations.
+    # In this example, 'runc' is the RuntimeHandler string to match.
     [plugins.cri.containerd.runtimes.runc]
       # runtime_type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux
       runtime_type = "io.containerd.runc.v1"
@@ -205,3 +167,25 @@ The explanation and default value of each configuration item are as follows:
       [plugins.cri.registry.mirrors."docker.io"]
         endpoint = ["https://registry-1.docker.io", ]
 ```
+
+## Untrusted Workload
+
+The recommended way to run untrusted workload is to use
+[`RuntimeClass`](https://kubernetes.io/docs/concepts/containers/runtime-class/) api
+introduced in Kubernetes 1.12 to select RuntimeHandlers configured to run
+untrusted workload in `plugins.cri.containerd.runtimes`.
+
+However, if you are using the legacy `io.kubernetes.cri.untrusted-workload`pod annotation
+to request a pod be run using a runtime for untrusted workloads, the RuntimeHandler
+`plugins.cri.containerd.runtimes.untrusted` must be defined first. When the annotation
+`io.kubernetes.cri.untrusted-workload` is set to `true` the `untrusted` runtime will be
+used. For example, see
+[Create an untrusted pod using Kata Containers](https://github.com/kata-containers/documentation/blob/master/how-to/how-to-use-k8s-with-cri-containerd-and-kata.md#create-an-untrusted-pod-using-kata-containers).
+
+## Deprecation
+The config options of the CRI plugin follow the [Kubernetes deprecation
+policy of "admin-facing CLI components"](https://kubernetes.io/docs/reference/using-api/deprecation-policy/#deprecating-a-flag-or-cli).
+
+In summary, when a config option is announced to be deprecated:
+* It is kept functional for 6 months or 1 release (whichever is longer);
+* A warning is emitted when it is used.

pkg/config/config.go

@@ -47,14 +47,14 @@ type Runtime struct {
 type ContainerdConfig struct {
 	// Snapshotter is the snapshotter used by containerd.
 	Snapshotter string `toml:"snapshotter" json:"snapshotter"`
+	// DefaultRuntimeName is the default runtime name to use from the runtimes table.
+	DefaultRuntimeName string `toml:"default_runtime_name" json:"defaultRuntimeName"`
 	// DefaultRuntime is the default runtime to use in containerd.
 	// This runtime is used when no runtime handler (or the empty string) is provided.
+	// DEPRECATED: use DefaultRuntimeName instead. Remove in containerd 1.4.
 	DefaultRuntime Runtime `toml:"default_runtime" json:"defaultRuntime"`
 	// UntrustedWorkloadRuntime is a runtime to run untrusted workloads on it.
-	// DEPRECATED: use Runtimes instead. If provided, this runtime is mapped to the runtime handler
-	//     named 'untrusted'. It is a configuration error to provide both the (now deprecated)
-	//     UntrustedWorkloadRuntime and a handler in the Runtimes handler map (below) for 'untrusted'
-	//     workloads at the same time. Please provide one or the other.
+	// DEPRECATED: use `untrusted` runtime in Runtimes instead. Remove in containerd 1.4.
 	UntrustedWorkloadRuntime Runtime `toml:"untrusted_workload_runtime" json:"untrustedWorkloadRuntime"`
 	// Runtimes is a map from CRI RuntimeHandler strings, which specify types of runtime
 	// configurations, to the matching configurations.
@@ -195,13 +195,14 @@ func DefaultConfig() PluginConfig {
 			NetworkPluginConfTemplate: "",
 		},
 		ContainerdConfig: ContainerdConfig{
-			Snapshotter: containerd.DefaultSnapshotter,
-			DefaultRuntime: Runtime{
-				Type:   "io.containerd.runtime.v1.linux",
-				Engine: "",
-				Root:   "",
+			Snapshotter:        containerd.DefaultSnapshotter,
+			DefaultRuntimeName: "runc",
+			NoPivot:            false,
+			Runtimes: map[string]Runtime{
+				"runc": {
+					Type: "io.containerd.runc.v1",
+				},
 			},
-			NoPivot: false,
 		},
 		StreamServerAddress: "127.0.0.1",
 		StreamServerPort:    "0",
@@ -229,4 +230,6 @@ func DefaultConfig() PluginConfig {
 const (
 	// RuntimeUntrusted is the implicit runtime defined for ContainerdConfig.UntrustedWorkloadRuntime
 	RuntimeUntrusted = "untrusted"
+	// RuntimeDefault is the implicit runtime defined for ContainerdConfig.DefaultRuntime
+	RuntimeDefault = "default"
 )

pkg/server/helpers_test.go

@@ -212,7 +212,8 @@ func TestGenerateRuntimeOptions(t *testing.T) {
 systemd_cgroup = true
 [containerd]
   no_pivot = true
-[containerd.default_runtime]
+  default_runtime_name = "default"
+[containerd.runtimes.legacy]
   runtime_type = "` + linuxRuntime + `"
 [containerd.runtimes.runc]
   runtime_type = "` + runcRuntimeV1 + `"
@@ -223,11 +224,12 @@ systemd_cgroup = true
 systemd_cgroup = true
 [containerd]
   no_pivot = true
-[containerd.default_runtime]
+  default_runtime_name = "default"
+[containerd.runtimes.legacy]
   runtime_type = "` + linuxRuntime + `"
-[containerd.default_runtime.options]
-  Runtime = "default"
-  RuntimeRoot = "/default"
+[containerd.runtimes.legacy.options]
+  Runtime = "legacy"
+  RuntimeRoot = "/legacy"
 [containerd.runtimes.runc]
   runtime_type = "` + runcRuntimeV1 + `"
 [containerd.runtimes.runc.options]
@@ -246,8 +248,8 @@ systemd_cgroup = true
 	require.NoError(t, err)
 	_, err = toml.Decode(nonNilOpts, &nonNilOptsConfig)
 	require.NoError(t, err)
-	require.Len(t, nilOptsConfig.Runtimes, 2)
-	require.Len(t, nonNilOptsConfig.Runtimes, 2)
+	require.Len(t, nilOptsConfig.Runtimes, 3)
+	require.Len(t, nonNilOptsConfig.Runtimes, 3)
 
 	for desc, test := range map[string]struct {
 		r               criconfig.Runtime
@@ -265,7 +267,7 @@ systemd_cgroup = true
 			expectedOptions: nil,
 		},
 		"when options is nil, should use legacy fields for legacy runtime": {
-			r: nilOptsConfig.DefaultRuntime,
+			r: nilOptsConfig.Runtimes["legacy"],
 			c: nilOptsConfig,
 			expectedOptions: &runctypes.RuncOptions{
 				SystemdCgroup: true,
@@ -290,11 +292,11 @@ systemd_cgroup = true
 			},
 		},
 		"when options is not nil, should be able to decode for legacy runtime": {
-			r: nonNilOptsConfig.DefaultRuntime,
+			r: nonNilOptsConfig.Runtimes["legacy"],
 			c: nonNilOptsConfig,
 			expectedOptions: &runctypes.RuncOptions{
-				Runtime:     "default",
-				RuntimeRoot: "/default",
+				Runtime:     "legacy",
+				RuntimeRoot: "/legacy",
 			},
 		},
 	} {

pkg/server/sandbox_run.go

@@ -628,16 +628,11 @@ func (c *criService) getSandboxRuntime(config *runtime.PodSandboxConfig, runtime
 			return criconfig.Runtime{}, errors.New("untrusted workload with host access is not allowed")
 		}
 
-		// Handle the deprecated UntrustedWorkloadRuntime.
-		if c.config.ContainerdConfig.UntrustedWorkloadRuntime.Type != "" {
-			return c.config.ContainerdConfig.UntrustedWorkloadRuntime, nil
-		}
-
 		runtimeHandler = criconfig.RuntimeUntrusted
 	}
 
 	if runtimeHandler == "" {
-		return c.config.ContainerdConfig.DefaultRuntime, nil
+		runtimeHandler = c.config.ContainerdConfig.DefaultRuntimeName
 	}
 
 	handler, ok := c.config.ContainerdConfig.Runtimes[runtimeHandler]