diff --git a/cluster/gce/configure.sh b/cluster/gce/configure.sh index b616c25a6..d069dc446 100755 --- a/cluster/gce/configure.sh +++ b/cluster/gce/configure.sh @@ -176,9 +176,11 @@ disabled_plugins = ["restart"] conf_template = "${cni_template_path}" [plugins.cri.registry.mirrors."docker.io"] endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"] -[plugins.cri.containerd.default_runtime] +[plugins.cri.containerd] + default_runtime_name = "${CONTAINERD_DEFAULT_RUNTIME:-"runc"}" +[plugins.cri.containerd.runtimes.runc] runtime_type = "io.containerd.runc.v1" -[plugins.cri.containerd.default_runtime.options] +[plugins.cri.containerd.runtimes.runc.options] BinaryName = "${CONTAINERD_HOME}/usr/local/sbin/runc" EOF chmod 644 "${config_path}" diff --git a/cri.go b/cri.go index 96486da40..e14324895 100644 --- a/cri.go +++ b/cri.go @@ -17,6 +17,7 @@ limitations under the License. package cri import ( + "context" "flag" "path/filepath" "time" @@ -73,7 +74,7 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) { } log.G(ctx).Infof("Start cri plugin with config %+v", c) - if err := validateConfig(&c); err != nil { + if err := validateConfig(ctx, &c); err != nil { return nil, errors.Wrap(err, "invalid config") } @@ -111,14 +112,36 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) { } // validateConfig validates the given configuration. -func validateConfig(c *criconfig.Config) error { - // It is an error to provide both an UntrustedWorkloadRuntime & define an 'untrusted' runtime. - if _, ok := c.ContainerdConfig.Runtimes[criconfig.RuntimeUntrusted]; ok { - if c.ContainerdConfig.UntrustedWorkloadRuntime.Type != "" { - return errors.New("conflicting definitions: configuration includes untrusted_workload_runtime and runtimes['untrusted']") +func validateConfig(ctx context.Context, c *criconfig.Config) error { + if c.ContainerdConfig.Runtimes == nil { + c.ContainerdConfig.Runtimes = make(map[string]criconfig.Runtime) + } + + // Validation for deprecated untrusted_workload_runtime. + if c.ContainerdConfig.UntrustedWorkloadRuntime.Type != "" { + log.G(ctx).Warning("`untrusted_workload_runtime` is deprecated, please use `untrusted` runtime in `runtimes` instead") + if _, ok := c.ContainerdConfig.Runtimes[criconfig.RuntimeUntrusted]; ok { + return errors.Errorf("conflicting definitions: configuration includes both `untrusted_workload_runtime` and `runtimes[%q]`", criconfig.RuntimeUntrusted) } + c.ContainerdConfig.Runtimes[criconfig.RuntimeUntrusted] = c.ContainerdConfig.UntrustedWorkloadRuntime + } + + // Validation for deprecated default_runtime field. + if c.ContainerdConfig.DefaultRuntime.Type != "" { + log.G(ctx).Warning("`default_runtime` is deprecated, please use `default_runtime_name` to reference the default configuration you have defined in `runtimes`") + c.ContainerdConfig.DefaultRuntimeName = criconfig.RuntimeDefault + c.ContainerdConfig.Runtimes[criconfig.RuntimeDefault] = c.ContainerdConfig.DefaultRuntime + } + + // Validation for default_runtime_name + if c.ContainerdConfig.DefaultRuntimeName == "" { + return errors.New("`default_runtime_name` is empty") + } + if _, ok := c.ContainerdConfig.Runtimes[c.ContainerdConfig.DefaultRuntimeName]; !ok { + return errors.New("no corresponding runtime configured in `runtimes` for `default_runtime_name`") } + // Validation for stream_idle_timeout if c.StreamIdleTimeout != "" { if _, err := time.ParseDuration(c.StreamIdleTimeout); err != nil { return errors.Wrap(err, "invalid stream idle timeout") diff --git a/docs/config.md b/docs/config.md index 368684cf5..f6da837e3 100644 --- a/docs/config.md +++ b/docs/config.md @@ -80,60 +80,22 @@ The explanation and default value of each configuration item are as follows: # For runtime "io.containerd.runc.v1", use the option `NoPivotRoot`. no_pivot = false + # default_runtime_name is the default runtime name to use. + default_runtime_name = "runc" + # "plugins.cri.containerd.default_runtime" is the runtime to use in containerd. + # DEPRECATED: use `default_runtime_name` and `plugins.cri.runtimes` instead. + # Remove in containerd 1.4. [plugins.cri.containerd.default_runtime] - # runtime_type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux - runtime_type = "io.containerd.runtime.v1.linux" - - # runtime_engine is the name of the runtime engine used by containerd. - # This only works for runtime type "io.containerd.runtime.v1.linux". - # DEPRECATED: use Runtime.Options for runtime specific config for shim v2 runtimes. - # For runtime "io.containerd.runc.v1", use the option `BinaryName`. - runtime_engine = "" - - # runtime_root is the directory used by containerd for runtime state. - # This only works for runtime type "io.containerd.runtime.v1.linux". - # DEPRECATED: use Runtime.Options for runtime specific config for shim v2 runtimes. - # For runtime "io.containerd.runc.v1", use the option `Root`. - runtime_root = "" - - # "plugins.cri.containerd.default_runtime.options" is options specific to - # the default runtime. The options type for "io.containerd.runtime.v1.linux" is: - # https://github.com/containerd/containerd/blob/v1.2.0-rc.1/runtime/linux/runctypes/runc.pb.go#L40 - # NOTE: when `options` is specified, all related deprecated options will - # be ignored, including `systemd_cgroup`, `no_pivot`, `runtime_engine` - # and `runtime_root`. - [plugins.cri.containerd.default_runtime.options] - # Runtime is the binary name of the runtime. - Runtime = "" - - # RuntimeRoot is the root directory of the runtime. - RuntimeRoot = "" - - # CriuPath is the criu binary path. - CriuPath = "" - - # SystemdCgroup enables systemd cgroups. - SystemdCgroup = false # "plugins.cri.containerd.untrusted_workload_runtime" is a runtime to run untrusted workloads on it. - # DEPRECATED: use plugins.cri.runtimes instead. If provided, this runtime is mapped to the - # runtime handler named 'untrusted'. It is a configuration error to provide both the (now - # deprecated) UntrustedWorkloadRuntime and a handler in the Runtimes handler map (below) for - # 'untrusted' workloads at the same time. Please provide one or the other. + # DEPRECATED: use `untrusted` runtime in `plugins.cri.runtimes` instead. + # Remove in containerd 1.4. [plugins.cri.containerd.untrusted_workload_runtime] - # runtime_type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux - runtime_type = "" - - # runtime_engine is the name of the runtime engine used by containerd. - runtime_engine = "" - - # runtime_root is the directory used by containerd for runtime state. - runtime_root = "" # plugins.cri.containerd.runtimes is a map from CRI RuntimeHandler strings, which specify types - # of runtime configurations, to the matching configurations. In this example, - # 'runc' is the RuntimeHandler string to match. + # of runtime configurations, to the matching configurations. + # In this example, 'runc' is the RuntimeHandler string to match. [plugins.cri.containerd.runtimes.runc] # runtime_type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux runtime_type = "io.containerd.runc.v1" @@ -205,3 +167,25 @@ The explanation and default value of each configuration item are as follows: [plugins.cri.registry.mirrors."docker.io"] endpoint = ["https://registry-1.docker.io", ] ``` + +## Untrusted Workload + +The recommended way to run untrusted workload is to use +[`RuntimeClass`](https://kubernetes.io/docs/concepts/containers/runtime-class/) api +introduced in Kubernetes 1.12 to select RuntimeHandlers configured to run +untrusted workload in `plugins.cri.containerd.runtimes`. + +However, if you are using the legacy `io.kubernetes.cri.untrusted-workload`pod annotation +to request a pod be run using a runtime for untrusted workloads, the RuntimeHandler +`plugins.cri.containerd.runtimes.untrusted` must be defined first. When the annotation +`io.kubernetes.cri.untrusted-workload` is set to `true` the `untrusted` runtime will be +used. For example, see +[Create an untrusted pod using Kata Containers](https://github.com/kata-containers/documentation/blob/master/how-to/how-to-use-k8s-with-cri-containerd-and-kata.md#create-an-untrusted-pod-using-kata-containers). + +## Deprecation +The config options of the CRI plugin follow the [Kubernetes deprecation +policy of "admin-facing CLI components"](https://kubernetes.io/docs/reference/using-api/deprecation-policy/#deprecating-a-flag-or-cli). + +In summary, when a config option is announced to be deprecated: +* It is kept functional for 6 months or 1 release (whichever is longer); +* A warning is emitted when it is used. diff --git a/pkg/config/config.go b/pkg/config/config.go index 3e7a2aedf..85cfa2aa2 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -47,14 +47,14 @@ type Runtime struct { type ContainerdConfig struct { // Snapshotter is the snapshotter used by containerd. Snapshotter string `toml:"snapshotter" json:"snapshotter"` + // DefaultRuntimeName is the default runtime name to use from the runtimes table. + DefaultRuntimeName string `toml:"default_runtime_name" json:"defaultRuntimeName"` // DefaultRuntime is the default runtime to use in containerd. // This runtime is used when no runtime handler (or the empty string) is provided. + // DEPRECATED: use DefaultRuntimeName instead. Remove in containerd 1.4. DefaultRuntime Runtime `toml:"default_runtime" json:"defaultRuntime"` // UntrustedWorkloadRuntime is a runtime to run untrusted workloads on it. - // DEPRECATED: use Runtimes instead. If provided, this runtime is mapped to the runtime handler - // named 'untrusted'. It is a configuration error to provide both the (now deprecated) - // UntrustedWorkloadRuntime and a handler in the Runtimes handler map (below) for 'untrusted' - // workloads at the same time. Please provide one or the other. + // DEPRECATED: use `untrusted` runtime in Runtimes instead. Remove in containerd 1.4. UntrustedWorkloadRuntime Runtime `toml:"untrusted_workload_runtime" json:"untrustedWorkloadRuntime"` // Runtimes is a map from CRI RuntimeHandler strings, which specify types of runtime // configurations, to the matching configurations. @@ -195,13 +195,14 @@ func DefaultConfig() PluginConfig { NetworkPluginConfTemplate: "", }, ContainerdConfig: ContainerdConfig{ - Snapshotter: containerd.DefaultSnapshotter, - DefaultRuntime: Runtime{ - Type: "io.containerd.runtime.v1.linux", - Engine: "", - Root: "", + Snapshotter: containerd.DefaultSnapshotter, + DefaultRuntimeName: "runc", + NoPivot: false, + Runtimes: map[string]Runtime{ + "runc": { + Type: "io.containerd.runc.v1", + }, }, - NoPivot: false, }, StreamServerAddress: "127.0.0.1", StreamServerPort: "0", @@ -229,4 +230,6 @@ func DefaultConfig() PluginConfig { const ( // RuntimeUntrusted is the implicit runtime defined for ContainerdConfig.UntrustedWorkloadRuntime RuntimeUntrusted = "untrusted" + // RuntimeDefault is the implicit runtime defined for ContainerdConfig.DefaultRuntime + RuntimeDefault = "default" ) diff --git a/pkg/server/helpers_test.go b/pkg/server/helpers_test.go index 2a137be93..18a3a95c1 100644 --- a/pkg/server/helpers_test.go +++ b/pkg/server/helpers_test.go @@ -212,7 +212,8 @@ func TestGenerateRuntimeOptions(t *testing.T) { systemd_cgroup = true [containerd] no_pivot = true -[containerd.default_runtime] + default_runtime_name = "default" +[containerd.runtimes.legacy] runtime_type = "` + linuxRuntime + `" [containerd.runtimes.runc] runtime_type = "` + runcRuntimeV1 + `" @@ -223,11 +224,12 @@ systemd_cgroup = true systemd_cgroup = true [containerd] no_pivot = true -[containerd.default_runtime] + default_runtime_name = "default" +[containerd.runtimes.legacy] runtime_type = "` + linuxRuntime + `" -[containerd.default_runtime.options] - Runtime = "default" - RuntimeRoot = "/default" +[containerd.runtimes.legacy.options] + Runtime = "legacy" + RuntimeRoot = "/legacy" [containerd.runtimes.runc] runtime_type = "` + runcRuntimeV1 + `" [containerd.runtimes.runc.options] @@ -246,8 +248,8 @@ systemd_cgroup = true require.NoError(t, err) _, err = toml.Decode(nonNilOpts, &nonNilOptsConfig) require.NoError(t, err) - require.Len(t, nilOptsConfig.Runtimes, 2) - require.Len(t, nonNilOptsConfig.Runtimes, 2) + require.Len(t, nilOptsConfig.Runtimes, 3) + require.Len(t, nonNilOptsConfig.Runtimes, 3) for desc, test := range map[string]struct { r criconfig.Runtime @@ -265,7 +267,7 @@ systemd_cgroup = true expectedOptions: nil, }, "when options is nil, should use legacy fields for legacy runtime": { - r: nilOptsConfig.DefaultRuntime, + r: nilOptsConfig.Runtimes["legacy"], c: nilOptsConfig, expectedOptions: &runctypes.RuncOptions{ SystemdCgroup: true, @@ -290,11 +292,11 @@ systemd_cgroup = true }, }, "when options is not nil, should be able to decode for legacy runtime": { - r: nonNilOptsConfig.DefaultRuntime, + r: nonNilOptsConfig.Runtimes["legacy"], c: nonNilOptsConfig, expectedOptions: &runctypes.RuncOptions{ - Runtime: "default", - RuntimeRoot: "/default", + Runtime: "legacy", + RuntimeRoot: "/legacy", }, }, } { diff --git a/pkg/server/sandbox_run.go b/pkg/server/sandbox_run.go index 3aeeda3a8..328edd716 100644 --- a/pkg/server/sandbox_run.go +++ b/pkg/server/sandbox_run.go @@ -628,16 +628,11 @@ func (c *criService) getSandboxRuntime(config *runtime.PodSandboxConfig, runtime return criconfig.Runtime{}, errors.New("untrusted workload with host access is not allowed") } - // Handle the deprecated UntrustedWorkloadRuntime. - if c.config.ContainerdConfig.UntrustedWorkloadRuntime.Type != "" { - return c.config.ContainerdConfig.UntrustedWorkloadRuntime, nil - } - runtimeHandler = criconfig.RuntimeUntrusted } if runtimeHandler == "" { - return c.config.ContainerdConfig.DefaultRuntime, nil + runtimeHandler = c.config.ContainerdConfig.DefaultRuntimeName } handler, ok := c.config.ContainerdConfig.Runtimes[runtimeHandler] diff --git a/pkg/server/sandbox_run_test.go b/pkg/server/sandbox_run_test.go index 244a94a1c..ff97ed25c 100644 --- a/pkg/server/sandbox_run_test.go +++ b/pkg/server/sandbox_run_test.go @@ -681,13 +681,11 @@ func TestGetSandboxRuntime(t *testing.T) { } for desc, test := range map[string]struct { - sandboxConfig *runtime.PodSandboxConfig - runtimeHandler string - defaultRuntime criconfig.Runtime - untrustedWorkloadRuntime criconfig.Runtime - runtimes map[string]criconfig.Runtime - expectErr bool - expectedRuntime criconfig.Runtime + sandboxConfig *runtime.PodSandboxConfig + runtimeHandler string + runtimes map[string]criconfig.Runtime + expectErr bool + expectedRuntime criconfig.Runtime }{ "should return error if untrusted workload requires host access": { sandboxConfig: &runtime.PodSandboxConfig{ @@ -705,9 +703,11 @@ func TestGetSandboxRuntime(t *testing.T) { annotations.UntrustedWorkload: "true", }, }, - defaultRuntime: defaultRuntime, - untrustedWorkloadRuntime: untrustedWorkloadRuntime, - expectErr: true, + runtimes: map[string]criconfig.Runtime{ + criconfig.RuntimeDefault: defaultRuntime, + criconfig.RuntimeUntrusted: untrustedWorkloadRuntime, + }, + expectErr: true, }, "should use untrusted workload runtime for untrusted workload": { sandboxConfig: &runtime.PodSandboxConfig{ @@ -715,15 +715,18 @@ func TestGetSandboxRuntime(t *testing.T) { annotations.UntrustedWorkload: "true", }, }, - defaultRuntime: defaultRuntime, - untrustedWorkloadRuntime: untrustedWorkloadRuntime, - expectedRuntime: untrustedWorkloadRuntime, + runtimes: map[string]criconfig.Runtime{ + criconfig.RuntimeDefault: defaultRuntime, + criconfig.RuntimeUntrusted: untrustedWorkloadRuntime, + }, + expectedRuntime: untrustedWorkloadRuntime, }, "should use default runtime for regular workload": { - sandboxConfig: &runtime.PodSandboxConfig{}, - defaultRuntime: defaultRuntime, - untrustedWorkloadRuntime: untrustedWorkloadRuntime, - expectedRuntime: defaultRuntime, + sandboxConfig: &runtime.PodSandboxConfig{}, + runtimes: map[string]criconfig.Runtime{ + criconfig.RuntimeDefault: defaultRuntime, + }, + expectedRuntime: defaultRuntime, }, "should use default runtime for trusted workload": { sandboxConfig: &runtime.PodSandboxConfig{ @@ -731,9 +734,11 @@ func TestGetSandboxRuntime(t *testing.T) { annotations.UntrustedWorkload: "false", }, }, - defaultRuntime: defaultRuntime, - untrustedWorkloadRuntime: untrustedWorkloadRuntime, - expectedRuntime: defaultRuntime, + runtimes: map[string]criconfig.Runtime{ + criconfig.RuntimeDefault: defaultRuntime, + criconfig.RuntimeUntrusted: untrustedWorkloadRuntime, + }, + expectedRuntime: defaultRuntime, }, "should return error if untrusted workload runtime is required but not configured": { sandboxConfig: &runtime.PodSandboxConfig{ @@ -741,8 +746,10 @@ func TestGetSandboxRuntime(t *testing.T) { annotations.UntrustedWorkload: "true", }, }, - defaultRuntime: defaultRuntime, - expectErr: true, + runtimes: map[string]criconfig.Runtime{ + criconfig.RuntimeDefault: defaultRuntime, + }, + expectErr: true, }, "should use 'untrusted' runtime for untrusted workload": { sandboxConfig: &runtime.PodSandboxConfig{ @@ -750,8 +757,10 @@ func TestGetSandboxRuntime(t *testing.T) { annotations.UntrustedWorkload: "true", }, }, - defaultRuntime: defaultRuntime, - runtimes: map[string]criconfig.Runtime{criconfig.RuntimeUntrusted: untrustedWorkloadRuntime}, + runtimes: map[string]criconfig.Runtime{ + criconfig.RuntimeDefault: defaultRuntime, + criconfig.RuntimeUntrusted: untrustedWorkloadRuntime, + }, expectedRuntime: untrustedWorkloadRuntime, }, "should use 'untrusted' runtime for untrusted workload & handler": { @@ -760,9 +769,11 @@ func TestGetSandboxRuntime(t *testing.T) { annotations.UntrustedWorkload: "true", }, }, - runtimeHandler: "untrusted", - defaultRuntime: defaultRuntime, - runtimes: map[string]criconfig.Runtime{criconfig.RuntimeUntrusted: untrustedWorkloadRuntime}, + runtimeHandler: "untrusted", + runtimes: map[string]criconfig.Runtime{ + criconfig.RuntimeDefault: defaultRuntime, + criconfig.RuntimeUntrusted: untrustedWorkloadRuntime, + }, expectedRuntime: untrustedWorkloadRuntime, }, "should return an error if untrusted annotation with conflicting handler": { @@ -771,26 +782,32 @@ func TestGetSandboxRuntime(t *testing.T) { annotations.UntrustedWorkload: "true", }, }, - runtimeHandler: "foo", - defaultRuntime: defaultRuntime, - untrustedWorkloadRuntime: untrustedWorkloadRuntime, - runtimes: map[string]criconfig.Runtime{"foo": fooRuntime}, - expectErr: true, + runtimeHandler: "foo", + runtimes: map[string]criconfig.Runtime{ + criconfig.RuntimeDefault: defaultRuntime, + criconfig.RuntimeUntrusted: untrustedWorkloadRuntime, + "foo": fooRuntime, + }, + expectErr: true, }, "should use correct runtime for a runtime handler": { - sandboxConfig: &runtime.PodSandboxConfig{}, - runtimeHandler: "foo", - defaultRuntime: defaultRuntime, - untrustedWorkloadRuntime: untrustedWorkloadRuntime, - runtimes: map[string]criconfig.Runtime{"foo": fooRuntime}, - expectedRuntime: fooRuntime, + sandboxConfig: &runtime.PodSandboxConfig{}, + runtimeHandler: "foo", + runtimes: map[string]criconfig.Runtime{ + criconfig.RuntimeDefault: defaultRuntime, + criconfig.RuntimeUntrusted: untrustedWorkloadRuntime, + "foo": fooRuntime, + }, + expectedRuntime: fooRuntime, }, "should return error if runtime handler is required but not configured": { sandboxConfig: &runtime.PodSandboxConfig{}, runtimeHandler: "bar", - defaultRuntime: defaultRuntime, - runtimes: map[string]criconfig.Runtime{"foo": fooRuntime}, - expectErr: true, + runtimes: map[string]criconfig.Runtime{ + criconfig.RuntimeDefault: defaultRuntime, + "foo": fooRuntime, + }, + expectErr: true, }, } { t.Run(desc, func(t *testing.T) { @@ -798,8 +815,7 @@ func TestGetSandboxRuntime(t *testing.T) { cri.config = criconfig.Config{ PluginConfig: criconfig.DefaultConfig(), } - cri.config.ContainerdConfig.DefaultRuntime = test.defaultRuntime - cri.config.ContainerdConfig.UntrustedWorkloadRuntime = test.untrustedWorkloadRuntime + cri.config.ContainerdConfig.DefaultRuntimeName = criconfig.RuntimeDefault cri.config.ContainerdConfig.Runtimes = test.runtimes r, err := cri.getSandboxRuntime(test.sandboxConfig, test.runtimeHandler) assert.Equal(t, test.expectErr, err != nil)