Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WIP: Security Context #6287

Closed
wants to merge 5 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions cmd/kube-apiserver/app/plugins.go
Expand Up @@ -36,4 +36,5 @@ import (
_ "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/namespace/exists"
_ "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/namespace/lifecycle"
_ "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/resourcequota"
_ "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/securitycontext"
)
4 changes: 2 additions & 2 deletions cmd/kube-apiserver/app/server.go
Expand Up @@ -33,10 +33,10 @@ import (
"github.com/GoogleCloudPlatform/kubernetes/pkg/admission"
"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
"github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver"
"github.com/GoogleCloudPlatform/kubernetes/pkg/capabilities"
"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
"github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider"
"github.com/GoogleCloudPlatform/kubernetes/pkg/master"
"github.com/GoogleCloudPlatform/kubernetes/pkg/securitycontext"
"github.com/GoogleCloudPlatform/kubernetes/pkg/tools"
"github.com/GoogleCloudPlatform/kubernetes/pkg/util"

Expand Down Expand Up @@ -203,7 +203,7 @@ func (s *APIServer) Run(_ []string) error {
glog.Fatalf("specify either --etcd_servers or --etcd_config")
}

capabilities.Initialize(capabilities.Capabilities{
securitycontext.Initialize(api.SecurityConstraints{
AllowPrivileged: s.AllowPrivileged,
// TODO(vmarmol): Implement support for HostNetworkSources.
HostNetworkSources: []string{},
Expand Down
54 changes: 51 additions & 3 deletions cmd/kubelet/app/server.go
Expand Up @@ -30,7 +30,6 @@ import (
"time"

"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
"github.com/GoogleCloudPlatform/kubernetes/pkg/capabilities"
"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
"github.com/GoogleCloudPlatform/kubernetes/pkg/client/chaosclient"
"github.com/GoogleCloudPlatform/kubernetes/pkg/client/record"
Expand All @@ -47,6 +46,7 @@ import (
"github.com/GoogleCloudPlatform/kubernetes/pkg/volume"

"github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider"
"github.com/GoogleCloudPlatform/kubernetes/pkg/securitycontext"
"github.com/golang/glog"
"github.com/spf13/pflag"
)
Expand Down Expand Up @@ -98,6 +98,7 @@ type KubeletServer struct {
CertDirectory string
NodeStatusUpdateFrequency time.Duration
ResourceContainer string
SecurityContextProvider string

// Flags intended for testing

Expand Down Expand Up @@ -201,6 +202,7 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
fs.StringVar(&s.CloudProvider, "cloud_provider", s.CloudProvider, "The provider for cloud services. Empty string for no provider.")
fs.StringVar(&s.CloudConfigFile, "cloud_config", s.CloudConfigFile, "The path to the cloud provider configuration file. Empty string for no configuration file.")
fs.StringVar(&s.ResourceContainer, "resource_container", s.ResourceContainer, "Absolute name of the resource-only container to create and run the Kubelet in (Default: /kubelet).")
fs.StringVar(&s.SecurityContextProvider, "security_context", s.SecurityContextProvider, "Name of the security context provider to use. Allowable values are <empty>|permit|restrict")

// Flags intended for testing, not recommended used in production environments.
fs.BoolVar(&s.ReallyCrashForTesting, "really_crash_for_testing", s.ReallyCrashForTesting, "If true, when panics occur crash. Intended for testing.")
Expand Down Expand Up @@ -302,6 +304,8 @@ func (s *KubeletServer) Run(_ []string) error {
ResourceContainer: s.ResourceContainer,
}

InitializeSecurityContextProvider(&kcfg, s.SecurityContextProvider)

RunKubelet(&kcfg, nil)

if s.HealthzPort > 0 {
Expand Down Expand Up @@ -406,6 +410,7 @@ func SimpleKubelet(client *client.Client,
Cloud: cloud,
NodeStatusUpdateFrequency: 10 * time.Second,
ResourceContainer: "/kubelet",
SecurityContextProvider: securitycontext.NewPermitSecurityContextProvider(securitycontext.Get()),
}
return &kcfg
}
Expand All @@ -415,6 +420,7 @@ func SimpleKubelet(client *client.Client,
// 2 Kubelet binary
// 3 Standalone 'kubernetes' binary
// Eventually, #2 will be replaced with instances of #3
// NOTE: please see the comments on InitializeSecurityContextProvider for important information about the SecurityContextProvider!!
func RunKubelet(kcfg *KubeletConfig, builder KubeletBuilder) {
kcfg.Hostname = util.GetHostname(kcfg.HostnameOverride)
eventBroadcaster := record.NewBroadcaster()
Expand All @@ -426,7 +432,10 @@ func RunKubelet(kcfg *KubeletConfig, builder KubeletBuilder) {
} else {
glog.Infof("No api server defined - no events will be sent to API server.")
}
capabilities.Setup(kcfg.AllowPrivileged, kcfg.HostNetworkSources)

//initialize the security context provider if it hasn't already happened
//NOTE: please see the comments on InitializeSecurityContextProvider!
InitializeSecurityContextProvider(kcfg, "")

credentialprovider.SetPreferredDockercfgPath(kcfg.RootDirectory)

Expand All @@ -449,6 +458,43 @@ func RunKubelet(kcfg *KubeletConfig, builder KubeletBuilder) {
glog.Infof("Started kubelet")
}

// InitializeSecurityContextProvider initializes a global security context and sets the security context provider.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: Suggest you wrap this text at no more than 100 chars.

// This should eventually be pulled as a resource but this is replacing the existing capabilities functionality for now.
// This method is called twice: once by Run and once by RunKubelet.
//
// Issue: the old global capabilities code has migrated into a security constraint. This is initialized only once
// for the system. We also want to pass that global context into the global provider until the security constraints
// are fetched at runtime based on the namespace. In order to have both items you MUST initialize the context and
// provider yourself before you use RunKubelet or SimpleKubelet since they set up a default security context provider
// to catch anything that is not set. (this is where the old capabilities code was originally called) unless you are ok
// with using the default permit provider which ignores the cap add/drop settings and priv container requests and
// allows them to be run.
//
// SimpleKubelet example:
// myKcfg := KubeletConfig{}
// InitializeSecurityContextProvider(myKcfg, type)
// simpleKubeletCfg := SimpleKubelet()
// simpleKubeletCfg.SecurityContextProvider = myKcfg.SecurityContextProvider
// RunKubelet example:
// myKcfg := KubeletConfig{}
// InitializeSecurityContextProvider(myKcfg, type)
// RunKubelet(myKcfg, ...)
//
//
// Using the Run method pre-initializes the provider base on the command line settings
func InitializeSecurityContextProvider(kcfg *KubeletConfig, providerType string) {
securitycontext.Setup(kcfg.AllowPrivileged, kcfg.HostNetworkSources)

if kcfg.SecurityContextProvider == nil {
switch providerType {
case "restrict":
kcfg.SecurityContextProvider = securitycontext.NewRestrictSecurityContextProvider(securitycontext.Get())
default:
kcfg.SecurityContextProvider = securitycontext.NewPermitSecurityContextProvider(securitycontext.Get())
}
}
}

func startKubelet(k KubeletBootstrap, podCfg *config.PodConfig, kc *KubeletConfig) {
// start the kubelet
go util.Forever(func() { k.Run(podCfg.Updates()) }, 0)
Expand Down Expand Up @@ -529,6 +575,7 @@ type KubeletConfig struct {
Cloud cloudprovider.Interface
NodeStatusUpdateFrequency time.Duration
ResourceContainer string
SecurityContextProvider securitycontext.SecurityContextProvider
}

func createAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.PodConfig, err error) {
Expand Down Expand Up @@ -572,7 +619,8 @@ func createAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.Pod
kc.ImageGCPolicy,
kc.Cloud,
kc.NodeStatusUpdateFrequency,
kc.ResourceContainer)
kc.ResourceContainer,
kc.SecurityContextProvider)

if err != nil {
return nil, nil, err
Expand Down
3 changes: 2 additions & 1 deletion hack/local-up-cluster.sh
Expand Up @@ -116,7 +116,7 @@ echo "Starting etcd"
kube::etcd::start

# Admission Controllers to invoke prior to persisting objects in cluster
ADMISSION_CONTROL=NamespaceLifecycle,NamespaceAutoProvision,LimitRanger,ResourceQuota
ADMISSION_CONTROL=NamespaceLifecycle,NamespaceAutoProvision,LimitRanger,ResourceQuota,SecurityContext

APISERVER_LOG=/tmp/kube-apiserver.log
sudo -E "${GO_OUT}/kube-apiserver" \
Expand Down Expand Up @@ -149,6 +149,7 @@ sudo -E "${GO_OUT}/kubelet" \
--address="127.0.0.1" \
--api_servers="${API_HOST}:${API_PORT}" \
--auth_path="${KUBE_ROOT}/hack/.test-cmd-auth" \
--security_context="restrict" \
--port="$KUBELET_PORT" >"${KUBELET_LOG}" 2>&1 &
KUBELET_PID=$!

Expand Down
110 changes: 106 additions & 4 deletions pkg/api/types.go
Expand Up @@ -599,12 +599,10 @@ type Container struct {
Lifecycle *Lifecycle `json:"lifecycle,omitempty"`
// Required.
TerminationMessagePath string `json:"terminationMessagePath,omitempty"`
// Optional: Default to false.
Privileged bool `json:"privileged,omitempty"`
// Required: Policy for pulling images for this container
ImagePullPolicy PullPolicy `json:"imagePullPolicy"`
// Optional: Capabilities for container.
Capabilities Capabilities `json:"capabilities,omitempty"`
// SecurityContext defines the security context the pod should run with
SecurityContext `json:",inline"`
}

// Handler defines a specific action that should be taken
Expand Down Expand Up @@ -1776,6 +1774,110 @@ type SecretList struct {
Items []Secret `json:"items"`
}

// SecurityContext holds security configuration that will be applied to a container. If a security context
// is set on the container spec then it must comply with any constraints defined in the SecurityConstraints context
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Which component is going to read this object? Kubelet?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The question was about which component is going to read SecurityConstraints.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe SecurityConstraints and its dependencies are part of the kubelet API but not necessarily part of the core kubenetes API? Take a look at how scheduler has its own API:
plugin/pkg/scheduler/api/...

SecurityContext seems like it is part of the core API since Pods use it.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What we've discussed so far was that enforcement of constraints by the SecurityPolicyProvider can belong on two levels: api server and kubelet. On both sides the policy can allow or disallow containers that meet the constraints to run either by outright preventing the docker run (kubelet) or rejecting the pod create/mod (api server). It is likely that the policy would be represented the same on both sides, I haven't thought of a scenario where they would differ.

#6287 (comment)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Apologies, poor pronoun use. we := myself and @smarterclayton discussed in the link provided above.

// it is running in. If a security context is not supplied and the pod is running under a SecurityConstraints context
// then a default SecurityContext may be applied.
type SecurityContext struct {
// Capabilities are the capabilities to add/drop when running the container
Capabilities *Capabilities `json:"capabilities,omitempty"`

// Run the container in privileged mode
Privileged bool `json:"privileged,omitempty"`

// SELinuxOptions are the labels to be applied to the container
// and volumes
SELinuxOptions *SELinuxOptions `json:"seLinuxOptions,omitempty"`

// RunAsUser is the UID to run the entrypoint of the container process. Corresponding option is --user or -u
RunAsUser int64 `json:"runAsUser,omitempty"`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This has to be *int64 because not specifying a RunAsUser is the same as saying "use the image's default".

}

// SELinuxOptions are the labels to be applied to the container
type SELinuxOptions struct {
// User --security-opt="label:user:USER"
User string `json:"user,omitempty"`

// Role --security-opt="label:role:ROLE"
Role string `json:"role,omitempty"`

// Type --security-opt="label:type:TYPE"
Type string `json:"type,omitempty"`

// Level --security-opt="label:level:LEVEL"
Level string `json:"level,omitempty"`

// Disabled --security-opt="label:disable"
Disabled bool `json:"disabled,omitempty"`
}

// SecurityConstraints provides the constraints that at security context provider will
// ensure that applied SecurityContext requests follow. When a setting is provided in the SecurityConstraints
// that conflicts with an actual request it will be implementation specific whether that request is
// ignored and the container is still run (without the requested constraint) or if the container will be failed
type SecurityConstraints struct {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are you expecting that there is one SecurityConstraints for the whole cluster, or per namespace, or per user or what?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These would eventually be scoped to a ServiceAccount.

// EnforcementPolicy will drive behavior for how the constraints are enforce
EnforcementPolicy SecurityConstraintPolicy `json:"enforcementPolicy,omitempty"`

// AllowPrivileged indicates whether this context allows privileged mode containers
AllowPrivileged bool `json:"allowPrivileged,omitempty"`

// SELinux provides the security constraint options for selinux
SELinux *SELinuxSecurityConstraints `json:"seLinux,omitempty"`

// AllowCapabilities dictates if a container can request to add or drop capabilites
AllowCapabilities bool `json:"allowCapabilities,omitempty"`

// AllowCapabilities dictates if a container can request to run the entry point process as a specific user
AllowRunAsUser bool `json:"allowRunAsUser,omitempty"`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is interesting, but there's probably a number of options here like:

  • allow Run as User to be set to arbitrary
  • allow run as user in this range
  • allow run as non-root user
  • allow run as with the image user

That means this probably needs to be a struct.


// Capabilities represents, if AllowCapabilities is true, the caps that requests
// are allowed to add or drop.
Capabilities *Capabilities `json:"capabilities,omitempty"`

// DefaultSecurityContext is applied to any container that does not have a security context set. It must
// also conform to the constraints defined in SecurityConstraints object
DefaultSecurityContext *SecurityContext `json:"defaultSecurityContext,omitempty"`

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Containers (not pods) have a security context. The context is declarative:

  • I want to run as user ID 6
  • I want to run as group ID 10
  • I want to run privileged
  • I want to add these capabilities
  • I want to drop these capabilities
  • I want to run with user namespaces in range mode using the UIDs from 2,810,000 to 2,820,000.
  • I want to run with these MCS labels for selinux

The security provider in the kubelet, and the security context that is associated with the service account, will say what the containers can do:

  • Containers under this security context can:
    • ask to be run as privileged
    • run as only this user and god
    • run as any user except root
    • must run as a random UID on the host
    • may ask for these capabilities
    • must run with these selinux labels
    • must run with a unique set of MCS labels on the host

One of these objects is part of the pod spec and is "do this"
One of these objects is used to constrain the pod spec - "reject pod specs that don't fit these bounds" or "default to these settings".

// List of pod sources for which using host network is allowed.
HostNetworkSources []string
}

// SELinuxSecurityConstraints defines what is allowed in SecurityContext requests with regards to SELinux label options
// that are currently supported by docker
type SELinuxSecurityConstraints struct {
// AllowUserLabel --security-opt="label:user:USER"
AllowUserLabel bool `json:"allowUserLabel,omitempty"`

// AllowRoleLabel --security-opt="label:role:ROLE"
AllowRoleLabel bool `json:"allowRoleLabel,omitempty"`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

All of these have complex constraints that can't be captured with booleans (i.e. this doesn't look like it's being driven by the concrete requirements in the security doc). Can you include the options necessary to satisfy the requirements?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do we need the ability to support the nuance of "I allow you to make requests on the container spec but will set it based on my security constraint settings if you don't"? If so the booleans are still useful and the SecurityConstraints need to be changed.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Almost certainly.

----- Original Message -----

  • // DefaultSecurityContext is applied to any container that does not have
    a security context set. It must
  • // also conform to the constraints defined in SecurityConstraints object
  • DefaultSecurityContext *SecurityContext
    json:"defaultSecurityContext,omitempty"
  • // List of pod sources for which using host network is allowed.
  • HostNetworkSources []string
    +}

+// SELinuxSecurityConstraints defines what is allowed in SecurityContext
requests with regards to SELinux label options
+// that are currently supported by docker
+type SELinuxSecurityConstraints struct {

  • // AllowUserLabel --security-opt="label:user:USER"
  • AllowUserLabel bool json:"allowUserLabel,omitempty"
  • // AllowRoleLabel --security-opt="label:role:ROLE"
  • AllowRoleLabel bool json:"allowRoleLabel,omitempty"

do we need the ability to support the nuance of "I allow you to make requests
on the container spec but will set it based on my security constraint
settings if you don't"? If so the booleans are still useful and the
SecurityConstraints need to be changed.


Reply to this email directly or view it on GitHub:
https://github.com/GoogleCloudPlatform/kubernetes/pull/6287/files#r29001573


// AllowTypeLabel --security-opt="label:type:TYPE"
AllowTypeLabel bool `json:"allowTypeLabel,omitempty"`

// AllowLevelLabel --security-opt="label:level:LEVEL"
AllowLevelLabel bool `json:"allowLevelLabel,omitempty"`

// AllowDisable --security-opt="label:disable"
AllowDisable bool `json:"allowDisable,omitempty"`
}

// SecurityConstraintPolicy dictates how the security context provider should behave with regards to contexts that
// do not meet the requirements of the policy.
type SecurityConstraintPolicy string

const (
// SecurityConstraintPolicyDisabled means that any containers that do not meet policy constraints are still
// allowed to run and will be given their requested permissions. This could be used if an admin needs to allow
// a pod to run for testing without deleting and recreating the policy. Implementation may log warnings for
// permission requests that do not comply with the policy that would be enforced
SecurityConstraintPolicyDisable = "Disable"

// SecurityConstraintPolicyReject means that any containers that do not meet policy constraints will be rejected
// (in the case of the api server) or not run (in the case of the kubelet)
SecurityConstraintPolicyReject = "Reject"
)

// These constants are for remote command execution and port forwarding and are
// used by both the client side and server side components.
//
Expand Down
6 changes: 6 additions & 0 deletions pkg/api/v1beta1/conversion.go
Expand Up @@ -579,6 +579,9 @@ func init() {
if err := s.Convert(&in.Capabilities, &out.Capabilities, 0); err != nil {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This conversion doesn't need to be called this way anymore, since this is a field on SecurityContext now.

return err
}
if err := s.Convert(&in.SecurityContext, &out.SecurityContext, 0); err != nil {
return err
}
return nil
},
// Internal API does not support CPU to be specified via an explicit field.
Expand Down Expand Up @@ -665,6 +668,9 @@ func init() {
if err := s.Convert(&in.Capabilities, &out.Capabilities, 0); err != nil {
return err
}
if err := s.Convert(&in.SecurityContext, &out.SecurityContext, 0); err != nil {
return err
}
return nil
},
func(in *newer.PodSpec, out *ContainerManifest, s conversion.Scope) error {
Expand Down
4 changes: 4 additions & 0 deletions pkg/api/v1beta1/register.go
Expand Up @@ -70,6 +70,8 @@ func init() {
&PodProxyOptions{},
&ComponentStatus{},
&ComponentStatusList{},
&SecurityContext{},
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think this is a top level object because it doesn't have a kind.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You're right, it needs to be removed.

&SecurityConstraints{},
)
// Future names are supported
api.Scheme.AddKnownTypeWithName("v1beta1", "Node", &Minion{})
Expand Down Expand Up @@ -114,3 +116,5 @@ func (*PodExecOptions) IsAnAPIObject() {}
func (*PodProxyOptions) IsAnAPIObject() {}
func (*ComponentStatus) IsAnAPIObject() {}
func (*ComponentStatusList) IsAnAPIObject() {}
func (*SecurityContext) IsAnAPIObject() {}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same comment here

func (*SecurityConstraints) IsAnAPIObject() {}