diff --git a/README.md b/README.md index 2c68f44..fc7f43a 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,10 @@ k-rail is a workload policy enforcement tool for Kubernetes. It can help you sec * [No Exec](#no-exec) * [No Bind Mounts](#no-bind-mounts) * [No Docker Sock Mount](#no-docker-sock-mount) - * [Mutate Default Seccomp Profile](#mutate-default-seccomp-profile) + * [EmptyDir size limit](#emptyDir-size-limit) + [Policy configuration](#policy-configuration) + * [Mutate Default Seccomp Profile](#mutate-default-seccomp-profile) + + [Policy configuration](#policy-configuration-1) * [Immutable Image Reference](#immutable-image-reference) * [No Host Network](#no-host-network) * [No Host PID](#no-host-pid) @@ -28,11 +30,11 @@ k-rail is a workload policy enforcement tool for Kubernetes. It can help you sec * [No Privileged Container](#no-privileged-container) * [No Helm Tiller](#no-helm-tiller) * [Trusted Image Repository](#trusted-image-repository) - + [Policy configuration](#policy-configuration-1) + + [Policy configuration](#policy-configuration-2) * [Safe to Evict (DEPRECATED)](#safe-to-evict--deprecated) * [Mutate Safe to Evict](#mutate-safe-to-evict) * [Require Ingress Exemption](#require-ingress-exemption) - + [Policy configuration](#policy-configuration-2) + + [Policy configuration](#policy-configuration-3) - [Configuration](#configuration) * [Logging](#logging) * [Modes of operation](#modes-of-operation) @@ -230,6 +232,21 @@ The Docker socket bind mount provides API access to the host Docker daemon, whic **Note:** It is recommended to use the `No Bind Mounts` policy to disable all `hostPath` mounts rather than only this policy. +## EmptyDir size limit +By [default](https://kubernetes.io/docs/concepts/storage/volumes/#example-pod), an `emptyDir` lacks a `sizeLimit` parameter, and is disk-based; +a Pod with access to said `emptyDir` can consume the Node's entire disk (i.e. the limit is unbounded) until the offending Pod is deleted or evicted, which can constitute a denial-of-service condition at the affected Node (i.e. DiskPressure). +This policy +* sets the configured default size when none is set for an `emptyDir` volume +* reports a violation when the size is greater then the configured max size + +### Policy configuration +```yaml +policy_config: + mutate_empty_dir_size_limit: + maximum_size_limit: "1Gi" + default_size_limit: "512Mi" +``` + ## Mutate Default Seccomp Profile Sets a default seccomp profile (`runtime/default` or a configured one) for Pods if they have no existing seccomp configuration. The default seccomp policy for Docker and Containerd both block over 40 syscalls, [many of which](https://docs.docker.com/engine/security/seccomp/#significant-syscalls-blocked-by-the-default-profile) are potentially dangerous. The default policies are [usually very compatible](https://blog.jessfraz.com/post/containers-security-and-echo-chambers/#breaking-changes) with applications, too. diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index 2815b08..bb48e68 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -36,6 +36,9 @@ config: - '^k8s.gcr.io/.*' # official k8s GCR repo - '^[A-Za-z0-9\-:@]+$' # official docker hub images policy_default_seccomp_policy: "runtime/default" + mutate_empty_dir_size_limit: + maximum_size_limit: "1Gi" + default_size_limit: "512Mi" policies: - name: "pod_no_exec" enabled: True @@ -73,6 +76,9 @@ config: - name: "pod_mutate_safe_to_evict" enabled: True report_only: False + - name: "pod_empty_dir_size_limit" + enabled: True + report_only: False - name: "pod_default_seccomp_policy" enabled: True report_only: False diff --git a/go.mod b/go.mod index 5bceac2..3064c91 100644 --- a/go.mod +++ b/go.mod @@ -21,11 +21,11 @@ require ( golang.org/x/text v0.3.0 // indirect gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/inf.v0 v0.9.0 // indirect - gopkg.in/yaml.v2 v2.2.2 + gopkg.in/yaml.v2 v2.2.2 // indirect k8s.io/api v0.0.0-20190301173355-16f65c82b8fa k8s.io/apimachinery v0.0.0-20190301173222-2f7e9cae4418 k8s.io/klog v0.0.0-20181108234604-8139d8cb77af // indirect - sigs.k8s.io/yaml v1.1.0 // indirect + sigs.k8s.io/yaml v1.1.0 ) replace git.apache.org/thrift.git => github.com/apache/thrift v0.12.0 diff --git a/policies/config.go b/policies/config.go index d7b4649..e563776 100644 --- a/policies/config.go +++ b/policies/config.go @@ -12,14 +12,57 @@ package policies +import ( + "encoding/json" + "errors" + "fmt" + + apiresource "k8s.io/apimachinery/pkg/api/resource" +) + // Config contains configuration for Policies type Config struct { // PolicyRequireIngressExemptionClasses contains the Ingress classes that an exemption is required for // to use. Typically this would include your public ingress classes. - PolicyRequireIngressExemptionClasses []string `yaml:"policy_require_ingress_exemption_classes"` + PolicyRequireIngressExemptionClasses []string `json:"policy_require_ingress_exemption_classes"` // PolicyTrustedRepositoryRegexes contains regexes that match image repositories that you want to allow. - PolicyTrustedRepositoryRegexes []string `yaml:"policy_trusted_repository_regexes"` + PolicyTrustedRepositoryRegexes []string `json:"policy_trusted_repository_regexes"` // PolicyDefaultSeccompPolicy contains the seccomp policy that you want to be applied on Pods by default. // Defaults to 'runtime/default' - PolicyDefaultSeccompPolicy string `yaml:"policy_default_seccomp_policy"` + PolicyDefaultSeccompPolicy string `json:"policy_default_seccomp_policy"` + + MutateEmptyDirSizeLimit MutateEmptyDirSizeLimit `json:"mutate_empty_dir_size_limit"` +} + +type MutateEmptyDirSizeLimit struct { + MaximumSizeLimit apiresource.Quantity `json:"maximum_size_limit"` + DefaultSizeLimit apiresource.Quantity `json:"default_size_limit"` +} + +func (m *MutateEmptyDirSizeLimit) UnmarshalJSON(value []byte) error { + var v map[string]json.RawMessage + if err := json.Unmarshal(value, &v); err != nil { + return err + } + + if max, ok := v["maximum_size_limit"]; ok { + if err := m.MaximumSizeLimit.UnmarshalJSON(max); err != nil { + return fmt.Errorf("maximum_size_limit failed: %s", err) + } + } + if def, ok := v["default_size_limit"]; ok { + if err := m.DefaultSizeLimit.UnmarshalJSON(def); err != nil { + return fmt.Errorf("default_size_limit failed: %s", err) + } + } + if m.DefaultSizeLimit.IsZero() { + return errors.New("default size must not be empty") + } + if m.MaximumSizeLimit.IsZero() { + return errors.New("max size must not be empty") + } + if m.DefaultSizeLimit.Cmp(m.MaximumSizeLimit) > 0 { + return errors.New("default size must not be greater than max size") + } + return nil } diff --git a/policies/config_test.go b/policies/config_test.go new file mode 100644 index 0000000..f60d096 --- /dev/null +++ b/policies/config_test.go @@ -0,0 +1,77 @@ +package policies + +import ( + "reflect" + "testing" + + apiresource "k8s.io/apimachinery/pkg/api/resource" + "sigs.k8s.io/yaml" +) + +func TestMutateEmptyDirSizeLimit(t *testing.T) { + specs := map[string]struct { + src string + exp *MutateEmptyDirSizeLimit + expErr bool + }{ + + "all good": { + src: ` +mutate_empty_dir_size_limit: + maximum_size_limit: "1Gi" + default_size_limit: "512Mi" +`, + exp: &MutateEmptyDirSizeLimit{ + MaximumSizeLimit: apiresource.MustParse("1Gi"), + DefaultSizeLimit: apiresource.MustParse("512Mi"), + }, + }, + "default > max": { + src: ` +mutate_empty_dir_size_limit: + maximum_size_limit: "1Gi" + default_size_limit: "2Gi" +`, + expErr: true, + }, + "default not set": { + src: ` +mutate_empty_dir_size_limit: + maximum_size_limit: "1Gi" +`, + expErr: true, + }, + "max not set": { + src: ` +mutate_empty_dir_size_limit: + default_size_limit: "2Gi" +`, + expErr: true, + }, + "unsupported type": { + src: ` +mutate_empty_dir_size_limit: + default_size_limit: "2ALX" + maximum_size_limit: "2ALX" +`, + expErr: true, + }, + } + for msg, spec := range specs { + t.Run(msg, func(t *testing.T) { + var cfg Config + switch err := yaml.Unmarshal([]byte(spec.src), &cfg); { + case spec.expErr && err != nil: + return + case spec.expErr: + t.Fatal("expected error") + case !spec.expErr && err != nil: + t.Fatalf("unexpected error: %+v", err) + } + if exp, got := *spec.exp, cfg.MutateEmptyDirSizeLimit; !reflect.DeepEqual(exp, got) { + t.Errorf("expected %v but got %v", exp, got) + } + }) + } + +} diff --git a/policies/exemption.go b/policies/exemption.go index 203caa3..d7f26d6 100644 --- a/policies/exemption.go +++ b/policies/exemption.go @@ -19,17 +19,17 @@ import ( "github.com/gobwas/glob" log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" authenticationv1 "k8s.io/api/authentication/v1" + "sigs.k8s.io/yaml" ) // RawExemption is the configuration for a policy exemption type RawExemption struct { - ResourceName string `yaml:"resource_name"` - Namespace string `yaml:"namespace"` - Username string `yaml:"username"` - Group string `yaml:"group"` - ExemptPolicies []string `yaml:"exempt_policies"` + ResourceName string `json:"resource_name"` + Namespace string `json:"namespace"` + Username string `json:"username"` + Group string `json:"group"` + ExemptPolicies []string `json:"exempt_policies"` } // CompiledExemption is the compiled configuration for a policy exemption diff --git a/policies/pod/empty_dir_size_limit.go b/policies/pod/empty_dir_size_limit.go new file mode 100644 index 0000000..b27eca7 --- /dev/null +++ b/policies/pod/empty_dir_size_limit.go @@ -0,0 +1,81 @@ +// Copyright 2019 Cruise LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// https://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.package ingress + +package pod + +import ( + "context" + "fmt" + + "github.com/cruise-automation/k-rail/policies" + "github.com/cruise-automation/k-rail/resource" + admissionv1beta1 "k8s.io/api/admission/v1beta1" +) + +type PolicyEmptyDirSizeLimit struct { +} + +func (p PolicyEmptyDirSizeLimit) Name() string { + return "pod_empty_dir_size_limit" +} + +const violationText = "Empty dir size limit: size limit exceeds the max value" + +func (p PolicyEmptyDirSizeLimit) Validate(ctx context.Context, config policies.Config, ar *admissionv1beta1.AdmissionRequest) ([]policies.ResourceViolation, []policies.PatchOperation) { + var resourceViolations []policies.ResourceViolation + + podResource := resource.GetPodResource(ar, ctx) + if podResource == nil { + return resourceViolations, nil + } + + cfg := config.MutateEmptyDirSizeLimit + var patches []policies.PatchOperation + + for i, volume := range podResource.PodSpec.Volumes { + if volume.EmptyDir == nil { + continue + } + if volume.EmptyDir.SizeLimit == nil || volume.EmptyDir.SizeLimit.IsZero() { + patches = append(patches, policies.PatchOperation{ + Op: "replace", + Path: fmt.Sprintf(volumePatchPath(podResource.ResourceKind)+"/%d/emptyDir/sizeLimit", i), + Value: cfg.DefaultSizeLimit.String(), + }) + continue + } + + if volume.EmptyDir.SizeLimit.Cmp(cfg.MaximumSizeLimit) > 0 { + resourceViolations = append(resourceViolations, policies.ResourceViolation{ + Namespace: ar.Namespace, + ResourceName: podResource.ResourceName, + ResourceKind: podResource.ResourceKind, + Violation: violationText, + Policy: p.Name(), + }) + } + } + return resourceViolations, patches +} + +const templateVolumePath = "/spec/template/spec/volumes" + +func volumePatchPath(podKind string) string { + nonTemplateKinds := map[string]string{ + "Pod": "/spec/volumes", + "CronJob": "/spec/jobTemplate/spec/template/spec/volumes", + } + if pathPath, ok := nonTemplateKinds[podKind]; ok { + return pathPath + } + return templateVolumePath +} diff --git a/policies/pod/empty_dir_size_limit_test.go b/policies/pod/empty_dir_size_limit_test.go new file mode 100644 index 0000000..fc0bcbb --- /dev/null +++ b/policies/pod/empty_dir_size_limit_test.go @@ -0,0 +1,213 @@ +package pod + +import ( + "context" + "encoding/json" + "reflect" + "testing" + + "github.com/cruise-automation/k-rail/policies" + admissionv1beta1 "k8s.io/api/admission/v1beta1" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + apiresource "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +func TestEmptyDirSizeLimit(t *testing.T) { + config := policies.Config{ + MutateEmptyDirSizeLimit: policies.MutateEmptyDirSizeLimit{ + DefaultSizeLimit: *apiresource.NewQuantity(1, apiresource.DecimalSI), + MaximumSizeLimit: *apiresource.NewQuantity(10, apiresource.DecimalSI), + }, + } + + specs := map[string]struct { + src v1.PodSpec + expViolations []policies.ResourceViolation + expPatches []policies.PatchOperation + }{ + "limit set within range": { + src: v1.PodSpec{ + Volumes: []v1.Volume{{ + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + SizeLimit: apiresource.NewQuantity(2, apiresource.DecimalSI)}, + }, + }}, + }, + }, + "limit set within range with multiple volumes": { + src: v1.PodSpec{ + Volumes: []v1.Volume{{ + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + SizeLimit: apiresource.NewQuantity(2, apiresource.DecimalSI)}, + }, + }, { + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + SizeLimit: apiresource.NewQuantity(3, apiresource.DecimalSI)}, + }, + }}, + }, + }, + "set default value when 0": { + src: v1.PodSpec{ + Volumes: []v1.Volume{{ + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + SizeLimit: apiresource.NewQuantity(0, apiresource.DecimalExponent), + }, + }}, + }, + }, + expPatches: []policies.PatchOperation{ + { + Path: "/spec/template/spec/volumes/0/emptyDir/sizeLimit", + Op: "replace", + Value: "1", + }, + }, + }, "set default value when empty": { + src: v1.PodSpec{ + Volumes: []v1.Volume{{ + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }}, + }, + }, + expPatches: []policies.PatchOperation{ + { + Path: "/spec/template/spec/volumes/0/emptyDir/sizeLimit", + Op: "replace", + Value: "1", + }, + }, + }, + "set default value when empty with multiple": { + src: v1.PodSpec{ + Volumes: []v1.Volume{{ + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }}, { + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }}, + }, + }, + expPatches: []policies.PatchOperation{ + { + Path: "/spec/template/spec/volumes/0/emptyDir/sizeLimit", + Op: "replace", + Value: "1", + }, + { + Path: "/spec/template/spec/volumes/1/emptyDir/sizeLimit", + Op: "replace", + Value: "1", + }, + }, + }, + "allow max limit size": { + src: v1.PodSpec{ + Volumes: []v1.Volume{{ + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + SizeLimit: apiresource.NewQuantity(10, apiresource.DecimalSI), + }, + }}, + }, + }, + }, + "prevent greater than max limit size": { + src: v1.PodSpec{ + Volumes: []v1.Volume{{ + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + SizeLimit: apiresource.NewQuantity(11, apiresource.DecimalSI), + }, + }}, + }, + }, + expViolations: []policies.ResourceViolation{ + { + ResourceName: "test", + ResourceKind: "Deployment", + Namespace: "test", + Violation: "Empty dir size limit: size limit exceeds the max value", + Policy: "pod_empty_dir_size_limit", + }, + }, + }, + "skip non empty dir volume": { + src: v1.PodSpec{ + Volumes: []v1.Volume{{ + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{}, + }}, + }, + }, + }, + } + for msg, spec := range specs { + t.Run(msg, func(t *testing.T) { + policy := PolicyEmptyDirSizeLimit{} + v, p := policy.Validate(context.TODO(), config, asFakeAdmissionRequest(spec.src)) + if exp, got := spec.expViolations, v; !reflect.DeepEqual(exp, got) { + t.Errorf("expected %#v but got %#v", exp, got) + } + if exp, got := spec.expPatches, p; !reflect.DeepEqual(exp, got) { + t.Errorf("expected %#v but got %#v", exp, got) + } + + }) + } +} + +func TestResourceVolumePatchPath(t *testing.T) { + specs := map[string]string{ + "Pod": "/spec/volumes", + "ReplicationController": "/spec/template/spec/volumes", + "Deployment": "/spec/template/spec/volumes", + "ReplicaSet": "/spec/template/spec/volumes", + "DaemonSet": "/spec/template/spec/volumes", + "StatefulSet": "/spec/template/spec/volumes", + "Job": "/spec/template/spec/volumes", + "CronJob": "/spec/jobTemplate/spec/template/spec/volumes", + } + for kind, exp := range specs { + t.Run(kind, func(t *testing.T) { + got := volumePatchPath(kind) + if exp != got { + t.Errorf("expected %q but got %q", exp, got) + } + }) + } + +} + +func asFakeAdmissionRequest(src v1.PodSpec) *admissionv1beta1.AdmissionRequest { + xxx := appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: appsv1.DeploymentSpec{ + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: src, + }, + }, + Status: appsv1.DeploymentStatus{}, + } + b, err := json.Marshal(&xxx) + if err != nil { + panic(err) + } + return &admissionv1beta1.AdmissionRequest{ + Resource: metav1.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"}, + Name: "any", + Namespace: "test", + Object: runtime.RawExtension{Raw: b}, + } +} diff --git a/server/config.go b/server/config.go index b4b38bf..39dd875 100644 --- a/server/config.go +++ b/server/config.go @@ -19,17 +19,17 @@ import ( type PolicySettings struct { Name string Enabled bool - ReportOnly bool `yaml:"report_only"` + ReportOnly bool `json:"report_only"` } type Config struct { - LogLevel string `yaml:"log_level"` - BlacklistedNamespaces []string `yaml:"blacklisted_namespaces"` + LogLevel string `json:"log_level"` + BlacklistedNamespaces []string `json:"blacklisted_namespaces"` TLS struct { Cert string Key string } - GlobalReportOnly bool `yaml:"global_report_only"` + GlobalReportOnly bool `json:"global_report_only"` Policies []PolicySettings - PolicyConfig policies.Config `yaml:"policy_config"` + PolicyConfig policies.Config `json:"policy_config"` } diff --git a/server/policies.go b/server/policies.go index e37ba72..afa6d1b 100644 --- a/server/policies.go +++ b/server/policies.go @@ -39,6 +39,7 @@ func (s *Server) registerPolicies() { s.registerPolicy(pod.PolicyNoExec{}) s.registerPolicy(pod.PolicyBindMounts{}) s.registerPolicy(pod.PolicyDockerSock{}) + s.registerPolicy(pod.PolicyEmptyDirSizeLimit{}) s.registerPolicy(pod.PolicyImageImmutableReference{}) s.registerPolicy(pod.PolicyNoTiller{}) s.registerPolicy(pod.PolicyTrustedRepository{}) diff --git a/server/server.go b/server/server.go index 74a57c5..b2f4870 100644 --- a/server/server.go +++ b/server/server.go @@ -25,7 +25,7 @@ import ( "github.com/cruise-automation/k-rail/policies" "github.com/gorilla/mux" log "github.com/sirupsen/logrus" - yaml "gopkg.in/yaml.v2" + "sigs.k8s.io/yaml" ) const ( diff --git a/server/webhook_test.go b/server/webhook_test.go index 3c10aa3..b19271d 100644 --- a/server/webhook_test.go +++ b/server/webhook_test.go @@ -22,6 +22,7 @@ import ( admissionv1beta1 "k8s.io/api/admission/v1beta1" authenticationv1 "k8s.io/api/authentication/v1" corev1 "k8s.io/api/core/v1" + apiresource "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -56,6 +57,12 @@ func test_setup() (Server, []test) { ReportOnly: false, }, }, + PolicyConfig: policies.Config{ + MutateEmptyDirSizeLimit: policies.MutateEmptyDirSizeLimit{ + MaximumSizeLimit: apiresource.MustParse("2Gi"), + DefaultSizeLimit: apiresource.MustParse("1Gi"), + }, + }, }, Exemptions: compiledExemptions, }