From e6ed4945c3054551e81756d5fb5ef512c384f0a7 Mon Sep 17 00:00:00 2001 From: yash Date: Thu, 4 Apr 2024 04:41:36 +0530 Subject: [PATCH] feat: prometheus-server podSecurity (#6361) --- pkg/k8sutil/labels.go | 11 ++++ pkg/operator/config_reloader.go | 38 ++++++++++--- pkg/prometheus/server/operator.go | 5 +- pkg/prometheus/server/statefulset.go | 83 ++++++++++++++++++++++------ pkg/prometheus/statefulset.go | 5 +- 5 files changed, 113 insertions(+), 29 deletions(-) diff --git a/pkg/k8sutil/labels.go b/pkg/k8sutil/labels.go index 259899fd60d..b710f1ea74e 100644 --- a/pkg/k8sutil/labels.go +++ b/pkg/k8sutil/labels.go @@ -15,11 +15,13 @@ package k8sutil import ( + "context" "fmt" "reflect" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" ) // LabelSelectionHasChanged returns true if the selector doesn't yield the same results @@ -42,3 +44,12 @@ func LabelSelectionHasChanged(old, current map[string]string, selector *metav1.L return sel.Matches(labels.Set(old)) != sel.Matches(labels.Set(current)), nil } + +func GetPodSecurityLabel(namespace string, ctx context.Context, client kubernetes.Interface) *string { + ns, _ := client.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) + labels := ns.GetLabels() + if val, ok := labels["pod-security.kubernetes.io/enforce"]; ok { + return &val + } + return nil +} diff --git a/pkg/operator/config_reloader.go b/pkg/operator/config_reloader.go index a923fef42d3..d66edccb00c 100644 --- a/pkg/operator/config_reloader.go +++ b/pkg/operator/config_reloader.go @@ -174,7 +174,7 @@ func ImagePullPolicy(imagePullPolicy v1.PullPolicy) ReloaderOption { // CreateConfigReloader returns the definition of the config-reloader // container. -func CreateConfigReloader(name string, options ...ReloaderOption) v1.Container { +func CreateConfigReloader(name string, podSecurityLabel *string, options ...ReloaderOption) v1.Container { configReloader := ConfigReloader{name: name} for _, option := range options { @@ -257,6 +257,34 @@ func CreateConfigReloader(name string, options ...ReloaderOption) v1.Container { Value: strconv.Itoa(int(*configReloader.shard)), }) } + securityContext := &v1.SecurityContext{} + if podSecurityLabel == nil { + securityContext = &v1.SecurityContext{ + AllowPrivilegeEscalation: ptr.To(false), + ReadOnlyRootFilesystem: ptr.To(true), + Capabilities: &v1.Capabilities{ + Drop: []v1.Capability{"ALL"}, + }, + } + } else { + switch *podSecurityLabel { + case "restricted": + securityContext = &v1.SecurityContext{ + AllowPrivilegeEscalation: ptr.To(false), + RunAsNonRoot: ptr.To(true), + SeccompProfile: &v1.SeccompProfile{ + Type: "RuntimeDefault", + }, + Capabilities: &v1.Capabilities{ + Drop: []v1.Capability{"ALL"}, + }, + } + case "baseline": + securityContext = &v1.SecurityContext{} + default: + securityContext = &v1.SecurityContext{} + } + } c := v1.Container{ Name: name, @@ -269,13 +297,7 @@ func CreateConfigReloader(name string, options ...ReloaderOption) v1.Container { Ports: ports, VolumeMounts: configReloader.volumeMounts, Resources: configReloader.config.ResourceRequirements(), - SecurityContext: &v1.SecurityContext{ - AllowPrivilegeEscalation: ptr.To(false), - ReadOnlyRootFilesystem: ptr.To(true), - Capabilities: &v1.Capabilities{ - Drop: []v1.Capability{"ALL"}, - }, - }, + SecurityContext: securityContext, } if !configReloader.runOnce && configReloader.config.EnableProbes { diff --git a/pkg/prometheus/server/operator.go b/pkg/prometheus/server/operator.go index c596c921791..edbe261b71b 100644 --- a/pkg/prometheus/server/operator.go +++ b/pkg/prometheus/server/operator.go @@ -827,7 +827,7 @@ func (c *Operator) sync(ctx context.Context, key string) error { if err != nil { return err } - + podSecurityLabel := k8sutil.GetPodSecurityLabel(p.Namespace, ctx, c.kclient) sset, err := makeStatefulSet( ssetName, p, @@ -848,7 +848,8 @@ func (c *Operator) sync(ctx context.Context, key string) error { ruleConfigMapNames, newSSetInputHash, int32(shard), - tlsAssets) + tlsAssets, + podSecurityLabel) if err != nil { return fmt.Errorf("making statefulset failed: %w", err) } diff --git a/pkg/prometheus/server/statefulset.go b/pkg/prometheus/server/statefulset.go index 122540afab3..03a0fae9191 100644 --- a/pkg/prometheus/server/statefulset.go +++ b/pkg/prometheus/server/statefulset.go @@ -104,6 +104,7 @@ func makeStatefulSet( inputHash string, shard int32, tlsSecrets *operator.ShardedSecret, + podSecurityLabel *string, ) (*appsv1.StatefulSet, error) { cpf := p.GetCommonPrometheusFields() objMeta := p.GetObjectMeta() @@ -117,7 +118,7 @@ func makeStatefulSet( // We need to re-set the common fields because cpf is only a copy of the original object. // We set some defaults if some fields are not present, and we want those fields set in the original Prometheus object before building the StatefulSetSpec. p.SetCommonPrometheusFields(cpf) - spec, err := makeStatefulSetSpec(baseImage, tag, sha, retention, retentionSize, rules, query, allowOverlappingBlocks, enableAdminAPI, queryLogFile, thanos, disableCompaction, p, config, cg, shard, ruleConfigMapNames, tlsSecrets) + spec, err := makeStatefulSetSpec(baseImage, tag, sha, retention, retentionSize, rules, query, allowOverlappingBlocks, enableAdminAPI, queryLogFile, thanos, disableCompaction, p, config, cg, shard, ruleConfigMapNames, tlsSecrets, podSecurityLabel) if err != nil { return nil, fmt.Errorf("make StatefulSet spec: %w", err) } @@ -228,6 +229,7 @@ func makeStatefulSetSpec( shard int32, ruleConfigMapNames []string, tlsSecrets *operator.ShardedSecret, + podSecurityLabel *string, ) (*appsv1.StatefulSetSpec, error) { cpf := p.GetCommonPrometheusFields() @@ -356,7 +358,7 @@ func makeStatefulSetSpec( var additionalContainers, operatorInitContainers []v1.Container - thanosContainer, err := createThanosContainer(&disableCompaction, p, thanos, c) + thanosContainer, err := createThanosContainer(&disableCompaction, p, thanos, c, podSecurityLabel) if err != nil { return nil, err } @@ -398,6 +400,7 @@ func makeStatefulSetSpec( true, configReloaderVolumeMounts, watchedDirectories, + podSecurityLabel, operator.Shard(shard), ), ) @@ -411,7 +414,34 @@ func makeStatefulSetSpec( if err != nil { return nil, err } - + securityContext := &v1.SecurityContext{} + if podSecurityLabel == nil { + securityContext = &v1.SecurityContext{ + AllowPrivilegeEscalation: ptr.To(false), + ReadOnlyRootFilesystem: ptr.To(true), + Capabilities: &v1.Capabilities{ + Drop: []v1.Capability{"ALL"}, + }, + } + } else { + switch *podSecurityLabel { + case "restricted": + securityContext = &v1.SecurityContext{ + AllowPrivilegeEscalation: ptr.To(false), + RunAsNonRoot: ptr.To(true), + SeccompProfile: &v1.SeccompProfile{ + Type: "RuntimeDefault", + }, + Capabilities: &v1.Capabilities{ + Drop: []v1.Capability{"ALL"}, + }, + } + case "baseline": + securityContext = &v1.SecurityContext{} + default: + securityContext = &v1.SecurityContext{} + } + } operatorContainers := append([]v1.Container{ { Name: "prometheus", @@ -425,13 +455,7 @@ func makeStatefulSetSpec( ReadinessProbe: readinessProbe, Resources: cpf.Resources, TerminationMessagePolicy: v1.TerminationMessageFallbackToLogsOnError, - SecurityContext: &v1.SecurityContext{ - ReadOnlyRootFilesystem: ptr.To(true), - AllowPrivilegeEscalation: ptr.To(false), - Capabilities: &v1.Capabilities{ - Drop: []v1.Capability{"ALL"}, - }, - }, + SecurityContext: securityContext, }, prompkg.BuildConfigReloader( p, @@ -439,7 +463,9 @@ func makeStatefulSetSpec( false, configReloaderVolumeMounts, watchedDirectories, + podSecurityLabel, operator.Shard(shard), + operator.WebConfigFile(configReloaderWebConfigFile), ), }, additionalContainers...) @@ -611,6 +637,7 @@ func createThanosContainer( p monitoringv1.PrometheusInterface, thanos *monitoringv1.ThanosSpec, c *prompkg.Config, + podSecurityLabel *string, ) (*v1.Container, error) { var container *v1.Container cpf := p.GetCommonPrometheusFields() @@ -656,19 +683,41 @@ func createThanosContainer( thanosArgs = append(thanosArgs, monitoringv1.Argument{Name: "grpc-server-tls-client-ca", Value: tls.CAFile}) } } + securityContext := &v1.SecurityContext{} + if podSecurityLabel == nil { + securityContext = &v1.SecurityContext{ + AllowPrivilegeEscalation: ptr.To(false), + ReadOnlyRootFilesystem: ptr.To(true), + Capabilities: &v1.Capabilities{ + Drop: []v1.Capability{"ALL"}, + }, + } + } else { + switch *podSecurityLabel { + case "restricted": + securityContext = &v1.SecurityContext{ + AllowPrivilegeEscalation: ptr.To(false), + RunAsNonRoot: ptr.To(true), + SeccompProfile: &v1.SeccompProfile{ + Type: "RuntimeDefault", + }, + Capabilities: &v1.Capabilities{ + Drop: []v1.Capability{"ALL"}, + }, + } + case "baseline": + securityContext = &v1.SecurityContext{} + default: + securityContext = &v1.SecurityContext{} + } + } container = &v1.Container{ Name: "thanos-sidecar", Image: thanosImage, ImagePullPolicy: cpf.ImagePullPolicy, TerminationMessagePolicy: v1.TerminationMessageFallbackToLogsOnError, - SecurityContext: &v1.SecurityContext{ - AllowPrivilegeEscalation: ptr.To(false), - ReadOnlyRootFilesystem: ptr.To(true), - Capabilities: &v1.Capabilities{ - Drop: []v1.Capability{"ALL"}, - }, - }, + SecurityContext: securityContext, Ports: []v1.ContainerPort{ { Name: "http", diff --git a/pkg/prometheus/statefulset.go b/pkg/prometheus/statefulset.go index 71f1d81d108..bd27600d273 100644 --- a/pkg/prometheus/statefulset.go +++ b/pkg/prometheus/statefulset.go @@ -412,6 +412,7 @@ func BuildConfigReloader( initContainer bool, mounts []v1.VolumeMount, watchedDirectories []string, + podSecurityLabel *string, opts ...operator.ReloaderOption, ) v1.Container { cpf := p.GetCommonPrometheusFields() @@ -432,7 +433,7 @@ func BuildConfigReloader( if initContainer { name = "init-config-reloader" reloaderOptions = append(reloaderOptions, operator.ReloaderRunOnce()) - return operator.CreateConfigReloader(name, reloaderOptions...) + return operator.CreateConfigReloader(name, podSecurityLabel, reloaderOptions...) } if ptr.Deref(cpf.ReloadStrategy, monitoringv1.HTTPReloadStrategyType) == monitoringv1.ProcessSignalReloadStrategyType { @@ -460,7 +461,7 @@ func BuildConfigReloader( ) } - return operator.CreateConfigReloader(name, reloaderOptions...) + return operator.CreateConfigReloader(name, podSecurityLabel, reloaderOptions...) } func ShareProcessNamespace(p monitoringv1.PrometheusInterface) *bool {