Skip to content

Commit

Permalink
feat: prometheus-server podSecurity (#6361)
Browse files Browse the repository at this point in the history
  • Loading branch information
@yp969803
yp969803 committed Apr 3, 2024
1 parent 48444eb commit e6ed494
Show file tree
Hide file tree
Showing 5 changed files with 113 additions and 29 deletions.
11 changes: 11 additions & 0 deletions pkg/k8sutil/labels.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,13 @@
package k8sutil

import (
"context"
"fmt"
"reflect"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/kubernetes"
)

// LabelSelectionHasChanged returns true if the selector doesn't yield the same results
Expand All @@ -42,3 +44,12 @@ func LabelSelectionHasChanged(old, current map[string]string, selector *metav1.L

return sel.Matches(labels.Set(old)) != sel.Matches(labels.Set(current)), nil
}

func GetPodSecurityLabel(namespace string, ctx context.Context, client kubernetes.Interface) *string {

Check warning on line 48 in pkg/k8sutil/labels.go

View workflow job for this annotation

GitHub Actions / Golang linter 

context-as-argument: context.Context should be the first parameter of a function (revive)
ns, _ := client.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{})
labels := ns.GetLabels()
if val, ok := labels["pod-security.kubernetes.io/enforce"]; ok {
return &val
}
return nil
}
38 changes: 30 additions & 8 deletions pkg/operator/config_reloader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ func ImagePullPolicy(imagePullPolicy v1.PullPolicy) ReloaderOption {

// CreateConfigReloader returns the definition of the config-reloader
// container.
func CreateConfigReloader(name string, options ...ReloaderOption) v1.Container {
func CreateConfigReloader(name string, podSecurityLabel *string, options ...ReloaderOption) v1.Container {
configReloader := ConfigReloader{name: name}

for _, option := range options {
Expand Down Expand Up @@ -257,6 +257,34 @@ func CreateConfigReloader(name string, options ...ReloaderOption) v1.Container {
Value: strconv.Itoa(int(*configReloader.shard)),
})
}
securityContext := &v1.SecurityContext{}
if podSecurityLabel == nil {
securityContext = &v1.SecurityContext{
AllowPrivilegeEscalation: ptr.To(false),
ReadOnlyRootFilesystem: ptr.To(true),
Capabilities: &v1.Capabilities{
Drop: []v1.Capability{"ALL"},
},
}
} else {
switch *podSecurityLabel {
case "restricted":
securityContext = &v1.SecurityContext{
AllowPrivilegeEscalation: ptr.To(false),
RunAsNonRoot: ptr.To(true),
SeccompProfile: &v1.SeccompProfile{
Type: "RuntimeDefault",
},
Capabilities: &v1.Capabilities{
Drop: []v1.Capability{"ALL"},
},
}
case "baseline":
securityContext = &v1.SecurityContext{}
default:
securityContext = &v1.SecurityContext{}
}
}

c := v1.Container{
Name: name,
Expand All @@ -269,13 +297,7 @@ func CreateConfigReloader(name string, options ...ReloaderOption) v1.Container {
Ports: ports,
VolumeMounts: configReloader.volumeMounts,
Resources: configReloader.config.ResourceRequirements(),
SecurityContext: &v1.SecurityContext{
AllowPrivilegeEscalation: ptr.To(false),
ReadOnlyRootFilesystem: ptr.To(true),
Capabilities: &v1.Capabilities{
Drop: []v1.Capability{"ALL"},
},
},
SecurityContext: securityContext,
}

if !configReloader.runOnce && configReloader.config.EnableProbes {
Expand Down
5 changes: 3 additions & 2 deletions pkg/prometheus/server/operator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -827,7 +827,7 @@ func (c *Operator) sync(ctx context.Context, key string) error {
if err != nil {
return err
}

podSecurityLabel := k8sutil.GetPodSecurityLabel(p.Namespace, ctx, c.kclient)
sset, err := makeStatefulSet(
ssetName,
p,
Expand All @@ -848,7 +848,8 @@ func (c *Operator) sync(ctx context.Context, key string) error {
ruleConfigMapNames,
newSSetInputHash,
int32(shard),
tlsAssets)
tlsAssets,
podSecurityLabel)
if err != nil {
return fmt.Errorf("making statefulset failed: %w", err)
}
Expand Down
83 changes: 66 additions & 17 deletions pkg/prometheus/server/statefulset.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ func makeStatefulSet(
inputHash string,
shard int32,
tlsSecrets *operator.ShardedSecret,
podSecurityLabel *string,
) (*appsv1.StatefulSet, error) {
cpf := p.GetCommonPrometheusFields()
objMeta := p.GetObjectMeta()
Expand All @@ -117,7 +118,7 @@ func makeStatefulSet(
// We need to re-set the common fields because cpf is only a copy of the original object.
// We set some defaults if some fields are not present, and we want those fields set in the original Prometheus object before building the StatefulSetSpec.
p.SetCommonPrometheusFields(cpf)
spec, err := makeStatefulSetSpec(baseImage, tag, sha, retention, retentionSize, rules, query, allowOverlappingBlocks, enableAdminAPI, queryLogFile, thanos, disableCompaction, p, config, cg, shard, ruleConfigMapNames, tlsSecrets)
spec, err := makeStatefulSetSpec(baseImage, tag, sha, retention, retentionSize, rules, query, allowOverlappingBlocks, enableAdminAPI, queryLogFile, thanos, disableCompaction, p, config, cg, shard, ruleConfigMapNames, tlsSecrets, podSecurityLabel)
if err != nil {
return nil, fmt.Errorf("make StatefulSet spec: %w", err)
}
Expand Down Expand Up @@ -228,6 +229,7 @@ func makeStatefulSetSpec(
shard int32,
ruleConfigMapNames []string,
tlsSecrets *operator.ShardedSecret,
podSecurityLabel *string,
) (*appsv1.StatefulSetSpec, error) {
cpf := p.GetCommonPrometheusFields()

Expand Down Expand Up @@ -356,7 +358,7 @@ func makeStatefulSetSpec(

var additionalContainers, operatorInitContainers []v1.Container

thanosContainer, err := createThanosContainer(&disableCompaction, p, thanos, c)
thanosContainer, err := createThanosContainer(&disableCompaction, p, thanos, c, podSecurityLabel)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -398,6 +400,7 @@ func makeStatefulSetSpec(
true,
configReloaderVolumeMounts,
watchedDirectories,
podSecurityLabel,
operator.Shard(shard),
),
)
Expand All @@ -411,7 +414,34 @@ func makeStatefulSetSpec(
if err != nil {
return nil, err
}

securityContext := &v1.SecurityContext{}
if podSecurityLabel == nil {
securityContext = &v1.SecurityContext{
AllowPrivilegeEscalation: ptr.To(false),
ReadOnlyRootFilesystem: ptr.To(true),
Capabilities: &v1.Capabilities{
Drop: []v1.Capability{"ALL"},
},
}
} else {
switch *podSecurityLabel {
case "restricted":
securityContext = &v1.SecurityContext{
AllowPrivilegeEscalation: ptr.To(false),
RunAsNonRoot: ptr.To(true),
SeccompProfile: &v1.SeccompProfile{
Type: "RuntimeDefault",
},
Capabilities: &v1.Capabilities{
Drop: []v1.Capability{"ALL"},
},
}
case "baseline":
securityContext = &v1.SecurityContext{}
default:
securityContext = &v1.SecurityContext{}
}
}
operatorContainers := append([]v1.Container{
{
Name: "prometheus",
Expand All @@ -425,21 +455,17 @@ func makeStatefulSetSpec(
ReadinessProbe: readinessProbe,
Resources: cpf.Resources,
TerminationMessagePolicy: v1.TerminationMessageFallbackToLogsOnError,
SecurityContext: &v1.SecurityContext{
ReadOnlyRootFilesystem: ptr.To(true),
AllowPrivilegeEscalation: ptr.To(false),
Capabilities: &v1.Capabilities{
Drop: []v1.Capability{"ALL"},
},
},
SecurityContext: securityContext,
},
prompkg.BuildConfigReloader(
p,
c,
false,
configReloaderVolumeMounts,
watchedDirectories,
podSecurityLabel,
operator.Shard(shard),

operator.WebConfigFile(configReloaderWebConfigFile),
),
}, additionalContainers...)
Expand Down Expand Up @@ -611,6 +637,7 @@ func createThanosContainer(
p monitoringv1.PrometheusInterface,
thanos *monitoringv1.ThanosSpec,
c *prompkg.Config,
podSecurityLabel *string,
) (*v1.Container, error) {
var container *v1.Container
cpf := p.GetCommonPrometheusFields()
Expand Down Expand Up @@ -656,19 +683,41 @@ func createThanosContainer(
thanosArgs = append(thanosArgs, monitoringv1.Argument{Name: "grpc-server-tls-client-ca", Value: tls.CAFile})
}
}
securityContext := &v1.SecurityContext{}
if podSecurityLabel == nil {
securityContext = &v1.SecurityContext{
AllowPrivilegeEscalation: ptr.To(false),
ReadOnlyRootFilesystem: ptr.To(true),
Capabilities: &v1.Capabilities{
Drop: []v1.Capability{"ALL"},
},
}
} else {
switch *podSecurityLabel {
case "restricted":
securityContext = &v1.SecurityContext{
AllowPrivilegeEscalation: ptr.To(false),
RunAsNonRoot: ptr.To(true),
SeccompProfile: &v1.SeccompProfile{
Type: "RuntimeDefault",
},
Capabilities: &v1.Capabilities{
Drop: []v1.Capability{"ALL"},
},
}
case "baseline":
securityContext = &v1.SecurityContext{}
default:
securityContext = &v1.SecurityContext{}
}
}

container = &v1.Container{
Name: "thanos-sidecar",
Image: thanosImage,
ImagePullPolicy: cpf.ImagePullPolicy,
TerminationMessagePolicy: v1.TerminationMessageFallbackToLogsOnError,
SecurityContext: &v1.SecurityContext{
AllowPrivilegeEscalation: ptr.To(false),
ReadOnlyRootFilesystem: ptr.To(true),
Capabilities: &v1.Capabilities{
Drop: []v1.Capability{"ALL"},
},
},
SecurityContext: securityContext,
Ports: []v1.ContainerPort{
{
Name: "http",
Expand Down
5 changes: 3 additions & 2 deletions pkg/prometheus/statefulset.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -412,6 +412,7 @@ func BuildConfigReloader(
initContainer bool,
mounts []v1.VolumeMount,
watchedDirectories []string,
podSecurityLabel *string,
opts ...operator.ReloaderOption,
) v1.Container {
cpf := p.GetCommonPrometheusFields()
Expand All @@ -432,7 +433,7 @@ func BuildConfigReloader(
if initContainer {
name = "init-config-reloader"
reloaderOptions = append(reloaderOptions, operator.ReloaderRunOnce())
return operator.CreateConfigReloader(name, reloaderOptions...)
return operator.CreateConfigReloader(name, podSecurityLabel, reloaderOptions...)
}

if ptr.Deref(cpf.ReloadStrategy, monitoringv1.HTTPReloadStrategyType) == monitoringv1.ProcessSignalReloadStrategyType {
Expand Down Expand Up @@ -460,7 +461,7 @@ func BuildConfigReloader(
)
}

return operator.CreateConfigReloader(name, reloaderOptions...)
return operator.CreateConfigReloader(name, podSecurityLabel, reloaderOptions...)
}

func ShareProcessNamespace(p monitoringv1.PrometheusInterface) *bool {
Expand Down

0 comments on commit e6ed494

Please sign in to comment.