Bug 1868976: jsonnet: configure SCCs

assets/alertmanager/alertmanager.yaml

@@ -112,6 +112,9 @@ spec:
   - alertmanager-main-tls
   - alertmanager-main-proxy
   - alertmanager-kube-rbac-proxy
-  securityContext: {}
+  securityContext:
+    fsGroup: 65534
+    runAsNonRoot: true
+    runAsUser: 65534
   serviceAccountName: alertmanager-main
   version: v0.21.0

assets/alertmanager/cluster-role.yaml

@@ -15,3 +15,11 @@ rules:
   - subjectaccessreviews
   verbs:
   - create
+- apiGroups:
+  - security.openshift.io
+  resourceNames:
+  - nonroot
+  resources:
+  - securitycontextconstraints
+  verbs:
+  - use

assets/prometheus-k8s/cluster-role.yaml

@@ -31,3 +31,11 @@ rules:
   - namespaces
   verbs:
   - get
+- apiGroups:
+  - security.openshift.io
+  resourceNames:
+  - nonroot
+  resources:
+  - securitycontextconstraints
+  verbs:
+  - use

assets/prometheus-k8s/prometheus.yaml

@@ -155,7 +155,10 @@ spec:
   - prometheus-k8s-proxy
   - prometheus-k8s-htpasswd
   - kube-rbac-proxy
-  securityContext: {}
+  securityContext:
+    fsGroup: 65534
+    runAsNonRoot: true
+    runAsUser: 65534
   serviceAccountName: prometheus-k8s
   serviceMonitorNamespaceSelector: {}
   serviceMonitorSelector: {}

assets/prometheus-user-workload/cluster-role.yaml

@@ -47,3 +47,11 @@ rules:
   - alertmanagers
   verbs:
   - get
+- apiGroups:
+  - security.openshift.io
+  resourceNames:
+  - nonroot
+  resources:
+  - securitycontextconstraints
+  verbs:
+  - use

assets/prometheus-user-workload/prometheus.yaml

@@ -101,7 +101,10 @@ spec:
       openshift.io/prometheus-rule-evaluation-scope: leaf-prometheus
   secrets:
   - prometheus-user-workload-tls
-  securityContext: {}
+  securityContext:
+    fsGroup: 65534
+    runAsNonRoot: true
+    runAsUser: 65534
   serviceAccountName: prometheus-user-workload
   serviceMonitorNamespaceSelector: {}
   serviceMonitorSelector: {}

assets/thanos-ruler/cluster-role.yaml

@@ -15,3 +15,11 @@ rules:
   - subjectaccessreviews
   verbs:
   - create
+- apiGroups:
+  - security.openshift.io
+  resourceNames:
+  - nonroot
+  resources:
+  - securitycontextconstraints
+  verbs:
+  - use

assets/thanos-ruler/thanos-ruler.yaml

@@ -84,6 +84,10 @@ spec:
       operator: NotIn
       values:
       - leaf-prometheus
+  securityContext:
+    fsGroup: 65534
+    runAsNonRoot: true
+    runAsUser: 65534
   serviceAccountName: thanos-ruler
   volumes:
   - configmap:

jsonnet/alertmanager.jsonnet

@@ -21,6 +21,20 @@ local authorizationRole = policyRule.new() +
                           ]) +
                           policyRule.withVerbs(['create']);
 
+// By default authenticated service accounts are assigned to the `restricted` SCC which implies MustRunAsRange.
+// This is problematic with statefulsets as UIDs (and file permissions) can change if SCCs are elevated.
+// Instead, this sets the `nonroot` SCC in conjunction with a static fsGroup and runAsUser security context below
+// to be immune against UID changes.
+local sccRole = policyRule.new() +
+                policyRule.withApiGroups(['security.openshift.io']) +
+                policyRule.withResources([
+                  'securitycontextconstraints',
+                ]) +
+                policyRule.withResourceNames([
+                  'nonroot',
+                ]) +
+                policyRule.withVerbs(['use']);
+
 {
   alertmanager+:: {
 
@@ -96,7 +110,7 @@ local authorizationRole = policyRule.new() +
     // requires the `create` action on both of these.
 
     clusterRole:
-      local rules = [authenticationRole, authorizationRole];
+      local rules = [authenticationRole, authorizationRole, sccRole];
 
       clusterRole.new() +
       clusterRole.mixin.metadata.withName('alertmanager-main') +
@@ -160,7 +174,11 @@ local authorizationRole = policyRule.new() +
     alertmanager+:
       {
         spec+: {
-          securityContext: {},
+          securityContext: {
+            fsGroup: 65534,
+            runAsNonRoot: true,
+            runAsUser: 65534,
+          },
           priorityClassName: 'system-cluster-critical',
           secrets: [
             'alertmanager-main-tls',

jsonnet/prometheus-user-workload.jsonnet

@@ -54,6 +54,22 @@ local alertmanagerRole =
   ]) +
   policyRule.withVerbs(['get']);
 
+// By default authenticated service accounts are assigned to the `restricted` SCC which implies MustRunAsRange.
+// This is problematic with statefulsets as UIDs (and file permissions) can change if SCCs are elevated.
+// Instead, this sets the `nonroot` SCC in conjunction with a static fsGroup and runAsUser security context below
+// to be immune against UID changes.
+local sccRole =
+  policyRule.new() +
+  policyRule.withApiGroups(['security.openshift.io']) +
+  policyRule.withResources([
+    'securitycontextconstraints',
+  ]) +
+  policyRule.withResourceNames([
+    'nonroot',
+  ]) +
+  policyRule.withVerbs(['use']);
+
+
 {
   prometheusUserWorkload+:: $.prometheus {
     name:: 'user-workload',
@@ -104,6 +120,7 @@ local alertmanagerRole =
         namespacesRole,
         discoveryRole,
         alertmanagerRole,
+        sccRole,
       ]),
 
     // This avoids creating service monitors which are already managed by the respective operators.
@@ -197,7 +214,11 @@ local alertmanagerRole =
               cpu: '6m',
             },
           },
-          securityContext: {},
+          securityContext: {
+            fsGroup: 65534,
+            runAsNonRoot: true,
+            runAsUser: 65534,
+          },
           secrets: [
             'prometheus-user-workload-tls',
           ],

jsonnet/prometheus.jsonnet

@@ -33,6 +33,21 @@ local namespacesRole =
   ]) +
   policyRule.withVerbs(['get']);
 
+// By default authenticated service accounts are assigned to the `restricted` SCC which implies MustRunAsRange.
+// This is problematic with statefulsets as UIDs (and file permissions) can change if SCCs are elevated.
+// Instead, this sets the `nonroot` SCC in conjunction with a static fsGroup and runAsUser security context below
+// to be immune against UID changes.
+local sccRole =
+  policyRule.new() +
+  policyRule.withApiGroups(['security.openshift.io']) +
+  policyRule.withResources([
+    'securitycontextconstraints',
+  ]) +
+  policyRule.withResourceNames([
+    'nonroot',
+  ]) +
+  policyRule.withVerbs(['use']);
+
 {
   prometheusK8s+:: {
     trustedCaBundle:
@@ -120,7 +135,7 @@ local namespacesRole =
     // SubjectAccessReview required by the Alertmanager instances.
 
     clusterRole+:
-      clusterRole.withRulesMixin([authenticationRole, authorizationRole, namespacesRole]),
+      clusterRole.withRulesMixin([authenticationRole, authorizationRole, namespacesRole, sccRole]),
 
     // The proxy secret is there to encrypt session created by the oauth proxy.
 
@@ -241,7 +256,7 @@ local namespacesRole =
       },
 
     // TODO: Adding this to our stack is not as easy
-    // as sidecar listens on 127.0.0.1:10902 and we 
+    // as sidecar listens on 127.0.0.1:10902 and we
     // need kube-rbac-proxy in front of it.
 
     serviceMonitorThanosSidecar:: null,
@@ -287,7 +302,11 @@ local namespacesRole =
               cpu: '70m',
             },
           },
-          securityContext: {},
+          securityContext: {
+            fsGroup: 65534,
+            runAsNonRoot: true,
+            runAsUser: 65534,
+          },
           secrets+: [
             'prometheus-k8s-tls',
             'prometheus-k8s-proxy',

jsonnet/thanos-ruler.jsonnet

@@ -25,6 +25,20 @@ local authorizationRole =
   ]) +
   policyRule.withVerbs(['create']);
 
+// By default authenticated service accounts are assigned to the `restricted` SCC which implies MustRunAsRange.
+// This is problematic with statefulsets as UIDs (and file permissions) can change if SCCs are elevated.
+// Instead, this sets the `nonroot` SCC in conjunction with a static fsGroup and runAsUser security context below
+// to be immune against UID changes.
+local sccRole = policyRule.new() +
+                policyRule.withApiGroups(['security.openshift.io']) +
+                policyRule.withResources([
+                  'securitycontextconstraints',
+                ]) +
+                policyRule.withResourceNames([
+                  'nonroot',
+                ]) +
+                policyRule.withVerbs(['use']);
+
 local thanosRulerRules =
   (import 'github.com/thanos-io/thanos/mixin/alerts/rule.libsonnet') {
     rule+:: {
@@ -95,7 +109,7 @@ local thanosRulerRules =
       clusterRole:
         clusterRole.new() +
         clusterRole.mixin.metadata.withName('thanos-ruler') +
-        clusterRole.withRules([authenticationRole, authorizationRole]),
+        clusterRole.withRules([authenticationRole, authorizationRole, sccRole]),
 
       clusterRoleBinding:
         local clusterRoleBinding = k.rbac.v1.clusterRoleBinding;
@@ -269,6 +283,11 @@ local thanosRulerRules =
           },
         },
         spec: {
+          securityContext: {
+            fsGroup: 65534,
+            runAsNonRoot: true,
+            runAsUser: 65534,
+          },
           replicas: 2,
           image: $._config.imageRepos.openshiftThanos + ':' + $._config.versions.openshiftThanos,
           grpcServerTlsConfig: {

manifests/0000_50_cluster-monitoring-operator_02-role.yaml

@@ -167,6 +167,14 @@ rules:
   - subjectaccessreviews
   verbs:
   - create
+- apiGroups:
+  - security.openshift.io
+  resourceNames:
+  - nonroot
+  resources:
+  - securitycontextconstraints
+  verbs:
+  - use
 - apiGroups:
   - ''
   resources: