core: run operator with rook user

The rook operator as well as the toolbox pod run with the "rook" user
with UID 2016. The UID was chosen based on the year of the initial
commit in the rook/rook repository.
No more root user running.

Closes: #8734
Signed-off-by: Sébastien Han <seb@redhat.com>

PendingReleaseNotes.md

@@ -20,3 +20,4 @@ v1.8...
 - Rook adds a finalizer `ceph.rook.io/disaster-protection` to resources critical to the Ceph cluster
   (rook-ceph-mon secrets and configmap) so that the resources will not be accidentally deleted.
 - Add support for [Kubernetes Authentication when using HashiCorp Vault Key Management Service](Documentation/ceph-kms.md##kubernetes-based-authentication).
+- The Rook Operator and the toolbox now run under the "rook" user and does not use "root" anymore.

cluster/charts/rook-ceph/templates/deployment.yaml

@@ -26,6 +26,10 @@ spec:
         image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
         imagePullPolicy: {{ .Values.image.pullPolicy }}
         args: ["ceph", "operator"]
+        securityContext:
+          runAsNonRoot: true
+          runAsUser: 2016
+          runAsGroup: 2016
         volumeMounts:
         - mountPath: /var/lib/rook
           name: rook-config

cluster/examples/kubernetes/ceph/operator-openshift.yaml

@@ -439,12 +439,16 @@ spec:
         - name: rook-ceph-operator
           image: rook/ceph:master
           args: ["ceph", "operator"]
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 2016
+            runAsGroup: 2016
           volumeMounts:
             - mountPath: /var/lib/rook
               name: rook-config
             - mountPath: /etc/ceph
               name: default-config-dir
-            - mountPath: /etc/webhook
+            - mountPath: /tmp/k8s-webhook-server/serving-certs
               name: webhook-cert
           ports:
             - containerPort: 9443

cluster/examples/kubernetes/ceph/operator.yaml

@@ -356,12 +356,16 @@ spec:
         - name: rook-ceph-operator
           image: rook/ceph:master
           args: ["ceph", "operator"]
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 2016
+            runAsGroup: 2016
           volumeMounts:
             - mountPath: /var/lib/rook
               name: rook-config
             - mountPath: /etc/ceph
               name: default-config-dir
-            - mountPath: /etc/webhook
+            - mountPath: /tmp/k8s-webhook-server/serving-certs
               name: webhook-cert
           ports:
             - containerPort: 9443

cluster/examples/kubernetes/ceph/toolbox.yaml

@@ -23,6 +23,10 @@ spec:
           args: ["-m", "-c", "/usr/local/bin/toolbox.sh"]
           imagePullPolicy: IfNotPresent
           tty: true
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 2016
+            runAsGroup: 2016
           env:
             - name: ROOK_CEPH_USERNAME
               valueFrom:

images/ceph/Dockerfile

@@ -20,5 +20,10 @@ COPY rook toolbox.sh set-ceph-debug-level /usr/local/bin/
 COPY ceph-monitoring /etc/ceph-monitoring
 COPY rook-external /etc/rook-external/
 COPY ceph-csv-templates /etc/ceph-csv-templates
+RUN useradd rook -u 2016 # 2016 is the UID of the rook user and also the year of the first commit in the project
+RUN mkdir -p /var/lib/rook /etc/webhook
+RUN chown rook:rook /var/lib/rook /etc/webhook
+RUN chmod 755 /var/lib/rook /etc/webhook
+USER 2016
 ENTRYPOINT ["/usr/local/bin/rook"]
 CMD [""]

pkg/daemon/ceph/osd/kms/volumes.go

@@ -42,8 +42,10 @@ func TLSSecretVolumeAndMount(config map[string]string) []v1.VolumeProjection {
 	// Projection list
 	secretVolumeProjections := []v1.VolumeProjection{}
 
-	// File mode
-	mode := int32(0400)
+	// File mode, anybody can read, this is a must-have since the container runs as "rook" and the
+	// secret is mounted as root. There is no non-ugly way to change this behavior and it's
+	// probably as less safe as doing this mode.
+	mode := int32(0444)
 
 	// Vault TLS Secrets
 	for _, tlsOption := range cephv1.VaultTLSConnectionDetails {

pkg/operator/ceph/cr_manager.go

@@ -18,6 +18,8 @@ package operator
 
 import (
 	"context"
+	"os"
+	"path/filepath"
 
 	"github.com/pkg/errors"
 	"github.com/rook/rook/pkg/clusterd"
@@ -52,57 +54,52 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 )
 
-const (
-	certDir = "/etc/webhook"
-)
-
 var (
 	resourcesSchemeFuncs = []func(*runtime.Scheme) error{
 		clientgoscheme.AddToScheme,
 		mapiv1.AddToScheme,
 		healthchecking.AddToScheme,
 		cephv1.AddToScheme,
 	}
-)
 
-var (
 	webhookResources = []webhook.Validator{&cephv1.CephCluster{}, &cephv1.CephBlockPool{}, &cephv1.CephObjectStore{}}
-)
 
-var (
 	// EnableMachineDisruptionBudget checks whether machine disruption budget is enabled
 	EnableMachineDisruptionBudget bool
-)
 
-// AddToManagerFuncsMaintenance is a list of functions to add all Controllers to the Manager (entrypoint for controller)
-var AddToManagerFuncsMaintenance = []func(manager.Manager, *controllerconfig.Context) error{
-	clusterdisruption.Add,
-}
+	// Default directory where TLS certs are stored - controller runtime's default
+	certDir = filepath.Join(os.TempDir(), "k8s-webhook-server", "serving-certs")
 
-// MachineDisruptionBudgetAddToManagerFuncs is a list of fencing related functions to add all Controllers to the Manager (entrypoint for controller)
-var MachineDisruptionBudgetAddToManagerFuncs = []func(manager.Manager, *controllerconfig.Context) error{
-	machinelabel.Add,
-	machinedisruption.Add,
-}
+	// AddToManagerFuncsMaintenance is a list of functions to add all Controllers to the Manager (entrypoint for controller)
+	AddToManagerFuncsMaintenance = []func(manager.Manager, *controllerconfig.Context) error{
+		clusterdisruption.Add,
+	}
 
-// AddToManagerFuncs is a list of functions to add all Controllers to the Manager (entrypoint for controller)
-var AddToManagerFuncs = []func(manager.Manager, *clusterd.Context, context.Context, opcontroller.OperatorConfig) error{
-	crash.Add,
-	pool.Add,
-	objectuser.Add,
-	realm.Add,
-	zonegroup.Add,
-	zone.Add,
-	object.Add,
-	file.Add,
-	nfs.Add,
-	rbd.Add,
-	client.Add,
-	mirror.Add,
-	Add,
-	csi.Add,
-	bucket.Add,
-}
+	// MachineDisruptionBudgetAddToManagerFuncs is a list of fencing related functions to add all Controllers to the Manager (entrypoint for controller)
+	MachineDisruptionBudgetAddToManagerFuncs = []func(manager.Manager, *controllerconfig.Context) error{
+		machinelabel.Add,
+		machinedisruption.Add,
+	}
+
+	// AddToManagerFuncs is a list of functions to add all Controllers to the Manager (entrypoint for controller)
+	AddToManagerFuncs = []func(manager.Manager, *clusterd.Context, context.Context, opcontroller.OperatorConfig) error{
+		crash.Add,
+		pool.Add,
+		objectuser.Add,
+		realm.Add,
+		zonegroup.Add,
+		zone.Add,
+		object.Add,
+		file.Add,
+		nfs.Add,
+		rbd.Add,
+		client.Add,
+		mirror.Add,
+		Add,
+		csi.Add,
+		bucket.Add,
+	}
+)
 
 // AddToManagerOpFunc is a list of functions to add all Controllers to the Manager (entrypoint for
 // controller)

pkg/operator/ceph/webhook.go

@@ -59,6 +59,9 @@ func isSecretPresent(ctx context.Context, context *clusterd.Context) (bool, erro
 
 	logger.Infof("admission webhook secret %q found", admissionControllerAppName)
 	for k, data := range s.Data {
+		if err = os.MkdirAll(certDir, 0755); err != nil {
+			return false, errors.Wrapf(err, "failed to create webhook certificate directory %q", certDir)
+		}
 		path := fmt.Sprintf("%s/%s", certDir, k)
 		err := ioutil.WriteFile(path, data, 0400)
 		if err != nil {