fix: reload openshift csr-signer ca certificate in Operator trusted C…

…A's (#1742)

When the csr-signer in Openshift rotates, Operator will do a copy of the certificate in a local secret and reload the CA certificates

* Add openshift test to deploy Operator from OperatorHub on minio-operator namespace

* Add empty `securityContext` and `containerSecurityContext` in the Openshift kustomization example.

* Fix to allow the tenant run with locally build operator image in crc test

Signed-off-by: pjuarezd <pjuarezd@users.noreply.github.com>

config/manifests/overlay/minio-operator-deployment.yaml

@@ -17,6 +17,8 @@ spec:
           volumeMounts:
             - name: openshift-service-ca
               mountPath: /tmp/service-ca
+            - name: openshift-csr-signer-ca
+              mountPath: /tmp/csr-signer-ca
             - name: sts-tls
               mountPath: /tmp/sts
       volumes:
@@ -39,3 +41,13 @@ spec:
               - key: service-ca.crt
                 path: service-ca.crt
             optional: true
+        - name: openshift-csr-signer-ca
+          projected:
+            defaultMode: 420
+            sources:
+              - secret:
+                  name: openshift-csr-signer-ca
+                  items:
+                    - key: tls.crt
+                      path: tls.crt
+                  optional: true

examples/kustomization/tenant-openshift/tenant.yaml

@@ -24,3 +24,5 @@ spec:
           resources:
             requests:
               storage: 2Gi
+      securityContext: {}
+      containerSecurityContext: {}

pkg/apis/minio.min.io/v2/helper.go

@@ -103,20 +103,29 @@ var (
 // GetPodCAFromFile assumes the operator is running inside a k8s pod and extract the
 // current ca certificate from /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
 func GetPodCAFromFile() []byte {
-	namespace, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt")
+	cert, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt")
 	if err != nil {
 		return nil
 	}
-	return namespace
+	return cert
 }
 
-// GetPodServiceCAFromFile extracts the service-ca.crt certificate in Openshift deployments coming from configmap openshift-service-ca.crt
-func GetPodServiceCAFromFile() []byte {
-	caPath, err := os.ReadFile("/tmp/service-ca/ca.crt")
+// GetOpenshiftServiceCAFromFile extracts the service-ca.crt certificate in Openshift deployments coming from configmap openshift-service-ca.crt
+func GetOpenshiftServiceCAFromFile() []byte {
+	cert, err := os.ReadFile("/tmp/service-ca/service-ca.crt")
 	if err != nil {
 		return nil
 	}
-	return caPath
+	return cert
+}
+
+// GetOpenshiftCSRSignerCAFromFile extracts the tls.crt certificate in Openshift deployments coming from the mounted secret openshift-csr-signer-ca
+func GetOpenshiftCSRSignerCAFromFile() []byte {
+	cert, err := os.ReadFile("/tmp/csr-signer-ca/tls.crt")
+	if err != nil {
+		return nil
+	}
+	return cert
 }
 
 // GetPublicCertFilePath return the path to the certificate file based for the serviceName

pkg/controller/main-controller.go

@@ -727,6 +727,13 @@ func (c *Controller) syncHandler(key string) (Result, error) {
 
 	namespace, tenantName := key2NamespaceName(key)
 
+	if utils.GetOperatorRuntime() == common.OperatorRuntimeOpenshift {
+		err := c.checkOpenshiftSignerCACertInOperatorNamespace(ctx)
+		if err != nil {
+			klog.Errorf("Error checking openshift-csr-signer-ca secret, %#v", err)
+		}
+	}
+
 	// Get the Tenant resource with this namespace/name
 	tenant, err := c.minioClientSet.MinioV2().Tenants(namespace).Get(context.Background(), tenantName, metav1.GetOptions{})
 	if err != nil {

pkg/controller/operator.go

@@ -17,18 +17,21 @@
 package controller
 
 import (
+	"bytes"
 	"context"
 	"crypto/tls"
 	"crypto/x509"
+	"fmt"
 	"net"
 	"net/http"
 	"time"
 
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+
 	"github.com/minio/operator/pkg/common"
 	"github.com/minio/operator/pkg/utils"
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
-	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
@@ -45,9 +48,15 @@ const (
 	CertPasswordEnv = "OPERATOR_CERT_PASSWD"
 	// OperatorDeploymentNameEnv Env variable to specify a custom deployment name for Operator
 	OperatorDeploymentNameEnv = "MINIO_OPERATOR_DEPLOYMENT_NAME"
-
 	// OperatorCATLSSecretName is the name of the secret for the operator CA
 	OperatorCATLSSecretName = "operator-ca-tls"
+	// OperatorCSRSignerCASecretName is the name of the secret for the signer-ca certificate
+	// this is a copy of the secret signer-ca in namespace
+	OperatorCSRSignerCASecretName = "openshift-csr-signer-ca"
+	// OpenshiftKubeControllerNamespace is the namespace of kube controller manager operator in Openshift
+	OpenshiftKubeControllerNamespace = "openshift-kube-controller-manager-operator"
+	// OpenshiftCATLSSecretName is the secret name of the CRD's signer in kubernetes under  OpenshiftKubeControllerNamespace namespace
+	OpenshiftCATLSSecretName = "csr-signer"
 	// DefaultDeploymentName is the default name of the operator deployment
 	DefaultDeploymentName = "minio-operator"
 	// DefaultOperatorImage is the version fo the operator being used
@@ -137,9 +146,15 @@ func (c *Controller) fetchTransportCACertificates() (pool *x509.CertPool) {
 	// Default kubernetes CA certificate
 	rootCAs.AppendCertsFromPEM(miniov2.GetPodCAFromFile())
 
-	// Openshift Service CA certificate
 	if utils.GetOperatorRuntime() == common.OperatorRuntimeOpenshift {
-		rootCAs.AppendCertsFromPEM(miniov2.GetPodServiceCAFromFile())
+		// Openshift Service CA certificate
+		if serviceCA := miniov2.GetOpenshiftServiceCAFromFile(); serviceCA != nil {
+			rootCAs.AppendCertsFromPEM(serviceCA)
+		}
+		// Openshift csr-signer CA certificate
+		if cert := miniov2.GetOpenshiftCSRSignerCAFromFile(); cert != nil {
+			rootCAs.AppendCertsFromPEM(cert)
+		}
 	}
 
 	// Custom ca certificate to be used by operator
@@ -155,37 +170,114 @@ func (c *Controller) fetchTransportCACertificates() (pool *x509.CertPool) {
 			rootCAs.AppendCertsFromPEM(val)
 		}
 	}
+	return rootCAs
+}
 
-	// These chunk of code is intended for OpenShift ONLY and it will help us trust the signer to solve issue:
-	// https://github.com/minio/operator/issues/1412
-	if utils.GetOperatorRuntime() == common.OperatorRuntimeOpenshift {
-		openShiftCATLSCert, err := c.kubeClientSet.CoreV1().Secrets("openshift-kube-controller-manager-operator").Get(
-			context.Background(), "csr-signer", metav1.GetOptions{})
-		klog.Info("Checking if this is OpenShift Environment to append the certificates...")
-		if err != nil {
-			if k8serrors.IsNotFound(err) {
-				// Do nothing special, because this is maybe k8s vanilla
-				klog.Info("csr-signer secret wasn't found, very likely this is not OpenShift but k8s Vanilla or other...")
-			} else {
-				// Lack of permissions to read the secret
-				klog.Errorf("csr-signer secret was found but we failed to get openShiftCATLSCert: %#v", err)
+// GetSignerCAFromSecret Retrieves the CA certificate for Openshift CSR signed certificates from
+// openshift-kube-controller-manager-operator namespace
+func (c *Controller) GetSignerCAFromSecret() ([]byte, error) {
+	var caCertificate []byte
+	openShiftCATLSCertSecret, err := c.kubeClientSet.CoreV1().Secrets(OpenshiftKubeControllerNamespace).Get(
+		context.Background(), OpenshiftCATLSSecretName, metav1.GetOptions{})
+	if err != nil {
+		if k8serrors.IsNotFound(err) {
+			return nil, fmt.Errorf("%s secret in %s wasn't found", OpenshiftCATLSSecretName, OpenshiftKubeControllerNamespace)
+		}
+		return nil, fmt.Errorf("%s secret was found but we failed to load the secret: %#v", OpenshiftCATLSSecretName, err)
+	} else if openShiftCATLSCertSecret != nil {
+		if val, ok := openShiftCATLSCertSecret.Data[common.TLSCRT]; ok {
+			caCertificate = val
+		}
+	}
+	return caCertificate, nil
+}
+
+// GetOpenshiftCSRSignerCAFromSecret loads the tls certificate in openshift-csr-signer-ca secret in operator namespace
+func (c *Controller) GetOpenshiftCSRSignerCAFromSecret() ([]byte, error) {
+	var caCertificate []byte
+	operatorNamespace := miniov2.GetNSFromFile()
+	openShifCSRSignerCATLSCertSecret, err := c.kubeClientSet.CoreV1().Secrets(operatorNamespace).Get(
+		context.Background(), OperatorCSRSignerCASecretName, metav1.GetOptions{})
+	if err != nil {
+		if k8serrors.IsNotFound(err) {
+			return nil, fmt.Errorf("%s secret wasn't found, Skip load the certificate", OperatorCSRSignerCASecretName)
+		}
+		// Lack of permissions to read the secret
+		return nil, fmt.Errorf("%s secret was found but we failed to get certificate: %#v", OperatorCSRSignerCASecretName, err)
+	} else if openShifCSRSignerCATLSCertSecret != nil {
+		// When secret was obtained with no errors
+		if val, ok := openShifCSRSignerCATLSCertSecret.Data[common.TLSCRT]; ok {
+			// OpenShift csr-signer secret has tls.crt certificates that we need to append in order
+			// to trust the signer. If we append the val, Operator will be able to provisioning the
+			// initial users and get Tenant Health, so tenant can be properly initialized and in
+			// green status, otherwise if we don't append it, it will get stuck and expose this
+			// issue in the log:
+			// Failed to get cluster health: Get "https://minio.tenant-lite.svc.cluster.local/minio/health/cluster":
+			// x509: certificate signed by unknown authority
+			caCertificate = val
+		}
+	}
+	return caCertificate, nil
+}
+
+// checkOpenshiftSignerCACertInOperatorNamespace checks if csr-signer secret in openshift changed and updates or create
+// a copy of the secret in operator namespace
+func (c *Controller) checkOpenshiftSignerCACertInOperatorNamespace(ctx context.Context) error {
+	// get the current certificate from openshift
+	csrSignerCertificate, err := c.GetSignerCAFromSecret()
+	if err != nil {
+		return err
+	}
+	namespace := miniov2.GetNSFromFile()
+	// get openshift-csr-signer-ca secret in minio-operator namespace
+	csrSignerSecret, err := c.kubeClientSet.CoreV1().Secrets(namespace).Get(ctx, OperatorCSRSignerCASecretName, metav1.GetOptions{})
+	if err != nil {
+		// if csrSignerCa doesnt exists create it
+		if k8serrors.IsNotFound(err) {
+			klog.Infof("'%s/%s' secret is missing, creating", namespace, OperatorCSRSignerCASecretName)
+			operatorDeployment, err := c.kubeClientSet.AppsV1().Deployments(namespace).Get(ctx, getOperatorDeploymentName(), metav1.GetOptions{})
+			if err != nil {
+				return err
+			}
+
+			ownerReference := metav1.OwnerReference{
+				APIVersion: appsv1.SchemeGroupVersion.Version,
+				Kind:       "Deployment",
+				Name:       operatorDeployment.Name,
+				UID:        operatorDeployment.UID,
 			}
-		} else if err == nil && openShiftCATLSCert != nil {
-			// When secret was obtained with no errors
-			if val, ok := openShiftCATLSCert.Data["tls.crt"]; ok {
-				// OpenShift csr-signer secret has tls.crt certificates that we need to append in order
-				// to trust the signer. If we append the val, Operator will be able to provisioning the
-				// initial users and get Tenant Health, so tenant can be properly initialized and in
-				// green status, otherwise if we don't append it, it will get stuck and expose this
-				// issue in the log:
-				// Failed to get cluster health: Get "https://minio.tenant-lite.svc.cluster.local/minio/health/cluster":
-				// x509: certificate signed by unknown authority
-				klog.Info("Appending OpenShift csr-signer to trust the Signer")
-				rootCAs.AppendCertsFromPEM(val)
+
+			csrSignerSecret := &v1.Secret{
+				Type: "Opaque",
+				ObjectMeta: metav1.ObjectMeta{
+					Name:            OperatorCSRSignerCASecretName,
+					Namespace:       miniov2.GetNSFromFile(),
+					OwnerReferences: []metav1.OwnerReference{ownerReference},
+				},
+				Data: map[string][]byte{
+					common.TLSCRT: csrSignerCertificate,
+				},
 			}
+			_, err = c.kubeClientSet.CoreV1().Secrets(namespace).Create(ctx, csrSignerSecret, metav1.CreateOptions{})
+			// Reload CA certificates
+			c.createTransport()
+			return err
 		}
+		return err
 	}
-	return rootCAs
+
+	if caCert, ok := csrSignerSecret.Data[common.TLSCRT]; ok && !bytes.Equal(caCert, csrSignerCertificate) {
+		csrSignerSecret.Data[common.TLSCRT] = csrSignerCertificate
+		_, err = c.kubeClientSet.CoreV1().Secrets(namespace).Update(ctx, csrSignerSecret, metav1.UpdateOptions{})
+		if err != nil {
+			return err
+		}
+		klog.Infof("'%s/%s' secret changed, updating '%s/%s' secret", OpenshiftKubeControllerNamespace, OpenshiftCATLSSecretName, namespace, OperatorCSRSignerCASecretName)
+		c.fetchTransportCACertificates()
+		// Reload CA certificates
+		c.createTransport()
+	}
+	return nil
 }
 
 func (c *Controller) createUsers(ctx context.Context, tenant *miniov2.Tenant, tenantConfiguration map[string][]byte) (err error) {

testing/deploy-openshift-4.13.6.sh

@@ -21,16 +21,16 @@ export SCRIPT_DIR
 source "${SCRIPT_DIR}/openshift-common.sh"
 
 function main() {
-
-    install_binaries
 
-    setup_crc "4.13.6"
+	install_binaries
 
-    create_marketplace_catalog "certified-operators"
+	setup_crc "4.13.6"
 
-    install_operator "certified-operators"
+	create_marketplace_catalog "certified-operators"
 
-    #destroy_crc
+	install_operator "certified-operators"
+
+	#destroy_crc
 }
 
 time main "$@"

testing/deploy-openshift-ownnamespace-4.13.6.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Copyright (C) 2023, MinIO, Inc.
+#
+# This code is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3,
+# as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License, version 3,
+# along with this program.  If not, see <http://www.gnu.org/licenses/>
+
+# This script requires: kubectl, kind
+
+SCRIPT_DIR=$(dirname "$0")
+export SCRIPT_DIR
+
+source "${SCRIPT_DIR}/openshift-common.sh"
+
+function main() {
+
+	install_binaries
+
+	setup_crc "4.13.6"
+
+	create_marketplace_catalog "certified-operators"
+
+	install_operator "certified-operators" "minio-operator"
+
+	#destroy_crc
+}
+
+time main "$@"