Console TLS (#1348)

* TLS methods refactor to allow create several Certificates

* Allow enable TLS for Operator console

* Add MINIO_CONSOLE_TLS_ENABLE (default off) env variable to turn on and off operator console TLS

go.mod

@@ -21,12 +21,13 @@ require (
 	golang.org/x/crypto v0.0.0-20221012134737-56aed061732a
 	golang.org/x/time v0.1.0
 	gopkg.in/yaml.v2 v2.4.0
-	k8s.io/api v0.25.3
+	k8s.io/api v0.25.4
 	k8s.io/apiextensions-apiserver v0.25.3
-	k8s.io/apimachinery v0.25.3
-	k8s.io/client-go v0.25.3
+	k8s.io/apimachinery v0.25.4
+	k8s.io/client-go v0.25.4
 	k8s.io/code-generator v0.25.3
 	k8s.io/klog/v2 v2.80.1
+	k8s.io/kubectl v0.25.4
 	sigs.k8s.io/controller-runtime v0.13.0
 )
 

go.sum

@@ -1819,14 +1819,14 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.2.1/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
-k8s.io/api v0.25.3 h1:Q1v5UFfYe87vi5H7NU0p4RXC26PPMT8KOpr1TLQbCMQ=
-k8s.io/api v0.25.3/go.mod h1:o42gKscFrEVjHdQnyRenACrMtbuJsVdP+WVjqejfzmI=
+k8s.io/api v0.25.4 h1:3YO8J4RtmG7elEgaWMb4HgmpS2CfY1QlaOz9nwB+ZSs=
+k8s.io/api v0.25.4/go.mod h1:IG2+RzyPQLllQxnhzD8KQNEu4c4YvyDTpSMztf4A0OQ=
 k8s.io/apiextensions-apiserver v0.25.3 h1:bfI4KS31w2f9WM1KLGwnwuVlW3RSRPuIsfNF/3HzR0k=
 k8s.io/apiextensions-apiserver v0.25.3/go.mod h1:ZJqwpCkxIx9itilmZek7JgfUAM0dnTsA48I4krPqRmo=
-k8s.io/apimachinery v0.25.3 h1:7o9ium4uyUOM76t6aunP0nZuex7gDf8VGwkR5RcJnQc=
-k8s.io/apimachinery v0.25.3/go.mod h1:jaF9C/iPNM1FuLl7Zuy5b9v+n35HGSh6AQ4HYRkCqwo=
-k8s.io/client-go v0.25.3 h1:oB4Dyl8d6UbfDHD8Bv8evKylzs3BXzzufLiO27xuPs0=
-k8s.io/client-go v0.25.3/go.mod h1:t39LPczAIMwycjcXkVc+CB+PZV69jQuNx4um5ORDjQA=
+k8s.io/apimachinery v0.25.4 h1:CtXsuaitMESSu339tfhVXhQrPET+EiWnIY1rcurKnAc=
+k8s.io/apimachinery v0.25.4/go.mod h1:jaF9C/iPNM1FuLl7Zuy5b9v+n35HGSh6AQ4HYRkCqwo=
+k8s.io/client-go v0.25.4 h1:3RNRDffAkNU56M/a7gUfXaEzdhZlYhoW8dgViGy5fn8=
+k8s.io/client-go v0.25.4/go.mod h1:8trHCAC83XKY0wsBIpbirZU4NTUpbuhc2JnI7OruGZw=
 k8s.io/code-generator v0.25.3 h1:BEH+wDi90bGyrYcY4abGtUqaOX7G94RRrEu8l+SvIeo=
 k8s.io/code-generator v0.25.3/go.mod h1:9F5fuVZOMWRme7MYj2YT3L9ropPWPokd9VRhVyD3+0w=
 k8s.io/gengo v0.0.0-20211129171323-c02415ce4185 h1:TT1WdmqqXareKxZ/oNXEUSwKlLiHzPMyB0t8BaFeBYI=
@@ -1837,6 +1837,8 @@ k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4=
 k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/kube-openapi v0.0.0-20220803164354-a70c9af30aea h1:3QOH5+2fGsY8e1qf+GIFpg+zw/JGNrgyZRQR7/m6uWg=
 k8s.io/kube-openapi v0.0.0-20220803164354-a70c9af30aea/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU=
+k8s.io/kubectl v0.25.4 h1:O3OA1z4V1ZyvxCvScjq0pxAP7ABgznr8UvnVObgI6Dc=
+k8s.io/kubectl v0.25.4/go.mod h1:CKMrQ67Bn2YCP26tZStPQGq62zr9pvzEf65A0navm8k=
 k8s.io/utils v0.0.0-20220823124924-e9cbc92d1a73 h1:H9TCJUUx+2VA0ZiD9lvtaX8fthFsMoD+Izn93E/hm8U=
 k8s.io/utils v0.0.0-20220823124924-e9cbc92d1a73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48=

helm/operator/values.yaml

@@ -5,6 +5,8 @@ operator:
   # env:
   #   - name: MINIO_OPERATOR_TLS_ENABLE
   #     value: "off"
+  #   - name: MINIO_CONSOLE_TLS_ENABLE
+  #     value: "off"
   #   - name: CLUSTER_DOMAIN
   #     value: "cluster.domain"
   #   - name: WATCHED_NAMESPACE

kubectl-minio/cmd/helpers/constants.go

@@ -45,7 +45,5 @@ const (
 	DefaultKESImage = "minio/kes:v0.18.0"
 )
 
-var (
-	// KESReplicas is the number of replicas for MinIO KES
-	KESReplicas int32 = 2
-)
+// KESReplicas is the number of replicas for MinIO KES
+var KESReplicas int32 = 2

kubectl-minio/cmd/init.go

@@ -80,6 +80,7 @@ func newInitCmd(out io.Writer, errOut io.Writer) *cobra.Command {
 	f.StringVar(&o.operatorOpts.NSToWatch, "namespace-to-watch", "", "namespace where operator looks for MinIO tenants, leave empty for all namespaces")
 	f.StringVar(&o.operatorOpts.ImagePullSecret, "image-pull-secret", "", "image pull secret to be used for pulling MinIO Operator")
 	f.StringVar(&o.operatorOpts.ConsoleImage, "console-image", "", "console image")
+	f.BoolVar(&o.operatorOpts.ConsoleTLS, "console-tls", false, "enable tls for Operator console")
 	f.StringVar(&o.operatorOpts.TenantMinIOImage, "default-minio-image", "", "default tenant MinIO image")
 	f.StringVar(&o.operatorOpts.TenantConsoleImage, "default-console-image", "", "default tenant Console image")
 	f.StringVar(&o.operatorOpts.TenantKesImage, "default-kes-image", "", "default tenant KES image")
@@ -158,6 +159,16 @@ func (o *operatorInitCmd) run(writer io.Writer) error {
 			},
 		})
 	}
+	if o.operatorOpts.ConsoleTLS {
+		operatorDepPatches = append(operatorDepPatches, opInterface{
+			Op:   "add",
+			Path: "/spec/template/spec/containers/0/env/0",
+			Value: corev1.EnvVar{
+				Name:  "MINIO_CONSOLE_TLS_ENABLE",
+				Value: "on",
+			},
+		})
+	}
 	if o.operatorOpts.TenantMinIOImage != "" {
 		operatorDepPatches = append(operatorDepPatches, opInterface{
 			Op:   "add",

kubectl-minio/cmd/resources/deployment.go

@@ -24,6 +24,7 @@ type OperatorOptions struct {
 	ClusterDomain       string
 	ImagePullSecret     string
 	ConsoleImage        string
+	ConsoleTLS          bool
 	TenantMinIOImage    string
 	TenantConsoleImage  string
 	TenantKesImage      string

kubectl-minio/go.mod

@@ -12,11 +12,11 @@ require (
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.6.0
-	k8s.io/api v0.25.3
+	k8s.io/api v0.25.4
 	k8s.io/apiextensions-apiserver v0.25.3
-	k8s.io/apimachinery v0.25.3
+	k8s.io/apimachinery v0.25.4
 	k8s.io/cli-runtime v0.25.3
-	k8s.io/client-go v0.25.3
+	k8s.io/client-go v0.25.4
 	k8s.io/klog/v2 v2.80.1
 	sigs.k8s.io/kustomize/api v0.12.1
 	sigs.k8s.io/kustomize/kyaml v0.13.9

kubectl-minio/go.sum

@@ -408,16 +408,16 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-k8s.io/api v0.25.3 h1:Q1v5UFfYe87vi5H7NU0p4RXC26PPMT8KOpr1TLQbCMQ=
-k8s.io/api v0.25.3/go.mod h1:o42gKscFrEVjHdQnyRenACrMtbuJsVdP+WVjqejfzmI=
+k8s.io/api v0.25.4 h1:3YO8J4RtmG7elEgaWMb4HgmpS2CfY1QlaOz9nwB+ZSs=
+k8s.io/api v0.25.4/go.mod h1:IG2+RzyPQLllQxnhzD8KQNEu4c4YvyDTpSMztf4A0OQ=
 k8s.io/apiextensions-apiserver v0.25.3 h1:bfI4KS31w2f9WM1KLGwnwuVlW3RSRPuIsfNF/3HzR0k=
 k8s.io/apiextensions-apiserver v0.25.3/go.mod h1:ZJqwpCkxIx9itilmZek7JgfUAM0dnTsA48I4krPqRmo=
-k8s.io/apimachinery v0.25.3 h1:7o9ium4uyUOM76t6aunP0nZuex7gDf8VGwkR5RcJnQc=
-k8s.io/apimachinery v0.25.3/go.mod h1:jaF9C/iPNM1FuLl7Zuy5b9v+n35HGSh6AQ4HYRkCqwo=
+k8s.io/apimachinery v0.25.4 h1:CtXsuaitMESSu339tfhVXhQrPET+EiWnIY1rcurKnAc=
+k8s.io/apimachinery v0.25.4/go.mod h1:jaF9C/iPNM1FuLl7Zuy5b9v+n35HGSh6AQ4HYRkCqwo=
 k8s.io/cli-runtime v0.25.3 h1:Zs7P7l7db/5J+KDePOVtDlArAa9pZXaDinGWGZl0aM8=
 k8s.io/cli-runtime v0.25.3/go.mod h1:InHHsjkyW5hQsILJGpGjeruiDZT/R0OkROQgD6GzxO4=
-k8s.io/client-go v0.25.3 h1:oB4Dyl8d6UbfDHD8Bv8evKylzs3BXzzufLiO27xuPs0=
-k8s.io/client-go v0.25.3/go.mod h1:t39LPczAIMwycjcXkVc+CB+PZV69jQuNx4um5ORDjQA=
+k8s.io/client-go v0.25.4 h1:3RNRDffAkNU56M/a7gUfXaEzdhZlYhoW8dgViGy5fn8=
+k8s.io/client-go v0.25.4/go.mod h1:8trHCAC83XKY0wsBIpbirZU4NTUpbuhc2JnI7OruGZw=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
 k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4=
 k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=

main.go

@@ -44,6 +44,13 @@ import (
 	"k8s.io/client-go/kubernetes"
 )
 
+const (
+	// OperatorWatchedNamespaceEnv Env variable name, the namespaces which the operator watches for MinIO tenants. Defaults to "" for all namespaces.
+	OperatorWatchedNamespaceEnv = "WATCHED_NAMESPACE"
+	// HostnameEnv Host name env variable
+	HostnameEnv = "HOSTNAME"
+)
+
 // version provides the version of this operator
 var version = "DEVELOPMENT.GOGET"
 
@@ -110,7 +117,7 @@ func main() {
 	}
 
 	// Get a comma separated list of namespaces to watch
-	namespacesENv, isNamespaced := os.LookupEnv("WATCHED_NAMESPACE")
+	namespacesENv, isNamespaced := os.LookupEnv(OperatorWatchedNamespaceEnv)
 	var namespaces set.StringSet
 	if isNamespaced {
 		namespaces = set.NewStringSet()
@@ -174,9 +181,9 @@ func main() {
 
 	kubeInformerFactory := kubeinformers.NewSharedInformerFactory(kubeClient, time.Second*30)
 	minioInformerFactory := informers.NewSharedInformerFactory(controllerClient, time.Second*30)
-	podName := os.Getenv("HOSTNAME")
+	podName := os.Getenv(HostnameEnv)
 	if podName == "" {
-		klog.Info("Could not determine $HOSTNAME, defaulting to pod name: operator-pod")
+		klog.Info("Could not determine $%s, defaulting to pod name: operator-pod", HostnameEnv)
 		podName = "operator-pod"
 	}
 

pkg/controller/cluster/console.go

@@ -18,9 +18,11 @@ package cluster
 
 import (
 	"context"
+	"os"
 
 	miniov2 "github.com/minio/operator/pkg/apis/minio.min.io/v2"
 	"github.com/minio/operator/pkg/resources/services"
+	"github.com/minio/pkg/env"
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -29,6 +31,15 @@ import (
 	"k8s.io/klog/v2"
 )
 
+const (
+	// ConsoleTLSEnv Env variable to turn on / off Console TLS.
+	ConsoleTLSEnv = "MINIO_CONSOLE_TLS_ENABLE"
+	// DefaultConsoleDeploymentName is the default name of the console deployment
+	DefaultConsoleDeploymentName = "console"
+	// OperatorConsoleTLSSecretName is the name of secret created with TLS certs for Operator console
+	OperatorConsoleTLSSecretName = "operator-console-tls"
+)
+
 // checkConsoleSvc validates the existence of the MinIO service and validate it's status against what the specification
 // states
 func (c *Controller) checkConsoleSvc(ctx context.Context, tenant *miniov2.Tenant, nsName types.NamespacedName) error {
@@ -87,3 +98,65 @@ func (c *Controller) checkConsoleSvc(ctx context.Context, tenant *miniov2.Tenant
 	}
 	return err
 }
+
+// generateConsoleTLSCert Issues the Operator Console TLS Certificate
+func (c *Controller) generateConsoleTLSCert() (*string, *string) {
+	return c.generateTLSCert("console", OperatorConsoleTLSSecretName, getConsoleDeploymentName())
+}
+
+func (c *Controller) recreateOperatorConsoleCertsIfRequired(ctx context.Context) error {
+	namespace := miniov2.GetNSFromFile()
+	operatorConsoleTLSSecret, err := c.getTLSSecret(ctx, namespace, OperatorConsoleTLSSecretName)
+	if err != nil {
+		if k8serrors.IsNotFound(err) {
+			klog.V(2).Info("TLS certificate not found. Generating one.")
+			// Generate new certificate KeyPair for Operator Console
+			c.generateConsoleTLSCert()
+			return nil
+		}
+		return err
+	}
+
+	needsRenewal, err := c.certNeedsRenewal(operatorConsoleTLSSecret)
+	if err != nil {
+		return err
+	}
+
+	if !needsRenewal {
+		return nil
+	}
+
+	// Expired cert. Delete the secret + CSR and re-create the cert
+	err = c.deleteCSR(ctx, consoleCSRName())
+	if err != nil {
+		return err
+	}
+	klog.V(2).Info("Deleting the TLS secret of expired console cert")
+	err = c.kubeClientSet.CoreV1().Secrets(namespace).Delete(ctx, OperatorConsoleTLSSecretName, metav1.DeleteOptions{})
+	if err != nil {
+		return err
+	}
+
+	klog.V(2).Info("Generating a fresh TLS certificate for Console")
+	// Generate new certificate KeyPair for Operator Console
+	c.generateConsoleTLSCert()
+
+	return nil
+}
+
+// isOperatorConsoleTLS Internal func, reads MINIO_OPERATOR_TLS_ENABLE ENV to identify if Operator Console TLS is enabled, default "off"
+// **WARNING** This will change and will be default to "on" in operator v5
+func isOperatorConsoleTLS() bool {
+	value, set := os.LookupEnv(ConsoleTLSEnv)
+	// By default, Console TLS is NOT used.
+	return (set && value == "on")
+}
+
+func getConsoleDeploymentName() string {
+	return env.Get("MINIO_CONSOLE_DEPLOYMENT_NAME", DefaultConsoleDeploymentName)
+}
+
+// consoleCSRName Internal func returns the given console CSR name
+func consoleCSRName() string {
+	return getCSRName("console")
+}

pkg/controller/cluster/main-controller.go

@@ -16,7 +16,6 @@ package cluster
 
 import (
 	"context"
-	"crypto/tls"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -344,14 +343,16 @@ func (c *Controller) Start(threadiness int, stopCh <-chan struct{}) error {
 		apiServerWillStart := make(chan interface{})
 		// we need to make sure the HTTP Upgrade server is ready before starting operator
 		upgradeServerWillStart := make(chan interface{})
+		// pausing the process until console has it's TLS certificate (if enabled)
+		consoleTLS := make(chan interface{})
 
 		go func() {
 			// Request kubernetes version from Kube ApiServer
 			apiCsrVersion := certificates.GetCertificatesAPIVersion(c.kubeClientSet)
 			klog.Infof("Using Kubernetes CSR Version: %s", apiCsrVersion)
 
 			if isOperatorTLS() {
-				publicCertPath, publicKeyPath := c.generateTLSCert()
+				publicCertPath, publicKeyPath := c.generateOperatorTLSCert()
 				klog.Infof("Starting HTTPS API server")
 				close(apiServerWillStart)
 				certsManager, err := xcerts.NewManager(ctx, *publicCertPath, *publicKeyPath, LoadX509KeyPair)
@@ -360,21 +361,7 @@ func (c *Controller) Start(threadiness int, stopCh <-chan struct{}) error {
 					panic(err)
 				}
 				serverCertsManager = certsManager
-				c.ws.TLSConfig = &tls.Config{
-					PreferServerCipherSuites: true,
-					CurvePreferences:         []tls.CurveID{tls.CurveP256},
-					NextProtos:               []string{"h2", "http/1.1"},
-					MinVersion:               tls.VersionTLS12,
-					CipherSuites: []uint16{
-						tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
-						tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
-						tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
-						tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
-						tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
-						tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
-					},
-					GetCertificate: certsManager.GetCertificate,
-				}
+				c.ws.TLSConfig = c.createTLSConfig(serverCertsManager)
 				if err := c.ws.ListenAndServeTLS("", ""); err != http.ErrServerClosed {
 					klog.Infof("HTTPS server ListenAndServeTLS failed: %v", err)
 					panic(err)
@@ -399,12 +386,35 @@ func (c *Controller) Start(threadiness int, stopCh <-chan struct{}) error {
 			}
 		}()
 
+		go func() {
+			klog.Infof("Starting console TLS certificate setup")
+			if isOperatorConsoleTLS() {
+				klog.Infof("Console TLS enabled")
+				err := c.recreateOperatorConsoleCertsIfRequired(ctx)
+				close(consoleTLS)
+				if err != nil {
+					panic(err)
+				}
+				klog.Infof("Restarting Console pods")
+				err = c.rolloutRestartDeployment(getConsoleDeploymentName())
+				if err != nil {
+					klog.Errorf("Console deployment didn't restart: %s", err)
+				}
+			} else {
+				klog.Infof("Console TLS is not enabled")
+				close(consoleTLS)
+			}
+		}()
+
 		klog.Info("Waiting for API to start")
 		<-apiServerWillStart
 
 		klog.Info("Waiting for Upgrade Server to start")
 		<-upgradeServerWillStart
 
+		klog.Info("Waiting for Console TLS")
+		<-consoleTLS
+
 		// Start the informer factories to begin populating the informer caches
 		klog.Info("Starting Tenant controller")
 

pkg/controller/cluster/minio.go

@@ -66,13 +66,22 @@ func (c *Controller) checkAndCreateMinIOCSR(ctx context.Context, nsName types.Na
 	return nil
 }
 
+// deleteCSR Removes a CSR
 func (c *Controller) deleteCSR(ctx context.Context, csrName string) error {
 	if certificates.GetCertificatesAPIVersion(c.kubeClientSet) == certificates.CSRV1 {
 		if err := c.kubeClientSet.CertificatesV1().CertificateSigningRequests().Delete(ctx, csrName, metav1.DeleteOptions{}); err != nil {
+			// CSR have a short time live, we should not return error when a NotFound is thrown
+			// https://kubernetes.io/docs/reference/access-authn-authz/certificate-signing-requests/#request-signing-process
+			if k8serrors.IsNotFound(err) {
+				return nil
+			}
 			return err
 		}
 	} else {
 		if err := c.kubeClientSet.CertificatesV1beta1().CertificateSigningRequests().Delete(ctx, csrName, metav1.DeleteOptions{}); err != nil {
+			if k8serrors.IsNotFound(err) {
+				return nil
+			}
 			return err
 		}
 	}
@@ -298,13 +307,7 @@ func (c *Controller) certNeedsRenewal(tlsSecret *corev1.Secret) (bool, error) {
 	var certPublicKey []byte
 	var certPrivateKey []byte
 
-	publicKey := "public.crt"
-	privateKey := "private.key"
-
-	if tlsSecret.Type == "kubernetes.io/tls" || tlsSecret.Type == "cert-manager.io/v1alpha2" || tlsSecret.Type == "cert-manager.io/v1" {
-		publicKey = "tls.crt"
-		privateKey = "tls.key"
-	}
+	publicKey, privateKey := c.getKeyNames(tlsSecret)
 
 	if _, exist := tlsSecret.Data[publicKey]; !exist {
 		return false, fmt.Errorf("missing '%s' in %s/%s secret", publicKey, tlsSecret.Namespace, tlsSecret.Name)