ceph: add support for tls certs via k8s tls secrets for rgw

With this PR the RGW can accept TLS certs as K8s TLS secrets

Fixes: 2079
Signed-off-by: Jiffin Tony Thottan <thottanjiffin@gmail.com>

Documentation/ceph-object-store-crd.md

@@ -91,7 +91,7 @@ When the `zone` section is set pools with the object stores name will not be cre
 The gateway settings correspond to the RGW daemon settings.
 
 * `type`: `S3` is supported
-* `sslCertificateRef`: If specified, this is the name of the Kubernetes secret that contains the TLS certificate to be used for secure connections to the object store. Rook will look in the secret provided at the `cert` key name. The value of the `cert` key must be in the format expected by the [RGW service](https://docs.ceph.com/docs/master/install/ceph-deploy/install-ceph-gateway/#using-ssl-with-civetweb): "The server key, server certificate, and any other CA or intermediate certificates be supplied in one file. Each of these items must be in PEM form."
+* `sslCertificateRef`: If specified, this is the name of the Kubernetes secret(`opaque` or `tls` type) that contains the TLS certificate to be used for secure connections to the object store. Rook will look in the secret provided at the `cert` key name. The value of the `cert` key must be in the format expected by the [RGW service](https://docs.ceph.com/docs/master/install/ceph-deploy/install-ceph-gateway/#using-ssl-with-civetweb): "The server key, server certificate, and any other CA or intermediate certificates be supplied in one file. Each of these items must be in PEM form."
 * `port`: The port on which the Object service will be reachable. If host networking is enabled, the RGW daemons will also listen on that port. If running on SDN, the RGW daemon listening port will be 8080 internally.
 * `securePort`: The secure port on which RGW pods will be listening. A TLS certificate must be specified either via `sslCerticateRef` or `service.annotations`
 * `instances`: The number of pods that will be started to load balance this object store.

PendingReleaseNotes.md

@@ -31,3 +31,4 @@ v1.7...
   - Checkout the [ceph docs](https://docs.ceph.com/en/latest/rados/operations/crush-map/#custom-crush-rules)
     for detailed information.
 - Add support cephfs mirroring peer configuration, refer to the [configuration](Documentation/ceph-filesystem-crd.md#mirroring) for more details
+- Add support for Kubernetes TLS secret for referring TLS certs needed for ceph RGW server. 

pkg/operator/ceph/object/config.go

@@ -26,6 +26,7 @@ import (
 	"github.com/pkg/errors"
 	cephconfig "github.com/rook/rook/pkg/operator/ceph/config"
 	"github.com/rook/rook/pkg/operator/ceph/config/keyring"
+	v1 "k8s.io/api/core/v1"
 )
 
 const (
@@ -71,7 +72,8 @@ func (c *clusterConfig) portString() string {
 			portString = fmt.Sprintf("ssl_port=%d ssl_certificate=%s",
 				c.store.Spec.Gateway.SecurePort, certPath)
 		}
-		if c.store.Spec.GetServiceServingCert() != "" {
+		secretType, _ := c.rgwTLSSecretType()
+		if c.store.Spec.GetServiceServingCert() != "" || secretType == v1.SecretTypeTLS {
 			privateKey := path.Join(certDir, certKeyFileName)
 			portString = fmt.Sprintf("%s ssl_private_key=%s", portString, privateKey)
 		}

pkg/operator/ceph/object/config_test.go

@@ -20,12 +20,14 @@ import (
 	"testing"
 
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+	"github.com/rook/rook/pkg/clusterd"
 	cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
 	cephver "github.com/rook/rook/pkg/operator/ceph/version"
+	"github.com/rook/rook/pkg/operator/test"
 	"github.com/stretchr/testify/assert"
 )
 
-func newConfig() *clusterConfig {
+func newConfig(t *testing.T) *clusterConfig {
 	clusterInfo := &cephclient.ClusterInfo{
 		CephVersion: cephver.Nautilus,
 	}
@@ -41,32 +43,33 @@ func newConfig() *clusterConfig {
 			}},
 		clusterInfo: clusterInfo,
 		clusterSpec: clusterSpec,
+		context:     &clusterd.Context{Clientset: test.New(t, 3)},
 	}
 }
 
 func TestPortString(t *testing.T) {
 	// No port or secure port on beast
-	cfg := newConfig()
+	cfg := newConfig(t)
 	result := cfg.portString()
 	assert.Equal(t, "", result)
 
 	// Insecure port on beast
-	cfg = newConfig()
+	cfg = newConfig(t)
 	// Set host networking
 	cfg.clusterSpec.Network.HostNetwork = true
 	cfg.store.Spec.Gateway.Port = 80
 	result = cfg.portString()
 	assert.Equal(t, "port=80", result)
 
 	// Secure port on beast
-	cfg = newConfig()
+	cfg = newConfig(t)
 	cfg.store.Spec.Gateway.SecurePort = 443
 	cfg.store.Spec.Gateway.SSLCertificateRef = "some-k8s-key-secret"
 	result = cfg.portString()
 	assert.Equal(t, "ssl_port=443 ssl_certificate=/etc/ceph/private/rgw-cert.pem", result)
 
 	// Both ports on beast
-	cfg = newConfig()
+	cfg = newConfig(t)
 	// Set host networking
 	cfg.clusterSpec.Network.HostNetwork = true
 	cfg.store.Spec.Gateway.Port = 80
@@ -76,13 +79,13 @@ func TestPortString(t *testing.T) {
 	assert.Equal(t, "port=80 ssl_port=443 ssl_certificate=/etc/ceph/private/rgw-cert.pem", result)
 
 	// Secure port requires the cert on beast
-	cfg = newConfig()
+	cfg = newConfig(t)
 	cfg.store.Spec.Gateway.SecurePort = 443
 	result = cfg.portString()
 	assert.Equal(t, "", result)
 
 	// Using SDN, no host networking so the rgw port internal is not the same
-	cfg = newConfig()
+	cfg = newConfig(t)
 	cfg.store.Spec.Gateway.Port = 80
 	result = cfg.portString()
 	assert.Equal(t, "port=8080", result)

pkg/operator/ceph/object/rgw.go

@@ -35,6 +35,7 @@ import (
 	"github.com/rook/rook/pkg/operator/ceph/pool"
 	"github.com/rook/rook/pkg/operator/k8sutil"
 	"github.com/rook/rook/pkg/util/exec"
+	v1 "k8s.io/api/core/v1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -333,7 +334,11 @@ func GetTlsCaCert(objContext *Context, objectStoreSpec *cephv1.ObjectStoreSpec)
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to get secret %s containing TLS certificate defined in %s", objectStoreSpec.Gateway.SSLCertificateRef, objContext.Name)
 		}
-		tlsCert = tlsSecretCert.Data[certKeyName]
+		if tlsSecretCert.Type == v1.SecretTypeOpaque {
+			tlsCert = tlsSecretCert.Data[certKeyName]
+		} else if tlsSecretCert.Type == v1.SecretTypeTLS {
+			tlsCert = tlsSecretCert.Data[v1.TLSCertKey]
+		}
 	} else if objectStoreSpec.GetServiceServingCert() != "" {
 		tlsCert, err = ioutil.ReadFile(ServiceServingCertCAFile)
 		if err != nil {

pkg/operator/ceph/object/spec.go

@@ -17,6 +17,7 @@ limitations under the License.
 package object
 
 import (
+	"context"
 	"fmt"
 	"path"
 	"reflect"
@@ -119,7 +120,7 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) (v1.PodTemplateSpec
 
 	// Set the ssl cert if specified
 	if c.store.Spec.Gateway.SecurePort != 0 {
-		secretVolSrc, err := generateVolumeSourceWithTLSSecret(c.store.Spec)
+		secretVolSrc, err := c.generateVolumeSourceWithTLSSecret()
 		if err != nil {
 			return v1.PodTemplateSpec{}, err
 		}
@@ -492,29 +493,50 @@ func getLabels(name, namespace string, includeNewLabels bool) map[string]string
 	return labels
 }
 
-func generateVolumeSourceWithTLSSecret(objSpec cephv1.ObjectStoreSpec) (*v1.SecretVolumeSource, error) {
+func (c *clusterConfig) generateVolumeSourceWithTLSSecret() (*v1.SecretVolumeSource, error) {
 	// Keep the TLS secret as secure as possible in the container. Give only user read perms.
 	// Because the Secret mount is owned by "root" and fsGroup breaks on OCP since we cannot predict it
 	// Also, we don't want to change the SCC for fsGroup to RunAsAny since it has a major broader impact
 	// Let's open the permissions a bit more so that everyone can read the cert.
 	userReadOnly := int32(0444)
 	var secretVolSrc *v1.SecretVolumeSource
-	if objSpec.Gateway.SSLCertificateRef != "" {
+	if c.store.Spec.Gateway.SSLCertificateRef != "" {
 		secretVolSrc = &v1.SecretVolumeSource{
-			SecretName: objSpec.Gateway.SSLCertificateRef,
-			Items: []v1.KeyToPath{
+			SecretName: c.store.Spec.Gateway.SSLCertificateRef,
+		}
+		secretType, err := c.rgwTLSSecretType()
+		if err != nil {
+			return nil, err
+		}
+		switch secretType {
+		case v1.SecretTypeOpaque:
+			secretVolSrc.Items = []v1.KeyToPath{
 				{Key: certKeyName, Path: certFilename, Mode: &userReadOnly},
-			}}
-	} else if objSpec.GetServiceServingCert() != "" {
+			}
+		case v1.SecretTypeTLS:
+			secretVolSrc.Items = []v1.KeyToPath{
+				{Key: v1.TLSCertKey, Path: certFilename, Mode: &userReadOnly},
+				{Key: v1.TLSPrivateKeyKey, Path: certKeyFileName, Mode: &userReadOnly},
+			}
+		}
+	} else if c.store.Spec.GetServiceServingCert() != "" {
 		secretVolSrc = &v1.SecretVolumeSource{
-			SecretName: objSpec.GetServiceServingCert(),
+			SecretName: c.store.Spec.GetServiceServingCert(),
 			Items: []v1.KeyToPath{
-				{Key: "tls.crt", Path: certFilename, Mode: &userReadOnly},
-				{Key: "tls.key", Path: certKeyFileName, Mode: &userReadOnly},
+				{Key: v1.TLSCertKey, Path: certFilename, Mode: &userReadOnly},
+				{Key: v1.TLSPrivateKeyKey, Path: certKeyFileName, Mode: &userReadOnly},
 			}}
 	} else {
 		return nil, errors.New("no TLS certificates found")
 	}
 
 	return secretVolSrc, nil
 }
+
+func (c *clusterConfig) rgwTLSSecretType() (v1.SecretType, error) {
+	rgwTlsSecret, err := c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Get(context.TODO(), c.store.Spec.Gateway.SSLCertificateRef, metav1.GetOptions{})
+	if rgwTlsSecret != nil {
+		return rgwTlsSecret.Type, nil
+	}
+	return "", errors.Wrapf(err, "no Kubernetes secrets referring TLS certificates found")
+}

pkg/operator/ceph/object/spec_test.go

@@ -91,6 +91,10 @@ func TestPodSpecs(t *testing.T) {
 }
 
 func TestSSLPodSpec(t *testing.T) {
+	ctx := context.TODO()
+	// Placeholder
+	context := &clusterd.Context{Clientset: test.New(t, 3)}
+
 	store := simpleStore()
 	store.Spec.Gateway.Resources = v1.ResourceRequirements{
 		Limits: v1.ResourceList{
@@ -105,12 +109,14 @@ func TestSSLPodSpec(t *testing.T) {
 	store.Spec.Gateway.PriorityClassName = "my-priority-class"
 	info := clienttest.CreateTestClusterInfo(1)
 	info.CephVersion = cephver.Nautilus
+	info.Namespace = store.Namespace
 	data := cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, "default", "rook-ceph", "/var/lib/rook/")
 	store.Spec.Gateway.SecurePort = 443
 
 	c := &clusterConfig{
 		clusterInfo: info,
 		store:       store,
+		context:     context,
 		rookVersion: "rook/rook:myversion",
 		clusterSpec: &cephv1.ClusterSpec{
 			CephVersion: cephv1.CephVersionSpec{Image: "ceph/ceph:v15"},
@@ -130,20 +136,58 @@ func TestSSLPodSpec(t *testing.T) {
 	assert.Error(t, err)
 
 	// Using SSLCertificateRef
+	// Opaque Secret
 	c.store.Spec.Gateway.SSLCertificateRef = "mycert"
-	secretVolSrc, _ := generateVolumeSourceWithTLSSecret(c.store.Spec)
+	rgwtlssecret := &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      c.store.Spec.Gateway.SSLCertificateRef,
+			Namespace: c.store.Namespace,
+		},
+		Data: map[string][]byte{
+			"cert": []byte("tlssecrettesting"),
+		},
+		Type: v1.SecretTypeOpaque,
+	}
+	_, err = c.context.Clientset.CoreV1().Secrets(store.Namespace).Create(ctx, rgwtlssecret, metav1.CreateOptions{})
+	assert.NoError(t, err)
+	secretVolSrc, err := c.generateVolumeSourceWithTLSSecret()
+	assert.NoError(t, err)
 	assert.Equal(t, secretVolSrc.SecretName, "mycert")
 	s, err := c.makeRGWPodSpec(rgwConfig)
 	assert.NoError(t, err)
 	podTemplate := cephtest.NewPodTemplateSpecTester(t, &s)
 	podTemplate.RunFullSuite(cephconfig.RgwType, "default", "rook-ceph-rgw", "mycluster", "ceph/ceph:myversion",
 		"200", "100", "1337", "500", /* resources */
 		"my-priority-class")
-
+	// TLS Secret
+	c.store.Spec.Gateway.SSLCertificateRef = "tlscert"
+	rgwtlssecret = &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      c.store.Spec.Gateway.SSLCertificateRef,
+			Namespace: c.store.Namespace,
+		},
+		Data: map[string][]byte{
+			"tls.crt": []byte("tlssecrettestingcert"),
+			"tls.key": []byte("tlssecrettestingkey"),
+		},
+		Type: v1.SecretTypeTLS,
+	}
+	_, err = c.context.Clientset.CoreV1().Secrets(store.Namespace).Create(ctx, rgwtlssecret, metav1.CreateOptions{})
+	assert.NoError(t, err)
+	secretVolSrc, err = c.generateVolumeSourceWithTLSSecret()
+	assert.NoError(t, err)
+	assert.Equal(t, secretVolSrc.SecretName, "tlscert")
+	s, err = c.makeRGWPodSpec(rgwConfig)
+	assert.NoError(t, err)
+	podTemplate = cephtest.NewPodTemplateSpecTester(t, &s)
+	podTemplate.RunFullSuite(cephconfig.RgwType, "default", "rook-ceph-rgw", "mycluster", "ceph/ceph:myversion",
+		"200", "100", "1337", "500", /* resources */
+		"my-priority-class")
 	// Using service serving cert
 	c.store.Spec.Gateway.SSLCertificateRef = ""
 	c.store.Spec.Gateway.Service = &(cephv1.RGWServiceSpec{Annotations: rook.Annotations{cephv1.ServiceServingCertKey: "rgw-cert"}})
-	secretVolSrc, _ = generateVolumeSourceWithTLSSecret(c.store.Spec)
+	secretVolSrc, err = c.generateVolumeSourceWithTLSSecret()
+	assert.NoError(t, err)
 	assert.Equal(t, secretVolSrc.SecretName, "rgw-cert")
 	s, err = c.makeRGWPodSpec(rgwConfig)
 	assert.NoError(t, err)