From 706e8b3b9147d42550408e86e29920ae1d813e53 Mon Sep 17 00:00:00 2001 From: parth-gr Date: Mon, 1 Apr 2024 18:22:03 +0530 Subject: [PATCH] object: add support for user s3 key for cephobjectstoreuser CephObjectStoreUser should optionally be able to reference a secret where S3 key is defined. This enables us to specify the accesskey and accesssecret rather than those values being randomly generated. Closes: https://github.com/rook/rook/issues/11563 Signed-off-by: parth-gr --- .../ceph-object-store-user-crd.md | 25 +++++++++ pkg/operator/ceph/object/user.go | 7 +++ pkg/operator/ceph/object/user/controller.go | 52 +++++++++++++++++-- .../ceph/object/user/controller_test.go | 8 +++ 4 files changed, 87 insertions(+), 5 deletions(-) diff --git a/Documentation/CRDs/Object-Storage/ceph-object-store-user-crd.md b/Documentation/CRDs/Object-Storage/ceph-object-store-user-crd.md index 4dde0d8bf1acf..0dcd50f344f03 100644 --- a/Documentation/CRDs/Object-Storage/ceph-object-store-user-crd.md +++ b/Documentation/CRDs/Object-Storage/ceph-object-store-user-crd.md @@ -60,3 +60,28 @@ spec: * `user-policy` * `odic-provider` * `ratelimit` + +### CephObjectStoreUser Reference Secret + +If a specific user key and secret is desired instead of randomly generated credentials, a specific user key and secret can be specified for an object store user. + +Create or update the Kubernetes secret with name, `rook-ceph-object-user--` in the same namespace of cephobjectUser, where: + +* `store-name`: The object store name in which the user will be created. This matches the name of the objectstore CRD. +* `user-name`: The metadata name of the cephObjectStoreUser + +```console +kubectl create -f +apiVersion: v1 +kind: Secret +metadata: + name: rook-ceph-object-user-my-store-my-user + namespace: rook-ceph + annotations: + rook.io/source-of-truth: secret +data: + AccessKey: *** + SecretKey: *** + Endpoint: *** +type: "kubernetes.io/rook" +``` diff --git a/pkg/operator/ceph/object/user.go b/pkg/operator/ceph/object/user.go index 7cb2b670b682a..d0a63595c7a51 100644 --- a/pkg/operator/ceph/object/user.go +++ b/pkg/operator/ceph/object/user.go @@ -45,6 +45,8 @@ const ( ErrorCodeFileExists = 17 ) +var UserKeysSource = false + // An ObjectUser defines the details of an object store user. type ObjectUser struct { UserID string `json:"userId"` @@ -252,6 +254,7 @@ func generateCephUserSecret(userConfig *admin.User, endpoint, namespace, storeNa if tlsSecretName != "" { secrets["SSLCertSecretName"] = tlsSecretName } + secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: secretName, @@ -266,6 +269,10 @@ func generateCephUserSecret(userConfig *admin.User, endpoint, namespace, storeNa StringData: secrets, Type: k8sutil.RookType, } + if UserKeysSource { + secret.Annotations = map[string]string{"administration.rook.io/source-of-truth": "secret"} + } + return secret } diff --git a/pkg/operator/ceph/object/user/controller.go b/pkg/operator/ceph/object/user/controller.go index 1c7624e6c7c46..7a3c6010b16d1 100644 --- a/pkg/operator/ceph/object/user/controller.go +++ b/pkg/operator/ceph/object/user/controller.go @@ -21,6 +21,7 @@ import ( "context" "fmt" "reflect" + "strings" "github.com/ceph/go-ceph/rgw/admin" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" @@ -48,8 +49,9 @@ import ( ) const ( - appName = object.AppName - controllerName = "ceph-object-store-user-controller" + appName = object.AppName + controllerName = "ceph-object-store-user-controller" + updateObjectUserSecretAnnotation = "rook.io/source-of-truth" ) // newMultisiteAdminOpsCtxFunc help us mocking the admin ops API client in unit test @@ -287,15 +289,49 @@ func (r *ReconcileObjectStoreUser) reconcileCephUser(cephObjectStoreUser *cephv1 return reconcile.Result{}, nil } +func forceUpdateObjectUserSecret(annotations map[string]string) bool { + if value, found := annotations[updateObjectUserSecretAnnotation]; found { + if strings.EqualFold(value, "secret") { + return true + } + } + return false +} + func (r *ReconcileObjectStoreUser) createOrUpdateCephUser(u *cephv1.CephObjectStoreUser) error { logger.Infof("creating ceph object user %q in namespace %q", u.Name, u.Namespace) logCreateOrUpdate := fmt.Sprintf("retrieved existing ceph object user %q", u.Name) var user admin.User + var userKeys []admin.UserKeySpec var err error + + // get the user defined k8s s3 keys if exists + secretName := object.GenerateCephUserSecretName(u.Spec.Store, u.Name) + namspacedName := types.NamespacedName{Namespace: u.Namespace, Name: secretName} + cephObjectStoreUserSecret := &corev1.Secret{} + err = r.client.Get(r.clusterInfo.Context, namspacedName, cephObjectStoreUserSecret) + if err != nil { + if kerrors.IsNotFound(err) { + logger.Debugf("no user secret %q provided for cephobjectuser", secretName) + } else { + return errors.Wrapf(err, "failed to get user cephobjectuser secret %q", secretName) + } + } else { + userKeys = []admin.UserKeySpec{ + {AccessKey: string(cephObjectStoreUserSecret.Data["AccessKey"]), + SecretKey: string(cephObjectStoreUserSecret.Data["SecretKey"])}, + } + if forceUpdateObjectUserSecret(cephObjectStoreUserSecret.GetAnnotations()) { + object.UserKeysSource = true + } + } + user, err = r.objContext.AdminOpsClient.GetUser(r.opManagerContext, *r.userConfig) if err != nil { if errors.Is(err, admin.ErrNoSuchUser) { + // if secret exists use the user specified keys + r.userConfig.Keys = userKeys user, err = r.objContext.AdminOpsClient.CreateUser(r.opManagerContext, *r.userConfig) if err != nil { return errors.Wrapf(err, "failed to create ceph object user %v", &r.userConfig.ID) @@ -306,10 +342,15 @@ func (r *ReconcileObjectStoreUser) createOrUpdateCephUser(u *cephv1.CephObjectSt } } + if object.UserKeysSource { + // if secret exists and source of truth is secret then use the user specified keys + r.userConfig.Keys = userKeys + } + // Update max bucket if necessary logger.Tracef("user capabilities(id: %s, caps: %#v, user caps: %s, op mask: %s)", user.ID, user.Caps, user.UserCaps, user.OpMask) - if *user.MaxBuckets != *r.userConfig.MaxBuckets { + if *user.MaxBuckets != *r.userConfig.MaxBuckets || object.UserKeysSource { user, err = r.objContext.AdminOpsClient.ModifyUser(r.opManagerContext, *r.userConfig) if err != nil { return errors.Wrapf(err, "failed to update ceph object user %q max buckets", r.userConfig.ID) @@ -365,9 +406,10 @@ func (r *ReconcileObjectStoreUser) createOrUpdateCephUser(u *cephv1.CephObjectSt // Set access and secret key if r.userConfig.Keys == nil { r.userConfig.Keys = make([]admin.UserKeySpec, 1) + r.userConfig.Keys[0].AccessKey = user.Keys[0].AccessKey + r.userConfig.Keys[0].SecretKey = user.Keys[0].SecretKey } - r.userConfig.Keys[0].AccessKey = user.Keys[0].AccessKey - r.userConfig.Keys[0].SecretKey = user.Keys[0].SecretKey + logger.Debugf("updated object user values are %v %v %t", r.userConfig.Keys, user.Keys, object.UserKeysSource) logger.Info(logCreateOrUpdate) return nil diff --git a/pkg/operator/ceph/object/user/controller_test.go b/pkg/operator/ceph/object/user/controller_test.go index b443ebca21362..f883dfce9efbd 100644 --- a/pkg/operator/ceph/object/user/controller_test.go +++ b/pkg/operator/ceph/object/user/controller_test.go @@ -30,6 +30,7 @@ import ( "github.com/coreos/pkg/capnslog" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake" + cephclient "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/operator/test" @@ -491,15 +492,22 @@ func TestCreateOrUpdateCephUser(t *testing.T) { return nil, fmt.Errorf("unexpected request: %q. method %q. path %q", req.URL.RawQuery, req.Method, req.URL.Path) }, } + s := scheme.Scheme + cl := fake.NewClientBuilder().WithScheme(s).Build() adminClient, err := admin.New("rook-ceph-rgw-my-store.mycluster.svc", "53S6B9S809NUP19IJ2K3", "1bXPegzsGClvoGAiJdHQD1uOW2sQBLAZM9j9VtXR", mockClient) assert.NoError(t, err) userConfig := generateUserConfig(objectUser) + clusterInfo := &cephclient.ClusterInfo{ + Context: context.TODO(), + } r := &ReconcileObjectStoreUser{ objContext: &cephobject.AdminOpsContext{ AdminOpsClient: adminClient, }, userConfig: &userConfig, opManagerContext: context.TODO(), + client: cl, + clusterInfo: clusterInfo, } maxsize, err := resource.ParseQuantity(maxsizestr) assert.NoError(t, err)