K8SPG-570 update custom secret with labels when they are missing, update missing secret data with autogeneration (#1218)

gkech · hors · web-flow · commit 19b203cb0d39 · 2025-07-17T19:37:20.000+03:00
* K8SPG-570 update custom secret with labels when they are missing

* use patch instead of update

* adjustsments on the logic

* fix concurrency issue by checking the secret update

* update e2e test

* add reset/set scenario on e2e test

* cr: bring back if err == nil for apply secrets, the secret is always overwritten

* cr: use OnError instead of RetryOnConflict

* cr: fix needsUpdate return early

---------

Co-authored-by: Viacheslav Sarzhan &lt;slava.sarzhan@percona.com&gt;
diff --git a/e2e-tests/tests/users/14-write-data-to-custom-db.yaml b/e2e-tests/tests/users/14-write-data-to-custom-db.yaml
@@ -13,7 +13,6 @@ commands:
       schema='public'
       hostname=$(get_pgbouncer_host chico-credentials)
 
-
       run_psql \
         'SET search_path TO public;CREATE TABLE IF NOT EXISTS customApp (id int PRIMARY KEY);' \
         "-h $hostname -U $user -d $db_name" "$password"
diff --git a/e2e-tests/tests/users/16-assert.yaml b/e2e-tests/tests/users/16-assert.yaml
@@ -0,0 +1,49 @@
+apiVersion: postgres-operator.crunchydata.com/v1beta1
+kind: PostgresCluster
+metadata:
+  name: users
+  ownerReferences:
+    - apiVersion: pgv2.percona.com/v2
+      kind: PerconaPGCluster
+      name: users
+      controller: true
+      blockOwnerDeletion: true
+  finalizers:
+    - postgres-operator.crunchydata.com/finalizer
+status:
+  instances:
+    - name: instance1
+      readyReplicas: 3
+      replicas: 3
+      updatedReplicas: 3
+  pgbackrest:
+    repoHost:
+      apiVersion: apps/v1
+      kind: StatefulSet
+      ready: true
+    repos:
+      - bound: true
+        name: repo1
+        replicaCreateBackupComplete: true
+        stanzaCreated: true
+  proxy:
+    pgBouncer:
+      readyReplicas: 3
+      replicas: 3
+---
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGCluster
+metadata:
+  name: users
+status:
+  pgbouncer:
+    ready: 3
+    size: 3
+  postgres:
+    instances:
+      - name: instance1
+        ready: 3
+        size: 3
+    ready: 3
+    size: 3
+  state: ready
diff --git a/e2e-tests/tests/users/16-create-user-with-predefined-password-secret.yaml b/e2e-tests/tests/users/16-create-user-with-predefined-password-secret.yaml
@@ -0,0 +1,15 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 10
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+      
+      kubectl -n ${NAMESPACE} create secret generic eagle-credentials --from-literal=password=eagle-db-password
+      sleep 5
+      
+      kubectl -n ${NAMESPACE} patch perconapgcluster/${test_name} --type=json -p '[{"op":"add", "path":"/spec/autoCreateUserSchema","value":true},{"op":"add", "path":"/spec/users","value":[{"name":"eagle","databases":["nest"],"password":{"type":"ASCII"},"secretName":"eagle-credentials"}]}]'
+      sleep 15
diff --git a/e2e-tests/tests/users/17-write-data-user-db-with-predefined-password.yaml b/e2e-tests/tests/users/17-write-data-user-db-with-predefined-password.yaml
@@ -0,0 +1,22 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+    - script: |-
+          set -o errexit
+          set -o xtrace
+          
+          source ../../functions
+          
+          predefinedPassword=eagle-db-password
+          user='eagle'
+          db_name='nest'
+          schema='eagle'
+          hostname=$(get_pgbouncer_host eagle-credentials)
+          
+          run_psql \
+            'CREATE TABLE IF NOT EXISTS customApp (id int PRIMARY KEY);' \
+            "-h $hostname -U $user -d $db_name" "$predefinedPassword"
+          run_psql \
+            "INSERT INTO $schema.customApp (id) VALUES (100500)" \
+            "-h $hostname -U $user -d $db_name" "$predefinedPassword"
+        
diff --git a/e2e-tests/tests/users/18-assert.yaml b/e2e-tests/tests/users/18-assert.yaml
@@ -0,0 +1,10 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 30
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: 18-read-from-user-db-with-predefined-password
+data:
+  data: ' 100500'
diff --git a/e2e-tests/tests/users/18-read-from-user-db-with-predefined-password.yaml b/e2e-tests/tests/users/18-read-from-user-db-with-predefined-password.yaml
@@ -0,0 +1,19 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 30
+commands:
+  - script: |-
+        set -o errexit
+        set -o xtrace
+
+        source ../../functions
+        
+        predefinedPassword=eagle-db-password
+        user='eagle'
+        db_name='nest'
+        schema='eagle'
+        hostname=$(get_pgbouncer_host eagle-credentials)
+
+        data=$(run_psql "SELECT * from $schema.customApp;" "-h $hostname -U $user -d $db_name" "$predefinedPassword")
+
+        kubectl create configmap -n "${NAMESPACE}" 18-read-from-user-db-with-predefined-password --from-literal=data="${data}"
diff --git a/e2e-tests/tests/users/19-reset-predefined-user-password.yaml b/e2e-tests/tests/users/19-reset-predefined-user-password.yaml
@@ -0,0 +1,13 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 10
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+       kubectl -n ${NAMESPACE} patch secret eagle-credentials --type merge --patch '{"stringData": {"password": "", "verifier":""}}'
+
+      sleep 5
diff --git a/e2e-tests/tests/users/20-write-data-user-db-with-reset-predefined-password.yaml b/e2e-tests/tests/users/20-write-data-user-db-with-reset-predefined-password.yaml
@@ -0,0 +1,22 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+    - script: |-
+          set -o errexit
+          set -o xtrace
+          
+          source ../../functions
+          
+          resetPassword=$(get_psql_user_pass eagle-credentials)
+          user='eagle'
+          db_name='nest'
+          schema='eagle'
+          hostname=$(get_pgbouncer_host eagle-credentials)
+          
+          run_psql \
+            'CREATE TABLE IF NOT EXISTS customApp (id int PRIMARY KEY);' \
+            "-h $hostname -U $user -d $db_name" "$resetPassword"
+          run_psql \
+            "INSERT INTO $schema.customApp (id) VALUES (100501)" \
+            "-h $hostname -U $user -d $db_name" "$resetPassword"
+        
diff --git a/e2e-tests/tests/users/21-set-again-user-with-predefined-password-secret.yaml b/e2e-tests/tests/users/21-set-again-user-with-predefined-password-secret.yaml
@@ -0,0 +1,12 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 10
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+        
+        kubectl -n ${NAMESPACE} patch secret eagle-credentials --type merge --patch '{"stringData": {"password": "eagle-db-password", "verifier":""}}'        
+        sleep 5
diff --git a/e2e-tests/tests/users/22-write-data-user-db-again-with-predefined-password.yaml b/e2e-tests/tests/users/22-write-data-user-db-again-with-predefined-password.yaml
@@ -0,0 +1,22 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+    - script: |-
+          set -o errexit
+          set -o xtrace
+          
+          source ../../functions
+          
+          resetPassword=eagle-db-password
+          user='eagle'
+          db_name='nest'
+          schema='eagle'
+          hostname=$(get_pgbouncer_host eagle-credentials)
+          
+          run_psql \
+            'CREATE TABLE IF NOT EXISTS customApp (id int PRIMARY KEY);' \
+            "-h $hostname -U $user -d $db_name" "$resetPassword"
+          run_psql \
+            "INSERT INTO $schema.customApp (id) VALUES (100502)" \
+            "-h $hostname -U $user -d $db_name" "$resetPassword"
+        
diff --git a/internal/controller/postgrescluster/postgres.go b/internal/controller/postgrescluster/postgres.go
@@ -19,10 +19,13 @@ import (
 	"github.com/pkg/errors"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/validation/field"
+	"k8s.io/client-go/util/retry"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/percona/percona-postgresql-operator/internal/feature"
@@ -494,6 +497,17 @@ func (r *Reconciler) reconcilePostgresUserSecrets(
 		userSpecs[string(specUsers[i].Name)] = &specUsers[i]
 	}
 
+	// K8SPG-570 for secrets that were created manually, update them
+	// with the right labels so that the selector called next to track them
+	// and utilize their data.
+	for _, user := range specUsers {
+		if user.SecretName != "" {
+			if err := r.updateCustomSecretLabels(ctx, cluster, user); err != nil {
+				return specUsers, nil, err
+			}
+		}
+	}
+
 	secrets := &corev1.SecretList{}
 	selector, err := naming.AsSelector(naming.ClusterPostgresUsers(cluster.Name))
 	if err == nil {
@@ -582,6 +596,109 @@ func (r *Reconciler) reconcilePostgresUserSecrets(
 	return specUsers, userSecrets, err
 }
 
+// K8SPG-570
+// updateCustomSecretLabels checks if a custom secret exists - can be created manually through
+// kubectl apply - and updates it with required labels if they are missing. This enables the
+// naming.AsSelector(naming.ClusterPostgresUsers(cluster.Name)) to identify these secrets.
+func (r *Reconciler) updateCustomSecretLabels(
+	ctx context.Context, cluster *v1beta1.PostgresCluster, user v1beta1.PostgresUserSpec,
+) error {
+	secretName := string(user.SecretName)
+	userName := string(user.Name)
+
+	secret := &corev1.Secret{}
+	err := r.Client.Get(ctx, types.NamespacedName{
+		Name:      secretName,
+		Namespace: cluster.Namespace,
+	}, secret)
+	if err != nil {
+		if k8serrors.IsNotFound(err) {
+			return nil
+		}
+		return errors.Wrap(err, fmt.Sprintf("failed to get user %s secret %s", userName, secretName))
+	}
+
+	requiredLabels := map[string]string{
+		naming.LabelCluster:      cluster.Name,
+		naming.LabelPostgresUser: userName,
+		naming.LabelRole:         naming.RolePostgresUser,
+	}
+
+	needsUpdate := false
+	if secret.Labels == nil {
+		secret.Labels = make(map[string]string)
+	}
+
+	for labelKey, labelValue := range requiredLabels {
+		if existing, exists := secret.Labels[labelKey]; !exists || existing != labelValue {
+			secret.Labels[labelKey] = labelValue
+			needsUpdate = true
+		}
+	}
+
+	if !needsUpdate {
+		return nil
+	}
+
+	updateErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		current := &corev1.Secret{}
+		if err := r.Client.Get(ctx, types.NamespacedName{
+			Name:      secretName,
+			Namespace: cluster.Namespace,
+		}, current); err != nil {
+			return err
+		}
+
+		currentOrig := current.DeepCopy()
+		if current.Labels == nil {
+			current.Labels = make(map[string]string)
+		}
+
+		updateNeeded := false
+		for labelKey, labelValue := range requiredLabels {
+			if existing, exists := current.Labels[labelKey]; !exists || existing != labelValue {
+				current.Labels[labelKey] = labelValue
+				updateNeeded = true
+			}
+		}
+
+		if !updateNeeded {
+			return nil
+		}
+
+		return r.Client.Patch(ctx, current, client.MergeFrom(currentOrig))
+	})
+
+	if updateErr != nil {
+		return errors.Wrap(updateErr, fmt.Sprintf("failed to update secret %s", secretName))
+	}
+
+	verifyErr := retry.OnError(
+		retry.DefaultRetry,
+		func(err error) bool {
+			return true
+		},
+		func() error {
+			verifySecret := &corev1.Secret{}
+			if err := r.Client.Get(ctx, types.NamespacedName{
+				Name:      secretName,
+				Namespace: cluster.Namespace,
+			}, verifySecret); err != nil {
+				return err
+			}
+
+			for labelKey, labelValue := range requiredLabels {
+				if existing, exists := verifySecret.Labels[labelKey]; !exists || existing != labelValue {
+					return errors.Errorf("secret %s label %s not yet propagated", secretName, labelKey)
+				}
+			}
+
+			return nil
+		})
+
+	return errors.Wrap(verifyErr, "failed to update secret")
+}
+
 // reconcilePostgresUsersInPostgreSQL creates users inside of PostgreSQL and
 // sets their options and database access as specified.
 func (r *Reconciler) reconcilePostgresUsersInPostgreSQL(
