CSPL-532: Set idxc.secret on the secrets mounted on the indexer pods when idxc.…

deploy/crds/enterprise.splunk.com_indexerclusters_crd.yaml

@@ -2236,6 +2236,11 @@ spec:
           description: IndexerClusterStatus defines the observed state of a Splunk
             Enterprise indexer cluster
           properties:
+            IdxcPasswordChangedSecrets:
+              additionalProperties:
+                type: boolean
+              description: Holds secrets whose IDXC password has changed
+              type: object
             clusterMasterPhase:
               description: current phase of the cluster master
               enum:

deploy/olm-catalog/splunk/0.2.0/enterprise.splunk.com_indexerclusters_crd.yaml

@@ -2236,6 +2236,11 @@ spec:
           description: IndexerClusterStatus defines the observed state of a Splunk
             Enterprise indexer cluster
           properties:
+            IdxcPasswordChangedSecrets:
+              additionalProperties:
+                type: boolean
+              description: Holds secrets whose IDXC password has changed
+              type: object
             clusterMasterPhase:
               description: current phase of the cluster master
               enum:

pkg/apis/enterprise/v1beta1/indexercluster_types.go

@@ -89,6 +89,9 @@ type IndexerClusterStatus struct {
 	// Indicates resource version of namespace scoped secret
 	NamespaceSecretResourceVersion string `json:"namespace_scoped_secret_resource_version"`
 
+	// Holds secrets whose IDXC password has changed
+	IdxcPasswordChangedSecrets map[string]bool `json:"IdxcPasswordChangedSecrets"`
+
 	// Indicates if the cluster is in maintenance mode.
 	MaintenanceMode bool `json:"maintenance_mode"`
 

pkg/splunk/enterprise/indexercluster.go

@@ -62,6 +62,9 @@ func ApplyIndexerCluster(client splcommon.ControllerClient, cr *enterprisev1.Ind
 	if cr.Status.IndexerSecretChanged == nil {
 		cr.Status.IndexerSecretChanged = []bool{}
 	}
+	if cr.Status.IdxcPasswordChangedSecrets == nil {
+		cr.Status.IdxcPasswordChangedSecrets = make(map[string]bool)
+	}
 	defer func() {
 		err = client.Status().Update(context.TODO(), cr)
 		if err != nil {
@@ -148,6 +151,7 @@ func ApplyIndexerCluster(client splcommon.ControllerClient, cr *enterprisev1.Ind
 		// Reset idxc secret changed and namespace secret revision
 		cr.Status.IndexerSecretChanged = []bool{}
 		cr.Status.NamespaceSecretResourceVersion = namespaceScopedSecret.ObjectMeta.ResourceVersion
+		cr.Status.IdxcPasswordChangedSecrets = make(map[string]bool)
 
 		result.Requeue = false
 	}
@@ -203,6 +207,7 @@ func SetClusterMaintenanceMode(c splcommon.ControllerClient, cr *enterprisev1.In
 
 // ApplyIdxcSecret checks if any of the indexer's have a different idxc_secret from namespace scoped secret and changes it
 func ApplyIdxcSecret(mgr *indexerClusterPodManager, replicas int32, mock bool) error {
+	var indIdxcSecret string
 	// Get namespace scoped secret
 	namespaceSecret, err := splutil.ApplyNamespaceScopedSecretObject(mgr.c, mgr.cr.GetNamespace())
 	if err != nil {
@@ -231,10 +236,17 @@ func ApplyIdxcSecret(mgr *indexerClusterPodManager, replicas int32, mock bool) e
 		// Get Indexer's name
 		indexerPodName := GetSplunkStatefulsetPodName(SplunkIndexer, mgr.cr.GetName(), i)
 
-		// Retrieve idxc_secret password from Pod
-		indIdxcSecret, err := splutil.GetSpecificSecretTokenFromPod(mgr.c, indexerPodName, mgr.cr.GetNamespace(), "idxc_secret")
+		// Retrieve secret from pod
+		podSecret, err := splutil.GetSecretFromPod(mgr.c, indexerPodName, mgr.cr.GetNamespace())
 		if err != nil {
-			return fmt.Errorf("Couldn't Retrieve idxc_secret from secret data %s", err.Error())
+			return fmt.Errorf("Couldn't retrieve secret from pod %s", err.Error())
+		}
+
+		// Retrieve idxc_secret token
+		if indIdxcSecretByte, ok := podSecret.Data["idxc_secret"]; ok {
+			indIdxcSecret = string(indIdxcSecretByte)
+		} else {
+			return fmt.Errorf("Couldn't retrieve idxc_secret from secret %s mounted on pod %s", podSecret.GetName(), indexerPodName)
 		}
 
 		// If idxc secret is different from namespace scoped secret change it
@@ -274,6 +286,9 @@ func ApplyIdxcSecret(mgr *indexerClusterPodManager, replicas int32, mock bool) e
 			}
 			scopedLog.Info("Restarted splunk")
 
+			// Keep a track of all the secrets on pods to change their idxc secret below
+			mgr.cr.Status.IdxcPasswordChangedSecrets[podSecret.GetName()] = true
+
 			// Set the idxc_secret changed flag to true
 			if i < int32(len(mgr.cr.Status.IndexerSecretChanged)) {
 				mgr.cr.Status.IndexerSecretChanged[i] = true
@@ -283,6 +298,43 @@ func ApplyIdxcSecret(mgr *indexerClusterPodManager, replicas int32, mock bool) e
 		}
 	}
 
+	/*
+		During the recycle of indexer pods due to an idxc secret change, if there is a container
+		restart(for example if the splunkd process dies) before the operator
+		deletes the pod, the container restart fails due to mismatch of idxc password between Cluster
+		master and that particular indexer.
+
+		Changing the idxc passwords on the secrets mounted on the indexer pods to avoid the above.
+	*/
+	if len(mgr.cr.Status.IdxcPasswordChangedSecrets) > 0 {
+		for podSecretName := range mgr.cr.Status.IdxcPasswordChangedSecrets {
+			if mgr.cr.Status.IdxcPasswordChangedSecrets[podSecretName] {
+				podSecret, err := splutil.GetSecretByName(mgr.c, mgr.cr, podSecretName)
+				if err != nil {
+					return fmt.Errorf("Could not read secret %s, reason - %v", podSecretName, err)
+				}
+
+				// Retrieve namespaced scoped secret data in splunk readable format
+				splunkReadableData, err := splutil.GetSplunkReadableNamespaceScopedSecretData(mgr.c, mgr.cr.GetNamespace())
+				if err != nil {
+					return err
+				}
+
+				podSecret.Data["idxc_secret"] = splunkReadableData["idxc_secret"]
+				podSecret.Data["default.yml"] = splunkReadableData["default.yml"]
+
+				_, err = splctrl.ApplySecret(mgr.c, podSecret)
+				if err != nil {
+					return err
+				}
+				scopedLog.Info("idxc password changed on the secret mounted on pod", "Secret on Pod:", podSecretName)
+
+				// Set to false marking the idxc password change in the secret
+				mgr.cr.Status.IdxcPasswordChangedSecrets[podSecretName] = false
+			}
+		}
+	}
+
 	return nil
 }
 

pkg/splunk/enterprise/indexercluster_test.go

@@ -575,7 +575,7 @@ func TestApplyIdxcSecret(t *testing.T) {
 			Namespace: "test",
 		},
 	}
-
+	cr.Status.IdxcPasswordChangedSecrets = make(map[string]bool)
 	cr.Spec.ClusterMasterRef.Name = cr.GetName()
 	mockSplunkClient := &spltest.MockHTTPClient{}
 	mockSplunkClient.AddHandlers(mockHandlers...)