[CSPL-1087] Full re-configuration when mcRef is changed/removed

pkg/splunk/enterprise/clustermaster.go

@@ -137,6 +137,14 @@ func ApplyClusterMaster(client splcommon.ControllerClient, cr *enterpriseApi.Clu
 	if err != nil {
 		return result, err
 	}
+
+	//make changes to respective mc configmap when changing/removing mcRef from spec
+	extraEnv, err := VerifyCMisMultisite(cr, namespaceScopedSecret)
+	err = validateMonitoringConsoleRef(client, statefulSet, extraEnv)
+	if err != nil {
+		return result, err
+	}
+
 	clusterMasterManager := splctrl.DefaultStatefulSetPodManager{}
 	phase, err := clusterMasterManager.Update(client, statefulSet, 1)
 	if err != nil {
@@ -154,7 +162,6 @@ func ApplyClusterMaster(client splcommon.ControllerClient, cr *enterpriseApi.Clu
 		}
 		//Update MC configmap
 		if cr.Spec.MonitoringConsoleRef.Name != "" {
-			extraEnv, err := VerifyCMisMultisite(cr, namespaceScopedSecret)
 			_, err = ApplyMonitoringConsoleEnvConfigMap(client, cr.GetNamespace(), cr.GetName(), cr.Spec.MonitoringConsoleRef.Name, extraEnv, true)
 			if err != nil {
 				return result, err

pkg/splunk/enterprise/clustermaster_test.go

@@ -45,6 +45,7 @@ func TestApplyClusterMaster(t *testing.T) {
 		{MetaName: "*v1.ConfigMap-test-splunk-stack1-clustermaster-app-list"},
 		{MetaName: "*v1.ConfigMap-test-splunk-stack1-clustermaster-smartstore"},
 		{MetaName: "*v1.StatefulSet-test-splunk-stack1-cluster-master"},
+		{MetaName: "*v1.StatefulSet-test-splunk-stack1-cluster-master"},
 	}
 
 	labels := map[string]string{
@@ -58,7 +59,7 @@ func TestApplyClusterMaster(t *testing.T) {
 	listmockCall := []spltest.MockFuncCall{
 		{ListOpts: listOpts}}
 	createCalls := map[string][]spltest.MockFuncCall{"Get": funcCalls, "Create": {funcCalls[0], funcCalls[2], funcCalls[3], funcCalls[5], funcCalls[9]}, "List": {listmockCall[0]}, "Update": {funcCalls[0]}}
-	updateCalls := map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[0], funcCalls[2], funcCalls[3], funcCalls[4], funcCalls[5], funcCalls[6], funcCalls[7], funcCalls[8], funcCalls[9]}, "Update": {funcCalls[9]}, "List": {listmockCall[0]}}
+	updateCalls := map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[0], funcCalls[2], funcCalls[3], funcCalls[4], funcCalls[5], funcCalls[6], funcCalls[7], funcCalls[8], funcCalls[9], funcCalls[10]}, "Update": {funcCalls[9]}, "List": {listmockCall[0]}}
 
 	current := enterpriseApi.ClusterMaster{
 		TypeMeta: metav1.TypeMeta{
@@ -167,6 +168,7 @@ func TestApplyClusterMasterWithSmartstore(t *testing.T) {
 		{MetaName: "*v1.ConfigMap-test-splunk-stack1-clustermaster-app-list"},
 		{MetaName: "*v1.ConfigMap-test-splunk-stack1-clustermaster-smartstore"},
 		{MetaName: "*v1.StatefulSet-test-splunk-stack1-cluster-master"},
+		{MetaName: "*v1.StatefulSet-test-splunk-stack1-cluster-master"},
 		{MetaName: "*v1.Pod-test-splunk-stack1-cluster-master-0"},
 		{MetaName: "*v1.StatefulSet-test-splunk-test-monitoring-console"},
 	}
@@ -181,7 +183,7 @@ func TestApplyClusterMasterWithSmartstore(t *testing.T) {
 	listmockCall := []spltest.MockFuncCall{
 		{ListOpts: listOpts}}
 	createCalls := map[string][]spltest.MockFuncCall{"Get": funcCalls, "Create": {funcCalls[6], funcCalls[7], funcCalls[9]}, "List": {listmockCall[0], listmockCall[0]}, "Update": {funcCalls[0], funcCalls[3]}}
-	updateCalls := map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[2], funcCalls[3], funcCalls[5], funcCalls[5], funcCalls[6], funcCalls[7], funcCalls[8], funcCalls[9], funcCalls[10], funcCalls[11], funcCalls[12], funcCalls[13]}, "Update": {funcCalls[10], funcCalls[13]}, "List": {listmockCall[0]}}
+	updateCalls := map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[2], funcCalls[3], funcCalls[5], funcCalls[5], funcCalls[6], funcCalls[7], funcCalls[8], funcCalls[9], funcCalls[10], funcCalls[11], funcCalls[12], funcCalls[13], funcCalls[14]}, "Update": {funcCalls[10], funcCalls[13]}, "List": {listmockCall[0]}}
 
 	current := enterpriseApi.ClusterMaster{
 		TypeMeta: metav1.TypeMeta{

pkg/splunk/enterprise/configuration.go

@@ -713,13 +713,17 @@ func updateSplunkPodTemplateWithConfig(client splcommon.ControllerClient, podTem
 		})
 	}
 
+	// append REF for monitoring console if configured
+	if spec.MonitoringConsoleRef.Name != "" {
+		extraEnv = append(extraEnv, corev1.EnvVar{
+			Name:  "SPLUNK_MONITORING_CONSOLE_REF",
+			Value: spec.MonitoringConsoleRef.Name,
+		})
+	}
+
 	// Add extraEnv from the CommonSplunkSpec config to the extraEnv variable list
-	// Exclude MC as it derives the Spec from multiple CRs
-	// ToDo: Remove the Check once the MC CRD is in place
-	if instanceType != SplunkMonitoringConsole {
-		for _, envVar := range spec.ExtraEnv {
-			extraEnv = append(extraEnv, envVar)
-		}
+	for _, envVar := range spec.ExtraEnv {
+		extraEnv = append(extraEnv, envVar)
 	}
 
 	// append any extra variables
@@ -741,17 +745,12 @@ func getLivenessProbe(cr splcommon.MetaObject, instanceType InstanceType, spec *
 
 	livenessDelay := int32(livenessProbeDefaultDelaySec)
 
-	// Exclude MC, as it derives the spec from Multiple CRs.
-	// ToDo: Remove the Check once the MC CRD is in place
-	if instanceType != SplunkMonitoringConsole {
-		// If configured, always use the Liveness initial delay from the CR
-		if spec.LivenessInitialDelaySeconds != 0 {
-			livenessDelay = spec.LivenessInitialDelaySeconds
-		} else {
-			livenessDelay += additionalDelay
-		}
+	// If configured, always use the Liveness initial delay from the CR
+	if spec.LivenessInitialDelaySeconds != 0 {
+		livenessDelay = spec.LivenessInitialDelaySeconds
+	} else {
+		livenessDelay += additionalDelay
 	}
-
 	scopedLog.Info("LivenessProbeInitialDelay", "configured", spec.LivenessInitialDelaySeconds, "additionalDelay", additionalDelay, "finalCalculatedValue", livenessDelay)
 
 	livenessCommand := []string{
@@ -768,15 +767,11 @@ func getReadinessProbe(cr splcommon.MetaObject, instanceType InstanceType, spec
 
 	readinessDelay := int32(readinessProbeDefaultDelaySec)
 
-	// Exclude MC, as it derives the spec from Multiple CRs.
-	// ToDo: Remove the Check once the MC CRD is in place
-	if instanceType != SplunkMonitoringConsole {
-		// If configured, always use the readiness initial delay from the CR
-		if spec.ReadinessInitialDelaySeconds != 0 {
-			readinessDelay = spec.ReadinessInitialDelaySeconds
-		} else {
-			readinessDelay += additionalDelay
-		}
+	// If configured, always use the readiness initial delay from the CR
+	if spec.ReadinessInitialDelaySeconds != 0 {
+		readinessDelay = spec.ReadinessInitialDelaySeconds
+	} else {
+		readinessDelay += additionalDelay
 	}
 
 	scopedLog.Info("ReadinessProbeInitialDelay", "configured", spec.ReadinessInitialDelaySeconds, "additionalDelay", additionalDelay, "finalCalculatedValue", readinessDelay)

pkg/splunk/enterprise/licensemaster.go

@@ -96,6 +96,13 @@ func ApplyLicenseMaster(client splcommon.ControllerClient, cr *enterpriseApi.Lic
 	if err != nil {
 		return result, err
 	}
+
+	//make changes to respective mc configmap when changing/removing mcRef from spec
+	err = validateMonitoringConsoleRef(client, statefulSet, getLicenseMasterURL(cr, &cr.Spec.CommonSplunkSpec))
+	if err != nil {
+		return result, err
+	}
+
 	mgr := splctrl.DefaultStatefulSetPodManager{}
 	phase, err := mgr.Update(client, statefulSet, 1)
 	if err != nil {

pkg/splunk/enterprise/licensemaster_test.go

@@ -38,6 +38,7 @@ func TestApplyLicenseMaster(t *testing.T) {
 		{MetaName: "*v1.Secret-test-splunk-stack1-license-master-secret-v1"},
 		{MetaName: "*v1.ConfigMap-test-splunk-stack1-licensemaster-app-list"},
 		{MetaName: "*v1.StatefulSet-test-splunk-stack1-license-master"},
+		{MetaName: "*v1.StatefulSet-test-splunk-stack1-license-master"},
 	}
 	labels := map[string]string{
 		"app.kubernetes.io/component":  "versionedSecrets",

pkg/splunk/enterprise/searchheadcluster.go

@@ -143,6 +143,13 @@ func ApplySearchHeadCluster(client splcommon.ControllerClient, cr *enterpriseApi
 	if err != nil {
 		return result, err
 	}
+
+	//make changes to respective mc configmap when changing/removing mcRef from spec
+	err = validateMonitoringConsoleRef(client, statefulSet, getSearchHeadEnv(cr))
+	if err != nil {
+		return result, err
+	}
+
 	mgr := searchHeadClusterPodManager{c: client, log: scopedLog, cr: cr, secrets: namespaceScopedSecret, newSplunkClient: splclient.NewSplunkClient}
 	phase, err = mgr.Update(client, statefulSet, cr.Spec.Replicas)
 	if err != nil {

pkg/splunk/enterprise/searchheadcluster_test.go

@@ -47,6 +47,7 @@ func TestApplySearchHeadCluster(t *testing.T) {
 		{MetaName: "*v1.Secret-test-splunk-test-secret"},
 		{MetaName: "*v1.Secret-test-splunk-stack1-search-head-secret-v1"},
 		{MetaName: "*v1.StatefulSet-test-splunk-stack1-search-head"},
+		{MetaName: "*v1.StatefulSet-test-splunk-stack1-search-head"},
 		{MetaName: "*v1.Secret-test-splunk-test-secret"},
 	}
 	labels := map[string]string{

pkg/splunk/enterprise/standalone.go

@@ -158,6 +158,12 @@ func ApplyStandalone(client splcommon.ControllerClient, cr *enterpriseApi.Standa
 		return result, err
 	}
 
+	//make changes to respective mc configmap when changing/removing mcRef from spec
+	err = validateMonitoringConsoleRef(client, statefulSet, getStandaloneExtraEnv(cr, cr.Spec.Replicas))
+	if err != nil {
+		return result, err
+	}
+
 	mgr := splctrl.DefaultStatefulSetPodManager{}
 	phase, err := mgr.Update(client, statefulSet, cr.Spec.Replicas)
 	cr.Status.ReadyReplicas = statefulSet.Status.ReadyReplicas

pkg/splunk/enterprise/standalone_test.go

@@ -42,6 +42,7 @@ func TestApplyStandalone(t *testing.T) {
 		{MetaName: "*v1.ConfigMap-test-splunk-stack1-standalone-app-list"},
 		{MetaName: "*v1.ConfigMap-test-splunk-stack1-standalone-smartstore"},
 		{MetaName: "*v1.StatefulSet-test-splunk-stack1-standalone"},
+		{MetaName: "*v1.StatefulSet-test-splunk-stack1-standalone"},
 	}
 	labels := map[string]string{
 		"app.kubernetes.io/component":  "versionedSecrets",
@@ -100,6 +101,7 @@ func TestApplyStandaloneWithSmartstore(t *testing.T) {
 		{MetaName: "*v1.ConfigMap-test-splunk-stack1-standalone-app-list"},
 		{MetaName: "*v1.ConfigMap-test-splunk-stack1-standalone-smartstore"},
 		{MetaName: "*v1.StatefulSet-test-splunk-stack1-standalone"},
+		{MetaName: "*v1.StatefulSet-test-splunk-stack1-standalone"},
 	}
 	labels := map[string]string{
 		"app.kubernetes.io/component":  "versionedSecrets",

pkg/splunk/enterprise/util.go

@@ -15,6 +15,7 @@
 package enterprise
 
 import (
+	"context"
 	"fmt"
 	"path/filepath"
 	"reflect"
@@ -23,8 +24,10 @@ import (
 	"strings"
 	"time"
 
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/types"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
 	enterpriseApi "github.com/splunk/splunk-operator/pkg/apis/enterprise/v2"
@@ -999,3 +1002,50 @@ func initAndCheckAppInfoStatus(client splcommon.ControllerClient, cr splcommon.M
 
 	return nil
 }
+
+//validateMonitoringConsoleRef validates the changes in monitoringConsoleRef
+func validateMonitoringConsoleRef(c splcommon.ControllerClient, revised *appsv1.StatefulSet, serviceURLs []corev1.EnvVar) error {
+	var err error
+	namespacedName := types.NamespacedName{Namespace: revised.GetNamespace(), Name: revised.GetName()}
+	var current appsv1.StatefulSet
+
+	err = c.Get(context.TODO(), namespacedName, &current)
+	if err == nil {
+		currEnv := current.Spec.Template.Spec.Containers[0].Env
+		revEnv := revised.Spec.Template.Spec.Containers[0].Env
+
+		var cEnv, rEnv corev1.EnvVar
+
+		for _, cEnvTemp := range currEnv {
+			if cEnvTemp.Name == "SPLUNK_MONITORING_CONSOLE_REF" {
+				cEnv.Value = cEnvTemp.Value
+			}
+		}
+
+		for _, rEnvTemp := range revEnv {
+			if rEnvTemp.Name == "SPLUNK_MONITORING_CONSOLE_REF" {
+				rEnv.Value = rEnvTemp.Value
+			}
+		}
+
+		if cEnv.Value != "" && rEnv.Value != "" && cEnv.Value != rEnv.Value {
+			//1. if revised Spec has different mcRef defined
+			_, err = ApplyMonitoringConsoleEnvConfigMap(c, current.ObjectMeta.GetNamespace(), current.ObjectMeta.GetName(), cEnv.Value, serviceURLs, false)
+			if err != nil {
+				return err
+			}
+			_, err = ApplyMonitoringConsoleEnvConfigMap(c, current.ObjectMeta.GetNamespace(), current.ObjectMeta.GetName(), rEnv.Value, serviceURLs, true)
+			if err != nil {
+				return err
+			}
+		} else if cEnv.Value != "" && rEnv.Value == "" {
+			//2. if revised Spec doesn't have mcRef defined
+			_, err = ApplyMonitoringConsoleEnvConfigMap(c, current.ObjectMeta.GetNamespace(), current.ObjectMeta.GetName(), cEnv.Value, serviceURLs, false)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	//if the sts doesn't exists no need for any change
+	return nil
+}

pkg/splunk/enterprise/util_test.go

@@ -20,6 +20,7 @@ import (
 	"testing"
 	"time"
 
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
@@ -789,3 +790,111 @@ func TestGetNextRequeueTime(t *testing.T) {
 		t.Errorf("Got wrong next requeue time")
 	}
 }
+
+func TestValidateMonitoringConsoleRef(t *testing.T) {
+	currentCM := corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "splunk-test-monitoring-console",
+			Namespace: "test",
+		},
+		Data: map[string]string{"a": "b"},
+	}
+
+	current := &appsv1.StatefulSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "splunk-s1-standalone",
+			Namespace: "test",
+		},
+		Spec: appsv1.StatefulSetSpec{
+			Template: corev1.PodTemplateSpec{
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Env: []corev1.EnvVar{
+								{
+									Name:  "SPLUNK_MONITORING_CONSOLE_REF",
+									Value: "test",
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	revised := &appsv1.StatefulSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "splunk-s1-standalone",
+			Namespace: "test",
+		},
+		Spec: appsv1.StatefulSetSpec{
+			Template: corev1.PodTemplateSpec{
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Env: []corev1.EnvVar{
+								{
+									Name:  "SPLUNK_MONITORING_CONSOLE_REF",
+									Value: "abc",
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	client := spltest.NewMockClient()
+
+	//create configmap
+	_, err := splctrl.ApplyConfigMap(client, &currentCM)
+	if err != nil {
+		t.Errorf("Failed to create the configMap. Error: %s", err.Error())
+	}
+
+	// Create statefulset
+	err = splutil.CreateResource(client, current)
+	if err != nil {
+		t.Errorf("Failed to create owner reference  %s", current.GetName())
+	}
+
+	var serviceURLs []corev1.EnvVar
+	serviceURLs = []corev1.EnvVar{
+		{
+			Name:  "A",
+			Value: "a",
+		},
+	}
+
+	err = validateMonitoringConsoleRef(client, revised, serviceURLs)
+	if err != nil {
+		t.Errorf("Couldn't validate monitoring console ref %s", current.GetName())
+	}
+
+	revised = &appsv1.StatefulSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "splunk-s1-standalone",
+			Namespace: "test",
+		},
+		Spec: appsv1.StatefulSetSpec{
+			Template: corev1.PodTemplateSpec{
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Env: []corev1.EnvVar{
+								{},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	err = validateMonitoringConsoleRef(client, revised, serviceURLs)
+	if err != nil {
+		t.Errorf("Couldn't validate monitoring console ref %s", current.GetName())
+	}
+}