CSPL-767: Operator trigger Instance pod to download app package from S3

pkg/splunk/client/awss3client.go

@@ -17,6 +17,7 @@ package client
 import (
 	"encoding/json"
 	"fmt"
+	"path/filepath"
 	"regexp"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -153,5 +154,8 @@ func (awsclient *AWSS3Client) GetInitContainerImage() string {
 
 // GetInitContainerCmd returns the init container command on a per app source basis to be used by the initContainer
 func (awsclient *AWSS3Client) GetInitContainerCmd(endpoint string, bucket string, path string, appSrcName string, appMnt string) []string {
-	return ([]string{fmt.Sprintf("--endpoint-url=%s", endpoint), "s3", "sync", fmt.Sprintf("s3://%s/%s", bucket, path), fmt.Sprintf("%s/%s", appMnt, appSrcName)})
+	s3AppSrcPath := filepath.Join(bucket, path) + "/"
+	podSyncPath := filepath.Join(appMnt, appSrcName) + "/"
+
+	return ([]string{fmt.Sprintf("--endpoint-url=%s", endpoint), "s3", "sync", fmt.Sprintf("s3://%s", s3AppSrcPath), podSyncPath})
 }

pkg/splunk/controller/configmap.go

@@ -19,6 +19,7 @@ import (
 	"reflect"
 
 	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 
 	splcommon "github.com/splunk/splunk-operator/pkg/splunk/common"
@@ -75,3 +76,16 @@ func GetConfigMapResourceVersion(client splcommon.ControllerClient, namespacedNa
 	}
 	return configMap.ResourceVersion, nil
 }
+
+// PrepareConfigMap prepares and returns a K8 ConfigMap object for the given data
+func PrepareConfigMap(configMapName, namespace string, dataMap map[string]string) *corev1.ConfigMap {
+	configMap := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      configMapName,
+			Namespace: namespace,
+		},
+	}
+	configMap.Data = dataMap
+
+	return configMap
+}

pkg/splunk/controller/configmap_test.go

@@ -15,6 +15,7 @@
 package controller
 
 import (
+	"reflect"
 	"testing"
 
 	corev1 "k8s.io/api/core/v1"
@@ -101,3 +102,26 @@ func TestGetConfigMapResourceVersion(t *testing.T) {
 		t.Errorf("Should not return an error, when the configMap exists")
 	}
 }
+
+func TestPrepareConfigMap(t *testing.T) {
+	var configMapName = "testConfgMap"
+	var namespace = "testNameSpace"
+	expectedCm := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      configMapName,
+			Namespace: namespace,
+		},
+	}
+
+	dataMap := make(map[string]string)
+	dataMap["a"] = "x"
+	dataMap["b"] = "y"
+	dataMap["z"] = "z"
+	expectedCm.Data = dataMap
+
+	returnedCM := PrepareConfigMap(configMapName, namespace, dataMap)
+
+	if !reflect.DeepEqual(expectedCm, returnedCM) {
+		t.Errorf("configMap preparation failed")
+	}
+}

pkg/splunk/enterprise/clustermaster.go

@@ -83,16 +83,14 @@ func ApplyClusterMaster(client splcommon.ControllerClient, cr *enterprisev1.Clus
 		}
 	}()
 
-	//check if the apps need to be downloaded from remote storage
+	initAppFrameWorkContext(&cr.Spec.AppFrameworkConfig, &cr.Status.AppContext)
+
+	// check if the apps need to be downloaded from remote storage
+	// ToDo: sgontla: Once after we have the flow, move the following logic into a seperate function(Note:- all errors should not cause a return in that function)
+	// ToDo: also take care of the Mock
 	if cr.Spec.CommonSplunkSpec.Mock != true && !reflect.DeepEqual(cr.Status.AppContext.AppFrameworkConfig, cr.Spec.AppFrameworkConfig) {
 		var sourceToAppsList map[string]splclient.S3Response
 
-		for _, vol := range cr.Spec.AppFrameworkConfig.VolList {
-			if _, ok := splclient.S3Clients[vol.Provider]; !ok {
-				splclient.RegisterS3Client(vol.Provider)
-			}
-		}
-
 		sourceToAppsList, err = GetAppListFromS3Bucket(client, cr, &cr.Spec.AppFrameworkConfig)
 		if len(sourceToAppsList) != len(cr.Spec.AppFrameworkConfig.AppSources) {
 			scopedLog.Error(err, "Unable to get apps list, will retry in next reconcile...")
@@ -109,6 +107,11 @@ func ApplyClusterMaster(client splcommon.ControllerClient, cr *enterprisev1.Clus
 			return result, err
 		}
 
+		_, _, err := ApplyAppListingConfigMap(client, cr, &cr.Spec.AppFrameworkConfig, cr.Status.AppContext.AppsSrcDeployStatus)
+		if err != nil {
+			return result, err
+		}
+
 		cr.Status.AppContext.AppFrameworkConfig = cr.Spec.AppFrameworkConfig
 	}
 
@@ -208,9 +211,9 @@ func getClusterMasterStatefulSet(client splcommon.ControllerClient, cr *enterpri
 	if err != nil {
 		return ss, err
 	}
-	_, exists := getSmartstoreConfigMap(client, cr, SplunkClusterMaster)
+	smartStoreConfigMap := getSmartstoreConfigMap(client, cr, SplunkClusterMaster)
 
-	if exists {
+	if smartStoreConfigMap != nil {
 		setupInitContainer(&ss.Spec.Template, cr.Spec.Image, cr.Spec.ImagePullPolicy, commandForCMSmartstore)
 	}
 
@@ -233,9 +236,9 @@ func CheckIfsmartstoreConfigMapUpdatedToPod(c splcommon.ControllerClient, cr *en
 		return fmt.Errorf("Failed to check config token value on pod. stdout=%s, stderror=%s, error=%v", stdOut, stdErr, err)
 	}
 
-	configMap, exists := getSmartstoreConfigMap(c, cr, SplunkClusterMaster)
-	if exists {
-		tokenFromConfigMap := configMap.Data[configToken]
+	smartStoreConfigMap := getSmartstoreConfigMap(c, cr, SplunkClusterMaster)
+	if smartStoreConfigMap != nil {
+		tokenFromConfigMap := smartStoreConfigMap.Data[configToken]
 		if tokenFromConfigMap == stdOut {
 			scopedLog.Info("Token Matched.", "on Pod=", stdOut, "from configMap=", tokenFromConfigMap)
 			return nil

pkg/splunk/enterprise/clustermaster_test.go

@@ -42,6 +42,7 @@ func TestApplyClusterMaster(t *testing.T) {
 		{MetaName: "*v1.Secret-test-splunk-test-secret"},
 		{MetaName: "*v1.Secret-test-splunk-stack1-cluster-master-secret-v1"},
 		{MetaName: "*v1.ConfigMap-test-splunk-stack1-clustermaster-smartstore"},
+		{MetaName: "*v1.ConfigMap-test-splunk-stack1-clustermaster-app-list"},
 		{MetaName: "*v1.ConfigMap-test-splunk-stack1-clustermaster-smartstore"},
 		{MetaName: "*v1.StatefulSet-test-splunk-stack1-cluster-master"},
 	}
@@ -56,8 +57,8 @@ func TestApplyClusterMaster(t *testing.T) {
 	}
 	listmockCall := []spltest.MockFuncCall{
 		{ListOpts: listOpts}}
-	createCalls := map[string][]spltest.MockFuncCall{"Get": funcCalls, "Create": {funcCalls[0], funcCalls[2], funcCalls[3], funcCalls[5], funcCalls[8]}, "List": {listmockCall[0]}, "Update": {funcCalls[0]}}
-	updateCalls := map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[0], funcCalls[2], funcCalls[3], funcCalls[4], funcCalls[5], funcCalls[6], funcCalls[7], funcCalls[8]}, "Update": {funcCalls[8]}, "List": {listmockCall[0]}}
+	createCalls := map[string][]spltest.MockFuncCall{"Get": funcCalls, "Create": {funcCalls[0], funcCalls[2], funcCalls[3], funcCalls[5], funcCalls[9]}, "List": {listmockCall[0]}, "Update": {funcCalls[0]}}
+	updateCalls := map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[0], funcCalls[2], funcCalls[3], funcCalls[4], funcCalls[5], funcCalls[6], funcCalls[7], funcCalls[8], funcCalls[9]}, "Update": {funcCalls[9]}, "List": {listmockCall[0]}}
 
 	current := enterprisev1.ClusterMaster{
 		TypeMeta: metav1.TypeMeta{
@@ -154,6 +155,7 @@ func TestApplyClusterMasterWithSmartstore(t *testing.T) {
 		{MetaName: "*v1.Secret-test-splunk-test-secret"},
 		{MetaName: "*v1.Secret-test-splunk-stack1-cluster-master-secret-v1"},
 		{MetaName: "*v1.ConfigMap-test-splunk-stack1-clustermaster-smartstore"},
+		{MetaName: "*v1.ConfigMap-test-splunk-stack1-clustermaster-app-list"},
 		{MetaName: "*v1.ConfigMap-test-splunk-stack1-clustermaster-smartstore"},
 		{MetaName: "*v1.StatefulSet-test-splunk-stack1-cluster-master"},
 		{MetaName: "*v1.Pod-test-splunk-stack1-cluster-master-0"},
@@ -176,8 +178,8 @@ func TestApplyClusterMasterWithSmartstore(t *testing.T) {
 	}
 	listmockCall := []spltest.MockFuncCall{
 		{ListOpts: listOpts}}
-	createCalls := map[string][]spltest.MockFuncCall{"Get": funcCalls, "Create": {funcCalls[6], funcCalls[7], funcCalls[9], funcCalls[15], funcCalls[16], funcCalls[17], funcCalls[18], funcCalls[20]}, "List": {listmockCall[0], listmockCall[0], listmockCall[0]}, "Update": {funcCalls[0], funcCalls[3], funcCalls[20]}}
-	updateCalls := map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[2], funcCalls[3], funcCalls[5], funcCalls[5], funcCalls[6], funcCalls[7], funcCalls[8], funcCalls[9], funcCalls[11], funcCalls[11], funcCalls[12]}, "Update": {funcCalls[10], funcCalls[12]}, "List": {listmockCall[0]}}
+	createCalls := map[string][]spltest.MockFuncCall{"Get": funcCalls, "Create": {funcCalls[6], funcCalls[7], funcCalls[9], funcCalls[16], funcCalls[17], funcCalls[18], funcCalls[19], funcCalls[21]}, "List": {listmockCall[0], listmockCall[0], listmockCall[0]}, "Update": {funcCalls[0], funcCalls[3], funcCalls[21]}}
+	updateCalls := map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[2], funcCalls[3], funcCalls[5], funcCalls[5], funcCalls[6], funcCalls[7], funcCalls[8], funcCalls[9], funcCalls[10], funcCalls[11], funcCalls[12], funcCalls[13]}, "Update": {funcCalls[10], funcCalls[13]}, "List": {listmockCall[0]}}
 
 	current := enterprisev1.ClusterMaster{
 		TypeMeta: metav1.TypeMeta{
@@ -593,7 +595,7 @@ func TestClusterMasterGetAppsListForAWSS3ClientShouldNotFail(t *testing.T) {
 	var allSuccess bool = true
 	for index, appSource := range appFrameworkRef.AppSources {
 
-		vol, err = GetVolume(client, &cm, appSource, &appFrameworkRef)
+		vol, err = GetAppSrcVolume(client, &cm, appSource, &appFrameworkRef)
 		if err != nil {
 			allSuccess = false
 			continue
@@ -717,7 +719,7 @@ func TestClusterMasterGetAppsListForAWSS3ClientShouldFail(t *testing.T) {
 	var vol enterprisev1.VolumeSpec
 
 	appSource := appFrameworkRef.AppSources[0]
-	vol, err = GetVolume(client, &cm, appSource, &appFrameworkRef)
+	vol, err = GetAppSrcVolume(client, &cm, appSource, &appFrameworkRef)
 	if err != nil {
 		t.Errorf("Unable to get Volume due to error=%s", err)
 	}

pkg/splunk/enterprise/configuration.go

@@ -17,6 +17,7 @@ package enterprise
 import (
 	"context"
 	"fmt"
+	"sort"
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -26,6 +27,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 
 	enterprisev1 "github.com/splunk/splunk-operator/pkg/apis/enterprise/v1"
+	splclient "github.com/splunk/splunk-operator/pkg/splunk/client"
 	splcommon "github.com/splunk/splunk-operator/pkg/splunk/common"
 	splctrl "github.com/splunk/splunk-operator/pkg/splunk/controller"
 	splutil "github.com/splunk/splunk-operator/pkg/splunk/util"
@@ -233,19 +235,6 @@ func getSplunkDefaults(identifier, namespace string, instanceType InstanceType,
 	}
 }
 
-// prepareSplunkSmartstoreConfigMap returns a K8 ConfigMap containing Splunk smartstore config in INI format
-func prepareSplunkSmartstoreConfigMap(identifier, namespace string, crKind string, dataIniMap map[string]string) *corev1.ConfigMap {
-	configMapIni := &corev1.ConfigMap{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      GetSplunkSmartstoreConfigMapName(identifier, crKind),
-			Namespace: namespace,
-		},
-	}
-	configMapIni.Data = dataIniMap
-
-	return configMapIni
-}
-
 // getSplunkPorts returns a map of ports to use for Splunk instances.
 func getSplunkPorts(instanceType InstanceType) map[string]int {
 	result := map[string]int{
@@ -472,26 +461,31 @@ func getSplunkStatefulSet(client splcommon.ControllerClient, cr splcommon.MetaOb
 	return statefulSet, nil
 }
 
-// getSmartstoreConfigMap returns the smartstore configMap, if it exists and applicable for that instanceType
-func getSmartstoreConfigMap(client splcommon.ControllerClient, cr splcommon.MetaObject, instanceType InstanceType) (*corev1.ConfigMap, bool) {
-	var smartStoreConfigMapName string
-	if instanceType == SplunkStandalone || instanceType == SplunkClusterMaster {
-		smartStoreConfigMapName = GetSplunkSmartstoreConfigMapName(cr.GetName(), cr.GetObjectKind().GroupVersionKind().Kind)
+// getAppListingConfigMap returns the App listing configMap, if it exists and applicable for that instanceType
+func getAppListingConfigMap(client splcommon.ControllerClient, cr splcommon.MetaObject, instanceType InstanceType) *corev1.ConfigMap {
+	var configMap *corev1.ConfigMap
+
+	// ToDo: sgontla: Exclude MC, once it's own CR is available
+	if instanceType != SplunkIndexer && instanceType != SplunkSearchHead && instanceType != SplunkMonitoringConsole {
+		appsConfigMapName := GetSplunkAppsConfigMapName(cr.GetName(), cr.GetObjectKind().GroupVersionKind().Kind)
+		namespacedName := types.NamespacedName{Namespace: cr.GetNamespace(), Name: appsConfigMapName}
+		configMap, _ = splctrl.GetConfigMap(client, namespacedName)
 	}
 
-	if smartStoreConfigMapName != "" {
-		namespacedName := types.NamespacedName{Namespace: cr.GetNamespace(), Name: smartStoreConfigMapName}
-		configMap, err := splctrl.GetConfigMap(client, namespacedName)
-		if err != nil {
-			// Do not return configMap name, unless the configMap really exists
-			return nil, false
-		}
+	return configMap
+}
+
+// getSmartstoreConfigMap returns the smartstore configMap, if it exists and applicable for that instanceType
+func getSmartstoreConfigMap(client splcommon.ControllerClient, cr splcommon.MetaObject, instanceType InstanceType) *corev1.ConfigMap {
+	var configMap *corev1.ConfigMap
 
-		return configMap, true
+	if instanceType == SplunkStandalone || instanceType == SplunkClusterMaster {
+		smartStoreConfigMapName := GetSplunkSmartstoreConfigMapName(cr.GetName(), cr.GetObjectKind().GroupVersionKind().Kind)
+		namespacedName := types.NamespacedName{Namespace: cr.GetNamespace(), Name: smartStoreConfigMapName}
+		configMap, _ = splctrl.GetConfigMap(client, namespacedName)
 	}
 
-	// Do not return configMap name, unless the configMap really exists
-	return nil, false
+	return configMap
 }
 
 // updateSplunkPodTemplateWithConfig modifies the podTemplateSpec object based on configuration of the Splunk Enterprise resource.
@@ -561,8 +555,8 @@ func updateSplunkPodTemplateWithConfig(client splcommon.ControllerClient, podTem
 		}
 	}
 
-	smartstoreConfigMap, exists := getSmartstoreConfigMap(client, cr, instanceType)
-	if exists {
+	smartstoreConfigMap := getSmartstoreConfigMap(client, cr, instanceType)
+	if smartstoreConfigMap != nil {
 		addSplunkVolumeToTemplate(podTemplateSpec, "mnt-splunk-operator", "/mnt/splunk-operator/local/", corev1.VolumeSource{
 			ConfigMap: &corev1.ConfigMapVolumeSource{
 				LocalObjectReference: corev1.LocalObjectReference{
@@ -584,7 +578,15 @@ func updateSplunkPodTemplateWithConfig(client splcommon.ControllerClient, podTem
 		if instanceType == SplunkStandalone {
 			podTemplateSpec.ObjectMeta.Annotations[smartStoreConfigRev] = smartstoreConfigMap.ResourceVersion
 		}
+	}
+
+	appListingConfigMap := getAppListingConfigMap(client, cr, instanceType)
+	if appListingConfigMap != nil {
+		appVolumeSource := getVolumeSourceMountFromConfigMapData(appListingConfigMap, &configMapVolDefaultMode)
+		addSplunkVolumeToTemplate(podTemplateSpec, "mnt-app-listing", appConfLocationOnPod, appVolumeSource)
 
+		// ToDo: sgontla: for Phase-2, we always reset the pod, need to change the behavior for phase-3
+		podTemplateSpec.ObjectMeta.Annotations[appListingRev] = appListingConfigMap.ResourceVersion
 	}
 
 	// update security context
@@ -642,6 +644,20 @@ func updateSplunkPodTemplateWithConfig(client splcommon.ControllerClient, podTem
 	if spec.Defaults != "" {
 		splunkDefaults = fmt.Sprintf("%s,%s", "/mnt/splunk-defaults/default.yml", splunkDefaults)
 	}
+	if appListingConfigMap != nil {
+		var appListingFiles []string
+		for key := range appListingConfigMap.Data {
+			if key != appsUpdateToken {
+				appListingFiles = append(appListingFiles, key)
+			}
+		}
+		// Always sort the slice, so that map entries are ordered, to avoid pod resets
+		sort.Strings(appListingFiles)
+
+		for _, fileName := range appListingFiles {
+			splunkDefaults = fmt.Sprintf("%s%s,%s", appConfLocationOnPod, fileName, splunkDefaults)
+		}
+	}
 
 	// prepare container env variables
 	role := instanceType.ToRole()
@@ -730,6 +746,26 @@ func updateSplunkPodTemplateWithConfig(client splcommon.ControllerClient, podTem
 	}
 }
 
+// getVolumeSourceMountFromConfigMapData returns a volume source with the configMap Data entries
+func getVolumeSourceMountFromConfigMapData(configMap *corev1.ConfigMap, mode *int32) corev1.VolumeSource {
+	volumeSource := corev1.VolumeSource{
+		ConfigMap: &corev1.ConfigMapVolumeSource{
+			LocalObjectReference: corev1.LocalObjectReference{
+				Name: configMap.GetName(),
+			},
+			DefaultMode: mode,
+		},
+	}
+
+	for key := range configMap.Data {
+		volumeSource.ConfigMap.Items = append(volumeSource.ConfigMap.Items, corev1.KeyToPath{Key: key, Path: key, Mode: mode})
+	}
+	//  Map traversal order is not guaranteed. Always sort the slice to avoid (random) pod resets due to the ordering
+	splcommon.SortSlice(volumeSource.ConfigMap.Items, splcommon.SortFieldKey)
+
+	return volumeSource
+}
+
 // isSmartstoreEnabled checks and returns true if smartstore is configured
 func isSmartstoreConfigured(smartstore *enterprisev1.SmartStoreSpec) bool {
 	if smartstore == nil {
@@ -785,6 +821,34 @@ func AreRemoteVolumeKeysChanged(client splcommon.ControllerClient, cr splcommon.
 	return false
 }
 
+// initAppFrameWorkContext used to initialize the app frame work context
+func initAppFrameWorkContext(appFrameworkConf *enterprisev1.AppFrameworkSpec, appStatusContext *enterprisev1.AppDeploymentContext) {
+	if appStatusContext.AppsSrcDeployStatus == nil {
+		appStatusContext.AppsSrcDeployStatus = make(map[string]enterprisev1.AppSrcDeployInfo)
+	}
+
+	for _, vol := range appFrameworkConf.VolList {
+		if _, ok := splclient.S3Clients[vol.Provider]; !ok {
+			splclient.RegisterS3Client(vol.Provider)
+		}
+	}
+}
+
+// getAppSrcScope returns the scope of a given appSource
+func getAppSrcScope(appFrameworkConf *enterprisev1.AppFrameworkSpec, appSrcName string) string {
+	for _, appSrc := range appFrameworkConf.AppSources {
+		if appSrc.Name == appSrcName {
+			if appSrc.Scope != "" {
+				return appSrc.Scope
+			}
+
+			break
+		}
+	}
+
+	return appFrameworkConf.Defaults.Scope
+}
+
 // CheckIfAppSrcExistsInConfig returns if the given appSource is available in the configuration or not
 func CheckIfAppSrcExistsInConfig(appFrameworkConf *enterprisev1.AppFrameworkSpec, appSrcName string) bool {
 	for _, appSrc := range appFrameworkConf.AppSources {