CSPL-663 Optimize deployment of Splunk apps on SHC

deploy/crds/enterprise.splunk.com_clustermasters_crd.yaml

@@ -659,6 +659,10 @@ spec:
               description: Full path or URL for one or more default.yml files, separated
                 by commas
               type: string
+            defaultsUrlApps:
+              description: Full path or URL for one or more default.yml files for installing apps, separated
+                by commas
+              type: string
             ephemeralStorage:
               description: If true, ephemeral (emptyDir) storage will be used for
                 /opt/splunk/etc and /opt/splunk/var volumes

deploy/crds/enterprise.splunk.com_licensemasters_crd.yaml

@@ -664,6 +664,10 @@ spec:
               description: Full path or URL for one or more default.yml files, separated
                 by commas
               type: string
+            defaultsUrlApps:
+              description: Full path or URL for one or more default.yml files for installing apps, separated
+                by commas
+              type: string
             ephemeralStorage:
               description: If true, ephemeral (emptyDir) storage will be used for
                 /opt/splunk/etc and /opt/splunk/var volumes

deploy/crds/enterprise.splunk.com_searchheadclusters_crd.yaml

@@ -681,6 +681,10 @@ spec:
               description: Full path or URL for one or more default.yml files, separated
                 by commas
               type: string
+            defaultsUrlApps:
+              description: Full path or URL for one or more default.yml files for installing apps, separated
+                by commas
+              type: string
             ephemeralStorage:
               description: If true, ephemeral (emptyDir) storage will be used for
                 /opt/splunk/etc and /opt/splunk/var volumes

deploy/crds/enterprise.splunk.com_standalones_crd.yaml

@@ -674,6 +674,10 @@ spec:
               description: Full path or URL for one or more default.yml files, separated
                 by commas
               type: string
+            defaultsUrlApps:
+              description: Full path or URL for one or more default.yml files for installing apps, separated
+                by commas
+              type: string
             ephemeralStorage:
               description: If true, ephemeral (emptyDir) storage will be used for
                 /opt/splunk/etc and /opt/splunk/var volumes

docs/Examples.md

@@ -504,6 +504,32 @@ You can also install apps hosted remotely using URLs:
         - "https://example.com/splunk-apps/app3.tgz"
 ```
 
+Also these application configuration parameters can be placed in a `defaults.yml`
+file and use the `defaultsUrlApps` parameter.  The `defaultsUrlApps` parameter
+is specific for applicastion installation and will install the apps in the
+correct instances as per the deployment.
+
+Unlike `defaultsUrl` which is applied at every instance created by the CR, the
+`defaultsUrlApps` will be applied on instances that will **not** get the application
+installed via a bundle push.  Search head and indexer cluster members will not have
+the `defaultsUrlApps` parameter applied.  This means:
+
+ - For Standalone & License Master, these applications will be installed as normal.
+ - For SearchHeadClusters, these applications will only be installed on the SHC Deployer
+and pushed to the members via SH Bundle push.
+ - For IndexerClusters, these applications will only be installed on the ClusterMaster
+and pushed to the indexers in the cluster via CM Bundle push.
+
+For application installation the preferred method will be through the `defaultsUrlApps`
+while other ansible defaults can be still be installed via `defaultsUrl`.  For backwards
+compatibility applications could be installed via `defaultsUrl` though this is not
+recommended.  Both options can be used in conjunction:
+
+```yaml
+    defaultsUrl : "http://myco.com/splunk/generic.yml"
+    defaultsUrlApps: "/mnt/defaults/apps.yml"
+```
+
 
 ## Using Apps for Splunk Configuration
 

pkg/apis/enterprise/v1beta1/common_types.go

@@ -58,6 +58,12 @@ type CommonSplunkSpec struct {
 	// Full path or URL for one or more default.yml files, separated by commas
 	DefaultsURL string `json:"defaultsUrl"`
 
+	// Full path or URL for one or more default.yml files specific to App install, separated by commas
+	// This is meant as a temporary fix until the SHC Deployer has its own CRD.  The defaults listed here will
+	// only be installed on a standalone, or a SHC Deployer or IDC CM to be pushed to SH/IDXs in a cluster.
+	// This parameter is ignored on individual SHs or IDXs within a cluster
+	DefaultsURLApps string `json:"defaultsUrlApps"`
+
 	// Full path or URL for a Splunk Enterprise license file
 	LicenseURL string `json:"licenseUrl"`
 

pkg/splunk/enterprise/clustermaster_test.go

@@ -124,6 +124,10 @@ func TestGetClusterMasterStatefulSet(t *testing.T) {
 	cr.Spec.LicenseMasterRef.Name = ""
 	cr.Spec.LicenseURL = "/mnt/splunk.lic"
 	test(`{"kind":"StatefulSet","apiVersion":"apps/v1","metadata":{"name":"splunk-stack1-cluster-master","namespace":"test","creationTimestamp":null,"ownerReferences":[{"apiVersion":"","kind":"","name":"stack1","uid":"","controller":true}]},"spec":{"replicas":1,"selector":{"matchLabels":{"app.kubernetes.io/component":"indexer","app.kubernetes.io/instance":"splunk-stack1-cluster-master","app.kubernetes.io/managed-by":"splunk-operator","app.kubernetes.io/name":"cluster-master","app.kubernetes.io/part-of":"splunk-stack1-indexer"}},"template":{"metadata":{"creationTimestamp":null,"labels":{"app.kubernetes.io/component":"indexer","app.kubernetes.io/instance":"splunk-stack1-cluster-master","app.kubernetes.io/managed-by":"splunk-operator","app.kubernetes.io/name":"cluster-master","app.kubernetes.io/part-of":"splunk-stack1-indexer"},"annotations":{"traffic.sidecar.istio.io/excludeOutboundPorts":"8089,8191,9997,7777,9000,17000,17500,19000","traffic.sidecar.istio.io/includeInboundPorts":"8000"}},"spec":{"volumes":[{"name":"mnt-splunk-secrets","secret":{"secretName":"splunk-stack1-cluster-master-secret-v1","defaultMode":420}}],"containers":[{"name":"splunk","image":"splunk/splunk","ports":[{"name":"splunkweb","containerPort":8000,"protocol":"TCP"},{"name":"splunkd","containerPort":8089,"protocol":"TCP"}],"env":[{"name":"SPLUNK_HOME","value":"/opt/splunk"},{"name":"SPLUNK_START_ARGS","value":"--accept-license"},{"name":"SPLUNK_DEFAULTS_URL","value":"/mnt/splunk-secrets/default.yml"},{"name":"SPLUNK_HOME_OWNERSHIP_ENFORCEMENT","value":"false"},{"name":"SPLUNK_ROLE","value":"splunk_cluster_master"},{"name":"SPLUNK_DECLARATIVE_ADMIN_PASSWORD","value":"true"},{"name":"SPLUNK_LICENSE_URI","value":"/mnt/splunk.lic"},{"name":"SPLUNK_CLUSTER_MASTER_URL","value":"localhost"}],"resources":{"limits":{"cpu":"4","memory":"8Gi"},"requests":{"cpu":"100m","memory":"512Mi"}},"volumeMounts":[{"name":"pvc-etc","mountPath":"/opt/splunk/etc"},{"name":"pvc-var","mountPath":"/opt/splunk/var"},{"name":"mnt-splunk-secrets","mountPath":"/mnt/splunk-secrets"}],"livenessProbe":{"exec":{"command":["/sbin/checkstate.sh"]},"initialDelaySeconds":300,"timeoutSeconds":30,"periodSeconds":30},"readinessProbe":{"exec":{"command":["/bin/grep","started","/opt/container_artifact/splunk-container.state"]},"initialDelaySeconds":10,"timeoutSeconds":5,"periodSeconds":5},"imagePullPolicy":"IfNotPresent"}],"securityContext":{"runAsUser":41812,"fsGroup":41812},"affinity":{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"weight":100,"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app.kubernetes.io/instance","operator":"In","values":["splunk-stack1-cluster-master"]}]},"topologyKey":"kubernetes.io/hostname"}}]}},"schedulerName":"default-scheduler"}},"volumeClaimTemplates":[{"metadata":{"name":"pvc-etc","namespace":"test","creationTimestamp":null,"labels":{"app.kubernetes.io/component":"indexer","app.kubernetes.io/instance":"splunk-stack1-cluster-master","app.kubernetes.io/managed-by":"splunk-operator","app.kubernetes.io/name":"cluster-master","app.kubernetes.io/part-of":"splunk-stack1-indexer"}},"spec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"10Gi"}}},"status":{}},{"metadata":{"name":"pvc-var","namespace":"test","creationTimestamp":null,"labels":{"app.kubernetes.io/component":"indexer","app.kubernetes.io/instance":"splunk-stack1-cluster-master","app.kubernetes.io/managed-by":"splunk-operator","app.kubernetes.io/name":"cluster-master","app.kubernetes.io/part-of":"splunk-stack1-indexer"}},"spec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"100Gi"}}},"status":{}}],"serviceName":"splunk-stack1-cluster-master-headless","podManagementPolicy":"Parallel","updateStrategy":{"type":"OnDelete"}},"status":{"replicas":0}}`)
+
+	cr.Spec.DefaultsURLApps = "/mnt/apps/apps.yml"
+	test(`{"kind":"StatefulSet","apiVersion":"apps/v1","metadata":{"name":"splunk-stack1-cluster-master","namespace":"test","creationTimestamp":null,"ownerReferences":[{"apiVersion":"","kind":"","name":"stack1","uid":"","controller":true}]},"spec":{"replicas":1,"selector":{"matchLabels":{"app.kubernetes.io/component":"indexer","app.kubernetes.io/instance":"splunk-stack1-cluster-master","app.kubernetes.io/managed-by":"splunk-operator","app.kubernetes.io/name":"cluster-master","app.kubernetes.io/part-of":"splunk-stack1-indexer"}},"template":{"metadata":{"creationTimestamp":null,"labels":{"app.kubernetes.io/component":"indexer","app.kubernetes.io/instance":"splunk-stack1-cluster-master","app.kubernetes.io/managed-by":"splunk-operator","app.kubernetes.io/name":"cluster-master","app.kubernetes.io/part-of":"splunk-stack1-indexer"},"annotations":{"traffic.sidecar.istio.io/excludeOutboundPorts":"8089,8191,9997,7777,9000,17000,17500,19000","traffic.sidecar.istio.io/includeInboundPorts":"8000"}},"spec":{"volumes":[{"name":"mnt-splunk-secrets","secret":{"secretName":"splunk-stack1-cluster-master-secret-v1","defaultMode":420}}],"containers":[{"name":"splunk","image":"splunk/splunk","ports":[{"name":"splunkweb","containerPort":8000,"protocol":"TCP"},{"name":"splunkd","containerPort":8089,"protocol":"TCP"}],"env":[{"name":"SPLUNK_HOME","value":"/opt/splunk"},{"name":"SPLUNK_START_ARGS","value":"--accept-license"},{"name":"SPLUNK_DEFAULTS_URL","value":"/mnt/apps/apps.yml,/mnt/splunk-secrets/default.yml"},{"name":"SPLUNK_HOME_OWNERSHIP_ENFORCEMENT","value":"false"},{"name":"SPLUNK_ROLE","value":"splunk_cluster_master"},{"name":"SPLUNK_DECLARATIVE_ADMIN_PASSWORD","value":"true"},{"name":"SPLUNK_LICENSE_URI","value":"/mnt/splunk.lic"},{"name":"SPLUNK_CLUSTER_MASTER_URL","value":"localhost"}],"resources":{"limits":{"cpu":"4","memory":"8Gi"},"requests":{"cpu":"100m","memory":"512Mi"}},"volumeMounts":[{"name":"pvc-etc","mountPath":"/opt/splunk/etc"},{"name":"pvc-var","mountPath":"/opt/splunk/var"},{"name":"mnt-splunk-secrets","mountPath":"/mnt/splunk-secrets"}],"livenessProbe":{"exec":{"command":["/sbin/checkstate.sh"]},"initialDelaySeconds":300,"timeoutSeconds":30,"periodSeconds":30},"readinessProbe":{"exec":{"command":["/bin/grep","started","/opt/container_artifact/splunk-container.state"]},"initialDelaySeconds":10,"timeoutSeconds":5,"periodSeconds":5},"imagePullPolicy":"IfNotPresent"}],"securityContext":{"runAsUser":41812,"fsGroup":41812},"affinity":{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"weight":100,"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app.kubernetes.io/instance","operator":"In","values":["splunk-stack1-cluster-master"]}]},"topologyKey":"kubernetes.io/hostname"}}]}},"schedulerName":"default-scheduler"}},"volumeClaimTemplates":[{"metadata":{"name":"pvc-etc","namespace":"test","creationTimestamp":null,"labels":{"app.kubernetes.io/component":"indexer","app.kubernetes.io/instance":"splunk-stack1-cluster-master","app.kubernetes.io/managed-by":"splunk-operator","app.kubernetes.io/name":"cluster-master","app.kubernetes.io/part-of":"splunk-stack1-indexer"}},"spec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"10Gi"}}},"status":{}},{"metadata":{"name":"pvc-var","namespace":"test","creationTimestamp":null,"labels":{"app.kubernetes.io/component":"indexer","app.kubernetes.io/instance":"splunk-stack1-cluster-master","app.kubernetes.io/managed-by":"splunk-operator","app.kubernetes.io/name":"cluster-master","app.kubernetes.io/part-of":"splunk-stack1-indexer"}},"spec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"100Gi"}}},"status":{}}],"serviceName":"splunk-stack1-cluster-master-headless","podManagementPolicy":"Parallel","updateStrategy":{"type":"OnDelete"}},"status":{"replicas":0}}`)
+
 }
 
 func TestApplyClusterMasterWithSmartstore(t *testing.T) {

pkg/splunk/enterprise/configuration.go

@@ -590,6 +590,14 @@ func updateSplunkPodTemplateWithConfig(client splcommon.ControllerClient, podTem
 
 	// prepare defaults variable
 	splunkDefaults := "/mnt/splunk-secrets/default.yml"
+	// Check for apps defaults and add it to only the standalone or deployer/cm instances
+	if spec.DefaultsURLApps != "" &&
+		(instanceType == SplunkDeployer ||
+			instanceType == SplunkStandalone ||
+			instanceType == SplunkClusterMaster ||
+			instanceType == SplunkLicenseMaster) {
+		splunkDefaults = fmt.Sprintf("%s,%s", spec.DefaultsURLApps, splunkDefaults)
+	}
 	if spec.DefaultsURL != "" {
 		splunkDefaults = fmt.Sprintf("%s,%s", spec.DefaultsURL, splunkDefaults)
 	}