Skip to content

Automation for CSPL-1089 #460

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Aug 11, 2021
Merged

Automation for CSPL-1089 #460

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e91798b
Update MC for idxc
kashok-splunk Aug 3, 2021
f168937
For indexer cluster retrieve mcref from cm spec
kashok-splunk Aug 4, 2021
ab2d180
Automation for CSPL-1089
pdhanoyasplunk Aug 10, 2021
73342ab
Merge branch 'feature-MC/CRD' into automation-cspl-1089
pdhanoya-splunk Aug 10, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
133 changes: 120 additions & 13 deletions test/monitoring_console/monitoring_console_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,7 @@ var _ = Describe("Monitoring Console test", func() {
})

Context("Standalone deployment with Scale up", func() {
It("monitoring_console: can deploy a MC with standalone instance and update MC when standalone is scaled up", func() {
It("monitoring_console, integration: can deploy a MC with standalone instance and update MC when standalone is scaled up", func() {
/*
Test Steps
1. Deploy Standalone
Expand Down Expand Up @@ -379,13 +379,28 @@ var _ = Describe("Monitoring Console test", func() {
})
})

XContext("Clustered deployment (C3 - clustered indexer, search head cluster)", func() {
It("monitoring_console, integration: MC can configure SHC, indexer instances after scale up and standalone in a namespace", func() {

Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() {
It("monitoring_console, smoke: MC can configure SHC, indexer instances after scale up and standalone in a namespace", func() {
/*
Test Steps
1. Deploy Single Site Indexer Cluster
2. Deploy Monitoring Console
3. Wait for Monitoring Console status to go back to READY
4. Verify SH are configured in MC Config Map
5. Verify SH are configured in peer strings on MC Pod
6. Verify Monitoring Console Pod has peers in Peer string on MC Pod
7. Scale SH Cluster
8. Scale Indexer Count on One Site
9. Add a standalone
10. Verify Standalone is configured in MC Config Map
11. Verify SH are configured in MC Config Map and Peers String
12. Verify Indexers are configured in Peer String
*/
defaultSHReplicas := 3
defaultIndexerReplicas := 3
err := deployment.DeploySingleSiteCluster(deployment.GetName(), defaultIndexerReplicas, true)
Expect(err).To(Succeed(), "Unable to deploy search head cluster")
mcName := deployment.GetName()
err := deployment.DeploySingleSiteClusterWithGivenMonitoringConsole(deployment.GetName(), defaultIndexerReplicas, true, mcName)
Expect(err).To(Succeed(), "Unable to deploy Cluster Master")

// Ensure that the cluster-master goes to Ready phase
testenv.ClusterMasterReady(deployment, testenvInstance)
Expand All @@ -396,12 +411,22 @@ var _ = Describe("Monitoring Console test", func() {
// Ensure search head cluster go to Ready phase
testenv.SearchHeadClusterReady(deployment, testenvInstance)

// Wait for Monitoring Console Pod to be in READY status
testenv.MCPodReady(testenvInstance.GetName(), deployment)
// Deploy Monitoring Console Pod
mc, err := deployment.DeployMonitoringConsole(deployment.GetName(), "")
Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance")

// Verify Monitoring Console is Ready and stays in ready state
testenv.VerifyMonitoringConsoleReady(deployment, deployment.GetName(), mc, testenvInstance)

// Get contents of MC config map
mcConfigMap, err := testenv.GetMCConfigMap(deployment, testenvInstance.GetName(), mcName)
Expect(err).To(Succeed(), "Unable to get MC config map")

// Check Monitoring console is configured with all search head instances in namespace
for i := 0; i < defaultSHReplicas; i++ {
podName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i)
testenvInstance.Log.Info("Checking for Search Head on MC Config Map", "Search Head Name", podName)
Expect(testenv.CheckPodNameInString(podName, mcConfigMap.Data["SPLUNK_SEARCH_HEAD_URL"])).To(Equal(true))
testenvInstance.Log.Info("Checking for Search Head on MC", "Search Head Name", podName)
found := testenv.CheckPodNameOnMC(testenvInstance.GetName(), deployment.GetName(), podName)
Expect(found).To(Equal(true))
Expand Down Expand Up @@ -452,9 +477,20 @@ var _ = Describe("Monitoring Console test", func() {
// Ensure Indxer cluster scales up and go to ScalingUp phase
testenv.VerifyIndexerClusterPhase(deployment, testenvInstance, splcommon.PhaseScalingUp, idxcName)

// Deploy Standalone
standalone, err := deployment.DeployStandalone(deployment.GetName())
Expect(err).To(Succeed(), "Unable to deploy standalone instance ")
// Deploy Standalone Pod
spec := enterpriseApi.StandaloneSpec{
CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{
Spec: splcommon.Spec{
ImagePullPolicy: "IfNotPresent",
},
Volumes: []corev1.Volume{},
MonitoringConsoleRef: corev1.ObjectReference{
Name: mcName,
},
},
}
standalone, err := deployment.DeployStandaloneWithGivenSpec(deployment.GetName(), spec)
Expect(err).To(Succeed(), "Unable to deploy standalone instance")

// Wait for Standalone to be in READY status
testenv.StandaloneReady(deployment, deployment.GetName(), standalone, testenvInstance)
Expand All @@ -466,17 +502,25 @@ var _ = Describe("Monitoring Console test", func() {
// Adding this check in the end as SHC take the longest time to scale up due recycle of SHC members
testenv.SearchHeadClusterReady(deployment, testenvInstance)

// Wait for Monitoring Console Pod to be in READY status
testenv.MCPodReady(testenvInstance.GetName(), deployment)
// Verify Monitoring Console is Ready and stays in ready state
testenv.VerifyMonitoringConsoleReady(deployment, deployment.GetName(), mc, testenvInstance)

// Get contents of MC config map
mcConfigMap, err = testenv.GetMCConfigMap(deployment, testenvInstance.GetName(), mcName)
Expect(err).To(Succeed(), "Unable to get MC config map")

// Check Standalone configured on Monitoring Console
podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)
testenvInstance.Log.Info("Checking for Standalone Pod on MC Config Map", "Standalone Pod Name", podName)
Expect(testenv.CheckPodNameInString(podName, mcConfigMap.Data["SPLUNK_STANDALONE_URL"])).To(Equal(true))
testenvInstance.Log.Info("Check standalone instance in MC Peer list", "Standalone Pod", podName)
Expect(testenv.CheckPodNameOnMC(testenvInstance.GetName(), deployment.GetName(), podName)).To(Equal(true))

// Verify all Search Head Members are configured on Monitoring Console
for i := 0; i < scaledSHReplicas; i++ {
podName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i)
testenvInstance.Log.Info("Checking for Search Head on MC Config Map", "Search Head Name", podName)
Expect(testenv.CheckPodNameInString(podName, mcConfigMap.Data["SPLUNK_SEARCH_HEAD_URL"])).To(Equal(true))
testenvInstance.Log.Info("Checking for Search Head on MC after adding Standalone", "Search Head Name", podName)
found := testenv.CheckPodNameOnMC(testenvInstance.GetName(), deployment.GetName(), podName)
Expect(found).To(Equal(true))
Expand All @@ -492,4 +536,67 @@ var _ = Describe("Monitoring Console test", func() {
}
})
})

Context("Multisite Clustered deployment (M4 - 3 Site clustered indexer, search head cluster)", func() {
It("monitoring_console, integration: MC can configure SHC, indexer instances after scale up and standalone in a namespace", func() {
/*
Test Steps
1. Deploy Multisite Indexer Cluster
2. Deploy Monitoring Console
3. Wait for Monitoring Console status to go back to READY
4. Verify SH are configured in MC Config Map
5. Verify SH are configured in peer strings on MC Pod
6. Verify Indexers are configured in MC Config Map
7. Verify Monitoring Console Pod has peers in Peer string on MC Pod
*/
defaultSHReplicas := 3
defaultIndexerReplicas := 1
siteCount := 3
mcName := deployment.GetName()
err := deployment.DeployMultisiteClusterWithMonitoringConsole(deployment.GetName(), defaultIndexerReplicas, siteCount, mcName, true)
Expect(err).To(Succeed(), "Unable to deploy Cluster Master")

// Ensure that the cluster-master goes to Ready phase
testenv.ClusterMasterReady(deployment, testenvInstance)

// Ensure indexers go to Ready phase
testenv.IndexersReady(deployment, testenvInstance, siteCount)

// Ensure indexer clustered is configured as multisite
testenv.IndexerClusterMultisiteStatus(deployment, testenvInstance, siteCount)

// Ensure search head cluster go to Ready phase
testenv.SearchHeadClusterReady(deployment, testenvInstance)

// Deploy Monitoring Console Pod
mc, err := deployment.DeployMonitoringConsole(deployment.GetName(), "")
Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance")

// Verify Monitoring Console is Ready and stays in ready state
testenv.VerifyMonitoringConsoleReady(deployment, deployment.GetName(), mc, testenvInstance)

// Get contents of MC config map
mcConfigMap, err := testenv.GetMCConfigMap(deployment, testenvInstance.GetName(), mcName)
Expect(err).To(Succeed(), "Unable to get MC config map")

// Check Monitoring console is configured with all search head instances in namespace
for i := 0; i < defaultSHReplicas; i++ {
podName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i)
testenvInstance.Log.Info("Checking for Search Head on MC Config Map", "Search Head Name", podName)
Expect(testenv.CheckPodNameInString(podName, mcConfigMap.Data["SPLUNK_SEARCH_HEAD_URL"])).To(Equal(true))
testenvInstance.Log.Info("Checking for Search Head on MC", "Search Head Name", podName)
found := testenv.CheckPodNameOnMC(testenvInstance.GetName(), deployment.GetName(), podName)
Expect(found).To(Equal(true))
}

// Check Monitoring console is configured with all Indexer in Name Space
for i := 1; i <= siteCount; i++ {
podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0)
podIP := testenv.GetPodIP(testenvInstance.GetName(), podName)
testenvInstance.Log.Info("Checking for Indexer Pod on MC", "Search Head Name", podName, "IP Address", podIP)
found := testenv.CheckPodNameOnMC(testenvInstance.GetName(), deployment.GetName(), podIP)
Expect(found).To(Equal(true))
}
})
})
})
168 changes: 168 additions & 0 deletions test/testenv/deployment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -741,3 +741,171 @@ func (d *Deployment) DeployMultisiteClusterWithSearchHeadAndAppFramework(name st
}
return nil
}

// DeploySingleSiteClusterWithGivenMonitoringConsole deploys indexer cluster (lm, shc optional) with given monitoring console
func (d *Deployment) DeploySingleSiteClusterWithGivenMonitoringConsole(name string, indexerReplicas int, shc bool, monitoringConsoleName string) error {

licenseMaster := ""

// If license file specified, deploy License Master
if d.testenv.licenseFilePath != "" {
// Deploy the license master
_, err := d.DeployLicenseMaster(name)
if err != nil {
return err
}

licenseMaster = name
}

// Deploy the cluster master
cmSpec := enterpriseApi.ClusterMasterSpec{
CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{
Spec: splcommon.Spec{
ImagePullPolicy: "Always",
},
Volumes: []corev1.Volume{},
LicenseMasterRef: corev1.ObjectReference{
Name: licenseMaster,
},
MonitoringConsoleRef: corev1.ObjectReference{
Name: monitoringConsoleName,
},
},
}
_, err := d.DeployClusterMasterWithGivenSpec(name, cmSpec)
if err != nil {
return err
}

// Deploy the indexer cluster
_, err = d.DeployIndexerCluster(name+"-idxc", licenseMaster, indexerReplicas, name, "")
if err != nil {
return err
}

shSpec := enterpriseApi.SearchHeadClusterSpec{
CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{
Spec: splcommon.Spec{
ImagePullPolicy: "Always",
},
Volumes: []corev1.Volume{},
ClusterMasterRef: corev1.ObjectReference{
Name: name,
},
LicenseMasterRef: corev1.ObjectReference{
Name: licenseMaster,
},
MonitoringConsoleRef: corev1.ObjectReference{
Name: monitoringConsoleName,
},
},
Replicas: 3,
}
if shc {
_, err = d.DeploySearchHeadClusterWithGivenSpec(name+"-shc", shSpec)
if err != nil {
return err
}
}

return nil
}

// DeployMultisiteClusterWithMonitoringConsole deploys cluster-master, indexers in multiple sites (SHC LM Optional) with monitoring console
func (d *Deployment) DeployMultisiteClusterWithMonitoringConsole(name string, indexerReplicas int, siteCount int, monitoringConsoleName string, shc bool) error {

licenseMaster := ""

// If license file specified, deploy License Master
if d.testenv.licenseFilePath != "" {
// Deploy the license master
_, err := d.DeployLicenseMaster(name)
if err != nil {
return err
}

licenseMaster = name
}

// Deploy the cluster-master
defaults := `splunk:
multisite_master: localhost
all_sites: site1,site2,site3
site: site1
multisite_replication_factor_origin: 1
multisite_replication_factor_total: 2
multisite_search_factor_origin: 1
multisite_search_factor_total: 2
idxc:
search_factor: 2
replication_factor: 2
`

// Cluster Master Spec
cmSpec := enterpriseApi.ClusterMasterSpec{
CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{
Spec: splcommon.Spec{
ImagePullPolicy: "Always",
},
Volumes: []corev1.Volume{},
LicenseMasterRef: corev1.ObjectReference{
Name: licenseMaster,
},
Defaults: defaults,
MonitoringConsoleRef: corev1.ObjectReference{
Name: monitoringConsoleName,
},
},
}

_, err := d.DeployClusterMasterWithGivenSpec(name, cmSpec)
if err != nil {
return err
}

// Deploy indexer sites
for site := 1; site <= siteCount; site++ {
siteName := fmt.Sprintf("site%d", site)
siteDefaults := fmt.Sprintf(`splunk:
multisite_master: splunk-%s-cluster-master-service
site: %s
`, name, siteName)
_, err := d.DeployIndexerCluster(name+"-"+siteName, licenseMaster, indexerReplicas, name, siteDefaults)
if err != nil {
return err
}
}

siteDefaults := fmt.Sprintf(`splunk:
multisite_master: splunk-%s-cluster-master-service
site: site0
`, name)
// Deploy the SH cluster
shSpec := enterpriseApi.SearchHeadClusterSpec{
CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{
Spec: splcommon.Spec{
ImagePullPolicy: "Always",
},
Volumes: []corev1.Volume{},
ClusterMasterRef: corev1.ObjectReference{
Name: name,
},
LicenseMasterRef: corev1.ObjectReference{
Name: licenseMaster,
},
Defaults: siteDefaults,
MonitoringConsoleRef: corev1.ObjectReference{
Name: monitoringConsoleName,
},
},
Replicas: 3,
}
if shc {
_, err = d.DeploySearchHeadClusterWithGivenSpec(name+"-shc", shSpec)
if err != nil {
return err
}
}
return nil
}