From 7d50c99ea1d39d8ab837d2d428da088aee6f5534 Mon Sep 17 00:00:00 2001 From: Param Dhanoya Date: Wed, 17 Mar 2021 15:52:17 -0700 Subject: [PATCH] CSPL-912: Scaling test for S1 C4 M4 SVA --- .circleci/config.yml | 2 +- .../monitoring_console_test.go | 7 +- test/scaling_test/scaling_suite_test.go | 61 +++ test/scaling_test/scaling_test.go | 398 ++++++++++++++++++ test/testenv/cmutil.go | 89 ++-- test/testenv/search_utils.go | 4 + test/testenv/verificationutils.go | 43 +- 7 files changed, 552 insertions(+), 52 deletions(-) create mode 100644 test/scaling_test/scaling_suite_test.go create mode 100644 test/scaling_test/scaling_test.go diff --git a/.circleci/config.yml b/.circleci/config.yml index 503c3e117..97834bf06 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -282,7 +282,7 @@ jobs: mkdir -p /tmp/test-results find ./test -name "*junit.xml" -exec cp {} /tmp/test-results \; environment: - TEST_FOCUS: "smoke|ingest_search|monitoring_console|smartstore|licensemaster" + TEST_FOCUS: "smoke|ingest_search|monitoring_console|smartstore|licensemaster|scaling_test" - store_test_results: name: Save test results path: /tmp/test-results diff --git a/test/monitoring_console/monitoring_console_test.go b/test/monitoring_console/monitoring_console_test.go index 8b6fd8990..fb6cffd36 100644 --- a/test/monitoring_console/monitoring_console_test.go +++ b/test/monitoring_console/monitoring_console_test.go @@ -21,6 +21,7 @@ import ( . "github.com/onsi/gomega" enterprisev1 "github.com/splunk/splunk-operator/pkg/apis/enterprise/v1" + splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" "github.com/splunk/splunk-operator/test/testenv" ) @@ -138,7 +139,7 @@ var _ = Describe("Monitoring Console test", func() { Expect(err).To(Succeed(), "Failed to scale Standalone") // Ensure standalone is scaling up - testenv.VerifyStandaloneScalingUp(deployment, testenvInstance) + testenv.VerifyStandalonePhase(deployment, testenvInstance, deployment.GetName(), splcommon.PhaseScalingUp) // Wait for Standalone to be in READY status testenv.StandaloneReady(deployment, deployment.GetName(), standalone, testenvInstance) @@ -213,7 +214,7 @@ var _ = Describe("Monitoring Console test", func() { Expect(err).To(Succeed(), "Failed to scale Search Head Cluster") // Ensure Search Head cluster scales up and go to ScalingUp phase - testenv.VerifySearchHeadClusterScalingUp(deployment, testenvInstance) + testenv.VerifySearchHeadClusterPhase(deployment, testenvInstance, splcommon.PhaseScalingUp) // Scale indexers scaledIndexerReplicas := defaultIndexerReplicas + 1 @@ -231,7 +232,7 @@ var _ = Describe("Monitoring Console test", func() { Expect(err).To(Succeed(), "Failed to scale Indxer Cluster") // Ensure Indxer cluster scales up and go to ScalingUp phase - testenv.VerifyIndexerClusterScalingUp(deployment, testenvInstance) + testenv.VerifyIndexerClusterPhase(deployment, testenvInstance, splcommon.PhaseScalingUp, idxcName) // Deploy Standalone standalone, err := deployment.DeployStandalone(deployment.GetName()) diff --git a/test/scaling_test/scaling_suite_test.go b/test/scaling_test/scaling_suite_test.go new file mode 100644 index 000000000..fbd92d64f --- /dev/null +++ b/test/scaling_test/scaling_suite_test.go @@ -0,0 +1,61 @@ +// Copyright (c) 2018-2021 Splunk Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scalingtest + +import ( + "testing" + "time" + + . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" + + "github.com/splunk/splunk-operator/test/testenv" +) + +const ( + // PollInterval specifies the polling interval + PollInterval = 5 * time.Second + + // ConsistentPollInterval is the interval to use to consistently check a state is stable + ConsistentPollInterval = 200 * time.Millisecond + ConsistentDuration = 2000 * time.Millisecond +) + +var ( + testenvInstance *testenv.TestEnv + testSuiteName = "scaling-" + testenv.RandomDNSName(2) +) + +// TestBasic is the main entry point +func TestBasic(t *testing.T) { + + RegisterFailHandler(Fail) + + junitReporter := reporters.NewJUnitReporter(testSuiteName + "_junit.xml") + RunSpecsWithDefaultAndCustomReporters(t, "Running "+testSuiteName, []Reporter{junitReporter}) +} + +var _ = BeforeSuite(func() { + var err error + testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) + Expect(err).ToNot(HaveOccurred()) +}) + +var _ = AfterSuite(func() { + if testenvInstance != nil { + Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) + } +}) diff --git a/test/scaling_test/scaling_test.go b/test/scaling_test/scaling_test.go new file mode 100644 index 000000000..49a5ab463 --- /dev/null +++ b/test/scaling_test/scaling_test.go @@ -0,0 +1,398 @@ +// Copyright (c) 2018-2021 Splunk Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package scalingtest + +import ( + "encoding/json" + "fmt" + "strings" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + enterprisev1 "github.com/splunk/splunk-operator/pkg/apis/enterprise/v1" + splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" + "github.com/splunk/splunk-operator/test/testenv" +) + +var _ = Describe("Scaling test", func() { + + var deployment *testenv.Deployment + + BeforeEach(func() { + var err error + deployment, err = testenvInstance.NewDeployment(testenv.RandomDNSName(3)) + Expect(err).To(Succeed(), "Unable to create deployment") + }) + + AfterEach(func() { + // When a test spec failed, skip the teardown so we can troubleshoot. + if CurrentGinkgoTestDescription().Failed { + testenvInstance.SkipTeardown = true + } + if !testenvInstance.SkipTeardown { + testenv.DeleteMCPod(testenvInstance.GetName()) + } + if deployment != nil { + deployment.Teardown() + } + }) + + Context("Standalone deployment (S1)", func() { + It("scaling_test: Can Scale Up and Scale Down Standalone CR", func() { + + standalone, err := deployment.DeployStandalone(deployment.GetName()) + Expect(err).To(Succeed(), "Unable to deploy standalone instance ") + + // Wait for Standalone to be in READY status + testenv.StandaloneReady(deployment, deployment.GetName(), standalone, testenvInstance) + + // Wait for Monitoring Console Pod to be in READY status + testenv.MCPodReady(testenvInstance.GetName(), deployment) + + // Check Monitoring console is configured with all standalone instances in namespace + peerList := testenv.GetConfiguredPeers(testenvInstance.GetName()) + testenvInstance.Log.Info("Peer List", "instance", peerList) + + // Scale Standalone instance + testenvInstance.Log.Info("Scaling Up Standalone CR") + scaledReplicaCount := 2 + standalone = &enterprisev1.Standalone{} + err = deployment.GetInstance(deployment.GetName(), standalone) + Expect(err).To(Succeed(), "Failed to get instance of Standalone") + + standalone.Spec.Replicas = int32(scaledReplicaCount) + + err = deployment.UpdateCR(standalone) + Expect(err).To(Succeed(), "Failed to scale up Standalone") + + // Ensure standalone is scaling up + testenv.VerifyStandalonePhase(deployment, testenvInstance, deployment.GetName(), splcommon.PhaseScalingUp) + + // Wait for Standalone to be in READY status + testenv.VerifyStandalonePhase(deployment, testenvInstance, deployment.GetName(), splcommon.PhaseReady) + + // Wait for Monitoring Console Pod to be in READY status + testenv.MCPodReady(testenvInstance.GetName(), deployment) + + // Scale Down Standalone + testenvInstance.Log.Info("Scaling Down Standalone CR") + scaledReplicaCount = scaledReplicaCount - 1 + standalone = &enterprisev1.Standalone{} + err = deployment.GetInstance(deployment.GetName(), standalone) + Expect(err).To(Succeed(), "Failed to get instance of Standalone") + + standalone.Spec.Replicas = int32(scaledReplicaCount) + + err = deployment.UpdateCR(standalone) + Expect(err).To(Succeed(), "Failed to scale down Standalone") + + // Ensure standalone is scaling down + testenv.VerifyStandalonePhase(deployment, testenvInstance, deployment.GetName(), splcommon.PhaseScalingDown) + + // Wait for Standalone to be in READY status + testenv.StandaloneReady(deployment, deployment.GetName(), standalone, testenvInstance) + + // Wait for Monitoring Console Pod to be in READY status + testenv.MCPodReady(testenvInstance.GetName(), deployment) + }) + }) + + Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + It("scaling_test: SHC and IDXC can be scaled up and data is searchable", func() { + + defaultSHReplicas := 3 + defaultIndexerReplicas := 3 + err := deployment.DeploySingleSiteCluster(deployment.GetName(), defaultIndexerReplicas, true) + Expect(err).To(Succeed(), "Unable to deploy search head cluster") + + // Ensure that the cluster-master goes to Ready phase + testenv.ClusterMasterReady(deployment, testenvInstance) + + // Ensure indexers go to Ready phase + testenv.SingleSiteIndexersReady(deployment, testenvInstance) + + // Ensure search head cluster go to Ready phase + testenv.SearchHeadClusterReady(deployment, testenvInstance) + + // Wait for Monitoring Console Pod to be in READY status + testenv.MCPodReady(testenvInstance.GetName(), deployment) + + // Scale Search Head Cluster + scaledSHReplicas := defaultSHReplicas + 1 + testenvInstance.Log.Info("Scaling up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) + shcName := deployment.GetName() + "-shc" + + // Get instance of current SHC CR with latest config + shc := &enterprisev1.SearchHeadCluster{} + err = deployment.GetInstance(shcName, shc) + Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + + // Update Replicas of SHC + shc.Spec.Replicas = int32(scaledSHReplicas) + err = deployment.UpdateCR(shc) + Expect(err).To(Succeed(), "Failed to scale Search Head Cluster") + + // Ensure Search Head cluster scales up and go to ScalingUp phase + testenv.VerifySearchHeadClusterPhase(deployment, testenvInstance, splcommon.PhaseScalingUp) + + // Scale indexers + scaledIndexerReplicas := defaultIndexerReplicas + 1 + testenvInstance.Log.Info("Scaling up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) + idxcName := deployment.GetName() + "-idxc" + + // Get instance of current Indexer CR with latest config + idxc := &enterprisev1.IndexerCluster{} + err = deployment.GetInstance(idxcName, idxc) + Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + + // Update Replicas of Indexer Cluster + idxc.Spec.Replicas = int32(scaledIndexerReplicas) + err = deployment.UpdateCR(idxc) + Expect(err).To(Succeed(), "Failed to scale Indxer Cluster") + + // Ensure Indexer cluster scales up and go to ScalingUp phase + testenv.VerifyIndexerClusterPhase(deployment, testenvInstance, splcommon.PhaseScalingUp, idxcName) + + // Ensure Indexer cluster go to Ready phase + testenv.SingleSiteIndexersReady(deployment, testenvInstance) + + // Verify New Indexer On Cluster Master + indexerName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), scaledIndexerReplicas-1) + testenvInstance.Log.Info("Checking for Indexer On CM", "Indexer Name", indexerName) + Expect(testenv.CheckIndexerOnCM(deployment, indexerName)).To(Equal(true)) + + // Ingest data on Indexers + for i := 0; i < scaledIndexerReplicas; i++ { + podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) + logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) + testenv.CreateMockLogfile(logFile, 2000) + testenv.IngestFileViaMonitor(logFile, "main", podName, deployment) + } + + // Ensure Search Head Cluster go to Ready Phase + // Adding this check in the end as SHC take the longest time to scale up due recycle of SHC members + testenv.SearchHeadClusterReady(deployment, testenvInstance) + + // Wait for Monitoring Console Pod to be in READY status + testenv.MCPodReady(testenvInstance.GetName(), deployment) + + // Verify New SearchHead is added to Cluster Master + searchHeadName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) + testenvInstance.Log.Info("Checking for Search Head On CM", "Search Head Name", searchHeadName) + Expect(testenv.CheckSearchHeadOnCM(deployment, searchHeadName)).To(Equal(true)) + + // Wait for RF SF is Met + testenv.VerifyRFSFMet(deployment, testenvInstance) + + // Search for data on newly added indexer + searchPod := searchHeadName + searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) + searchResultsResp, err := testenv.PerformSearchSync(searchPod, searchString, deployment) + Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) + + // Verify result + searchResponse := strings.Split(searchResultsResp, "\n")[0] + var searchResults map[string]interface{} + jsonErr := json.Unmarshal([]byte(searchResponse), &searchResults) + Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) + + testenvInstance.Log.Info("Search results :", "searchResults", searchResults["result"]) + Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) + + resultLine := searchResults["result"].(map[string]interface{}) + testenvInstance.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) + testHostname := strings.Compare(resultLine["host"].(string), indexerName) + Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) + + // Scale Down Indexer Cluster + testenvInstance.Log.Info("Scaling Down Indexer Cluster", "Current Replicas", scaledIndexerReplicas, "New Replicas", scaledIndexerReplicas-1) + scaledIndexerReplicas = scaledIndexerReplicas - 1 + idxcName = deployment.GetName() + "-idxc" + + // Get instance of current Indexer CR with latest config + idxc = &enterprisev1.IndexerCluster{} + err = deployment.GetInstance(idxcName, idxc) + Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + + // Update Replicas of Indexer Cluster + idxc.Spec.Replicas = int32(scaledIndexerReplicas) + err = deployment.UpdateCR(idxc) + Expect(err).To(Succeed(), "Failed to scale down Indxer Cluster") + + // Ensure Indxer cluster scales Down and go to ScalingDown phase + testenv.VerifyIndexerClusterPhase(deployment, testenvInstance, splcommon.PhaseScalingDown, idxcName) + + // Ensure Indexer cluster go to Ready phase + testenv.SingleSiteIndexersReady(deployment, testenvInstance) + + // Wait for RF SF is Met + testenv.VerifyRFSFMet(deployment, testenvInstance) + + // Search for data from removed indexer + searchPod = searchHeadName + searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) + searchResultsResp, err = testenv.PerformSearchSync(searchPod, searchString, deployment) + Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) + + // Verify result + searchResponse = strings.Split(searchResultsResp, "\n")[0] + jsonErr = json.Unmarshal([]byte(searchResponse), &searchResults) + Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) + + testenvInstance.Log.Info("Search results :", "searchResults", searchResults["result"]) + Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) + + resultLine = searchResults["result"].(map[string]interface{}) + testenvInstance.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) + testHostname = strings.Compare(resultLine["host"].(string), indexerName) + Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) + + }) + }) + + Context("Multisite Indexer Cluster (M4 - Multisite indexer Cluster, search head cluster)", func() { + It("scaling_test: Multisite IDXC can be scaled up and data is searchable", func() { + + defaultIndexerReplicas := 1 + siteCount := 3 + err := deployment.DeployMultisiteClusterWithSearchHead(deployment.GetName(), defaultIndexerReplicas, siteCount) + Expect(err).To(Succeed(), "Unable to deploy search head cluster") + + // Ensure that the cluster-master goes to Ready phase + testenv.ClusterMasterReady(deployment, testenvInstance) + + // Ensure indexers go to Ready phase + testenv.IndexersReady(deployment, testenvInstance, siteCount) + + // Ensure search head cluster go to Ready phase + testenv.SearchHeadClusterReady(deployment, testenvInstance) + + // Ensure cluster configured as multisite + testenv.IndexerClusterMultisiteStatus(deployment, testenvInstance, siteCount) + + // Wait for Monitoring Console Pod to be in READY status + testenv.MCPodReady(testenvInstance.GetName(), deployment) + + // Ingest data on Indexers + for i := 1; i <= siteCount; i++ { + siteName := fmt.Sprintf("site%d", i) + podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteName, 0) + logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) + testenv.CreateMockLogfile(logFile, 2000) + testenv.IngestFileViaMonitor(logFile, "main", podName, deployment) + } + + // Scale indexers + scaledIndexerReplicas := defaultIndexerReplicas + 1 + testenvInstance.Log.Info("Scaling up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) + idxcName := deployment.GetName() + "-" + "site1" + + // Get instance of current Indexer CR with latest config + idxc := &enterprisev1.IndexerCluster{} + err = deployment.GetInstance(idxcName, idxc) + Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + + // Update Replicas of Indexer Cluster + idxc.Spec.Replicas = int32(scaledIndexerReplicas) + err = deployment.UpdateCR(idxc) + Expect(err).To(Succeed(), "Failed to Scale Up Indexer Cluster") + + // Ensure Indxer cluster scales up and go to ScalingUp phase + testenv.VerifyIndexerClusterPhase(deployment, testenvInstance, splcommon.PhaseScalingUp, idxcName) + + // Ensure Indexer cluster go to Ready phase + testenv.IndexersReady(deployment, testenvInstance, siteCount) + + // Ingest data on new Indexers + podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), "site1", 1) + logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) + testenv.CreateMockLogfile(logFile, 2000) + testenv.IngestFileViaMonitor(logFile, "main", podName, deployment) + + // Verify New Indexer On Cluster Master + indexerName := podName + testenvInstance.Log.Info("Checking for Indexer On CM", "Indexer Name", indexerName) + Expect(testenv.CheckIndexerOnCM(deployment, indexerName)).To(Equal(true)) + + // Wait for Monitoring Console Pod to be in READY status + testenv.MCPodReady(testenvInstance.GetName(), deployment) + + // Wait for RF SF is Met + testenv.VerifyRFSFMet(deployment, testenvInstance) + + // Search for data on newly added indexer + searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) + searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) + searchResultsResp, err := testenv.PerformSearchSync(searchPod, searchString, deployment) + Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) + + // Verify result. + searchResponse := strings.Split(searchResultsResp, "\n")[0] + var searchResults map[string]interface{} + jsonErr := json.Unmarshal([]byte(searchResponse), &searchResults) + Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) + + testenvInstance.Log.Info("Search results :", "searchResults", searchResults["result"]) + Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) + + resultLine := searchResults["result"].(map[string]interface{}) + testenvInstance.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) + testHostname := strings.Compare(resultLine["host"].(string), indexerName) + Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) + + // Scale Down Indexer Cluster + testenvInstance.Log.Info("Scaling Down Indexer Cluster Site", "Current Replicas", scaledIndexerReplicas, "New Replicas", scaledIndexerReplicas-1) + scaledIndexerReplicas = scaledIndexerReplicas - 1 + + // Get instance of current Indexer CR with latest config + idxc = &enterprisev1.IndexerCluster{} + err = deployment.GetInstance(idxcName, idxc) + Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + + // Update Replicas of Indexer Cluster + idxc.Spec.Replicas = int32(scaledIndexerReplicas) + err = deployment.UpdateCR(idxc) + Expect(err).To(Succeed(), "Failed to scale down Indxer Cluster") + + // Ensure Indxer cluster scales Down and go to ScalingDown phase + testenv.VerifyIndexerClusterPhase(deployment, testenvInstance, splcommon.PhaseScalingDown, idxcName) + + // Ensure Indexer cluster go to Ready phase + testenv.IndexersReady(deployment, testenvInstance, siteCount) + + // Wait for RF SF is Met + testenv.VerifyRFSFMet(deployment, testenvInstance) + + // Search for data from removed indexer + searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) + searchResultsResp, err = testenv.PerformSearchSync(searchPod, searchString, deployment) + Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) + + // Verify result. + searchResponse = strings.Split(searchResultsResp, "\n")[0] + jsonErr = json.Unmarshal([]byte(searchResponse), &searchResults) + Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) + + testenvInstance.Log.Info("Search results :", "searchResults", searchResults["result"]) + Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) + + resultLine = searchResults["result"].(map[string]interface{}) + testenvInstance.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) + testHostname = strings.Compare(resultLine["host"].(string), indexerName) + Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) + + }) + }) +}) diff --git a/test/testenv/cmutil.go b/test/testenv/cmutil.go index 119025d02..687a11ec7 100644 --- a/test/testenv/cmutil.go +++ b/test/testenv/cmutil.go @@ -97,46 +97,83 @@ func CheckRFSF(deployment *Deployment) bool { return rfMet && sfMet } -// ClusterMasterSearchHeadResponse /services/cluster/master/searchhead response -type ClusterMasterSearchHeadResponse struct { - Entries []ClusterMasterSearchHeadEntry `json:"entry"` -} - -// ClusterMasterSearchHeadEntry represents a single search head -type ClusterMasterSearchHeadEntry struct { - Name string `json:"name"` - Content ClusterMasterSearchHeadContent `json:"content"` -} - -// ClusterMasterSearchHeadContent represents detailed information about a search head -type ClusterMasterSearchHeadContent struct { - EaiACL interface{} `json:"eai:acl"` - HostPortPair string `json:"host_port_pair"` - Label string `json:"label"` - Site string `json:"site"` - Status string `json:"status"` +// ClusterMasterPeersAndSearchHeadResponse /services/cluster/master/peers and /services/cluster/master/searchhead response +type ClusterMasterPeersAndSearchHeadResponse struct { + Entry []struct { + Content struct { + Label string `json:"label"` + RegisterSearchAddress string `json:"register_search_address"` + ReplicationCount int `json:"replication_count"` + ReplicationPort int `json:"replication_port"` + ReplicationUseSsl bool `json:"replication_use_ssl"` + RestartRequiredForApplyingDryRunBundle bool `json:"restart_required_for_applying_dry_run_bundle"` + Site string `json:"site"` + SplunkVersion string `json:"splunk_version"` + Status string `json:"status"` + SummaryReplicationCount int `json:"summary_replication_count"` + } `json:"content"` + } `json:"entry"` } -// CheckSearchHeadRemoved check if search head is removed from Indexer Cluster -func CheckSearchHeadRemoved(deployment *Deployment) bool { +// GetIndexersOrSearchHeadsOnCM get indexers or search head on Cluster Master +func GetIndexersOrSearchHeadsOnCM(deployment *Deployment, endpoint string) ClusterMasterPeersAndSearchHeadResponse { + url := "" + if endpoint == "sh" { + url = "https://localhost:8089/services/cluster/master/searchheads?output_mode=json" + } else { + url = "https://localhost:8089/services/cluster/master/peers?output_mode=json" + } //code to execute podName := fmt.Sprintf("splunk-%s-cluster-master-0", deployment.GetName()) - stdin := "curl -ks -u admin:$(cat /mnt/splunk-secrets/password) https://localhost:8089/services/cluster/master/searchheads?output_mode=json" + stdin := fmt.Sprintf("curl -ks -u admin:$(cat /mnt/splunk-secrets/password) %s", url) command := []string{"/bin/sh"} stdout, stderr, err := deployment.PodExecCommand(podName, command, stdin, false) + restResponse := ClusterMasterPeersAndSearchHeadResponse{} if err != nil { logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "command", command) - return false + return restResponse } logf.Log.Info("Command executed on pod", "pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) - restResponse := ClusterMasterSearchHeadResponse{} err = json.Unmarshal([]byte(stdout), &restResponse) if err != nil { - logf.Log.Error(err, "Failed to parse cluster searchheads") - return false + logf.Log.Error(err, "Failed to parse cluster peers") } + return restResponse +} + +// CheckIndexerOnCM check given Indexer on cluster master +func CheckIndexerOnCM(deployment *Deployment, indexerName string) bool { + restResponse := GetIndexersOrSearchHeadsOnCM(deployment, "peer") + found := false + for _, entry := range restResponse.Entry { + logf.Log.Info("Peer found On CM", "Indexer Name", entry.Content.Label, "Status", entry.Content.Status) + if entry.Content.Label == indexerName { + found = true + break + } + } + return found +} + +// CheckSearchHeadOnCM check given search head on cluster master +func CheckSearchHeadOnCM(deployment *Deployment, searchHeadName string) bool { + restResponse := GetIndexersOrSearchHeadsOnCM(deployment, "sh") + found := false + for _, entry := range restResponse.Entry { + logf.Log.Info("Search Head On CM", "Search Head", entry.Content.Label, "Status", entry.Content.Status) + if entry.Content.Label == searchHeadName { + found = true + break + } + } + return found +} + +// CheckSearchHeadRemoved check if search head is removed from Indexer Cluster +func CheckSearchHeadRemoved(deployment *Deployment) bool { + restResponse := GetIndexersOrSearchHeadsOnCM(deployment, "sh") searchHeadRemoved := true - for _, entry := range restResponse.Entries { + for _, entry := range restResponse.Entry { logf.Log.Info("Search Found", "Search Head", entry.Content.Label, "Status", entry.Content.Status) if entry.Content.Status == "Disconnected" { searchHeadRemoved = false diff --git a/test/testenv/search_utils.go b/test/testenv/search_utils.go index cebad00ec..76c48cbae 100644 --- a/test/testenv/search_utils.go +++ b/test/testenv/search_utils.go @@ -75,6 +75,8 @@ func PerformSearchSync(podName string, search string, deployment *Deployment) (s return "", err } + logf.Log.Info("Output of search Query", "Search", search, "Output", searchReqResp) + // Since results can have multiple formats depending on the search SPL, leave this response as a string return searchReqResp, err } @@ -98,6 +100,8 @@ func PerformSearchReq(podName string, search string, deployment *Deployment) (st return "", err } + logf.Log.Info("Output of search Query", "Search", search, "Output", stdout) + // Get SID var searchReqResult map[string]interface{} jsonErr := json.Unmarshal([]byte(stdout), &searchReqResult) diff --git a/test/testenv/verificationutils.go b/test/testenv/verificationutils.go index f0241b2de..b6852a8a7 100644 --- a/test/testenv/verificationutils.go +++ b/test/testenv/verificationutils.go @@ -332,22 +332,8 @@ func VerifyConfOnPod(deployment *Deployment, namespace string, podName string, c }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(true)) } -// VerifyStandaloneScalingUp verify give CR is scaling up -func VerifyStandaloneScalingUp(deployment *Deployment, testenvInstance *TestEnv) { - gomega.Eventually(func() splcommon.Phase { - standalone := &enterprisev1.Standalone{} - err := deployment.GetInstance(deployment.GetName(), standalone) - if err != nil { - return splcommon.PhaseError - } - testenvInstance.Log.Info("Waiting for standalone status to be Scaling Up", "instance", standalone.ObjectMeta.Name, "Phase", standalone.Status.Phase) - DumpGetPods(testenvInstance.GetName()) - return standalone.Status.Phase - }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(splcommon.PhaseScalingUp)) -} - -// VerifySearchHeadClusterScalingUp verify give CR is scaling up -func VerifySearchHeadClusterScalingUp(deployment *Deployment, testenvInstance *TestEnv) { +// VerifySearchHeadClusterPhase verify the phase of SHC matches given phase +func VerifySearchHeadClusterPhase(deployment *Deployment, testenvInstance *TestEnv, phase splcommon.Phase) { gomega.Eventually(func() splcommon.Phase { shc := &enterprisev1.SearchHeadCluster{} shcName := deployment.GetName() + "-shc" @@ -355,23 +341,36 @@ func VerifySearchHeadClusterScalingUp(deployment *Deployment, testenvInstance *T if err != nil { return splcommon.PhaseError } - testenvInstance.Log.Info("Waiting for Search Head Cluster CR status to be Scaling Up", "instance", shc.ObjectMeta.Name, "Phase", shc.Status.Phase) + testenvInstance.Log.Info("Waiting for Search Head Cluster Phase", "instance", shc.ObjectMeta.Name, "Expected", phase, "Phase", shc.Status.Phase) DumpGetPods(testenvInstance.GetName()) return shc.Status.Phase }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(splcommon.PhaseScalingUp)) } -// VerifyIndexerClusterScalingUp verify give CR is scaling up -func VerifyIndexerClusterScalingUp(deployment *Deployment, testenvInstance *TestEnv) { +// VerifyIndexerClusterPhase verify the phase of idxc matches the given phase +func VerifyIndexerClusterPhase(deployment *Deployment, testenvInstance *TestEnv, phase splcommon.Phase, idxcName string) { gomega.Eventually(func() splcommon.Phase { idxc := &enterprisev1.IndexerCluster{} - idxcName := deployment.GetName() + "-idxc" err := deployment.GetInstance(idxcName, idxc) if err != nil { return splcommon.PhaseError } - testenvInstance.Log.Info("Waiting for Indexer Cluster CR status to be Scaling Up", "instance", idxc.ObjectMeta.Name, "Phase", idxc.Status.Phase) + testenvInstance.Log.Info("Waiting for Indexer Cluster Phase", "instance", idxc.ObjectMeta.Name, "Expected", phase, "Phase", idxc.Status.Phase) DumpGetPods(testenvInstance.GetName()) return idxc.Status.Phase - }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(splcommon.PhaseScalingUp)) + }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(phase)) +} + +// VerifyStandalonePhase verify the phase of Standalone CR +func VerifyStandalonePhase(deployment *Deployment, testenvInstance *TestEnv, crName string, phase splcommon.Phase) { + gomega.Eventually(func() splcommon.Phase { + standalone := &enterprisev1.Standalone{} + err := deployment.GetInstance(deployment.GetName(), standalone) + if err != nil { + return splcommon.PhaseError + } + testenvInstance.Log.Info("Waiting for standalone status", "instance", standalone.ObjectMeta.Name, "Expected", phase, " Actual Phase", standalone.Status.Phase) + DumpGetPods(testenvInstance.GetName()) + return standalone.Status.Phase + }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(phase)) }