CSPL-827: Improve automation to include License Master on several deployments

.circleci/config.yml

@@ -282,7 +282,7 @@ jobs:
             mkdir -p /tmp/test-results
             find ./test -name "*junit.xml" -exec cp {} /tmp/test-results \;
           environment: 
-            TEST_FOCUS: "smoke|ingest_search|monitoring_console|deletecr|smartstore"
+            TEST_FOCUS: "smoke|ingest_search|monitoring_console|deletecr|smartstore|licensemaster"
       - store_test_results:
           name: Save test results
           path: /tmp/test-results

test/licensemaster/lm_suite_test.go

@@ -0,0 +1,60 @@
+// Copyright (c) 2018-2021 Splunk Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package licensemaster
+
+import (
+	"testing"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	"github.com/onsi/ginkgo/reporters"
+	. "github.com/onsi/gomega"
+
+	"github.com/splunk/splunk-operator/test/testenv"
+)
+
+const (
+	// PollInterval specifies the polling interval
+	PollInterval = 5 * time.Second
+
+	// ConsistentPollInterval is the interval to use to consistently check a state is stable
+	ConsistentPollInterval = 200 * time.Millisecond
+	ConsistentDuration     = 2000 * time.Millisecond
+)
+
+var (
+	testenvInstance *testenv.TestEnv
+	testSuiteName   = "licensemaster-" + testenv.RandomDNSName(2)
+)
+
+// TestBasic is the main entry point
+func TestBasic(t *testing.T) {
+
+	RegisterFailHandler(Fail)
+
+	junitReporter := reporters.NewJUnitReporter(testSuiteName + "_junit.xml")
+	RunSpecsWithDefaultAndCustomReporters(t, "Running "+testSuiteName, []Reporter{junitReporter})
+}
+
+var _ = BeforeSuite(func() {
+	var err error
+	testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName)
+	Expect(err).ToNot(HaveOccurred())
+})
+
+var _ = AfterSuite(func() {
+	if testenvInstance != nil {
+		Expect(testenvInstance.Teardown()).ToNot(HaveOccurred())
+	}
+})

test/licensemaster/lm_test.go

@@ -0,0 +1,244 @@
+// Copyright (c) 2018-2021 Splunk Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package licensemaster
+
+import (
+	"fmt"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/splunk/splunk-operator/test/testenv"
+)
+
+var _ = Describe("Licensemaster test", func() {
+
+	var deployment *testenv.Deployment
+
+	BeforeEach(func() {
+		var err error
+		deployment, err = testenvInstance.NewDeployment(testenv.RandomDNSName(3))
+		Expect(err).To(Succeed(), "Unable to create deployment")
+	})
+
+	AfterEach(func() {
+		// When a test spec failed, skip the teardown so we can troubleshoot.
+		if CurrentGinkgoTestDescription().Failed {
+			testenvInstance.SkipTeardown = true
+		}
+		if deployment != nil {
+			deployment.Teardown()
+		}
+	})
+
+	Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() {
+		It("licensemaster: can deploy indexers and search head cluster", func() {
+
+			// Download License File
+			licenseFilePath, err := testenv.DownloadFromS3Bucket()
+			Expect(err).To(Succeed(), "Unable to download license file")
+
+			// Create License Config Map
+			testenvInstance.CreateLicenseConfigMap(licenseFilePath)
+
+			err = deployment.DeploySingleSiteCluster(deployment.GetName(), 3, true /*shc*/)
+			Expect(err).To(Succeed(), "Unable to deploy cluster")
+
+			// Ensure that the cluster-master goes to Ready phase
+			testenv.ClusterMasterReady(deployment, testenvInstance)
+
+			// Ensure indexers go to Ready phase
+			testenv.SingleSiteIndexersReady(deployment, testenvInstance)
+
+			// Ensure search head cluster go to Ready phase
+			testenv.SearchHeadClusterReady(deployment, testenvInstance)
+
+			// Verify MC Pod is Ready
+			testenv.MCPodReady(testenvInstance.GetName(), deployment)
+
+			// Verify RF SF is met
+			testenv.VerifyRFSFMet(deployment, testenvInstance)
+
+			// Verify LM is configured on indexers
+			indexerPodName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), 0)
+			testenv.VerifyLMConfiguredOnPod(deployment, indexerPodName)
+			indexerPodName = fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), 1)
+			testenv.VerifyLMConfiguredOnPod(deployment, indexerPodName)
+			indexerPodName = fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), 2)
+			testenv.VerifyLMConfiguredOnPod(deployment, indexerPodName)
+
+			// Verify LM is configured on SHs
+			searchHeadPodName := fmt.Sprintf(testenv.SearchHeadSHCPod, deployment.GetName(), 0)
+			testenv.VerifyLMConfiguredOnPod(deployment, searchHeadPodName)
+			searchHeadPodName = fmt.Sprintf(testenv.SearchHeadSHCPod, deployment.GetName(), 1)
+			testenv.VerifyLMConfiguredOnPod(deployment, searchHeadPodName)
+			searchHeadPodName = fmt.Sprintf(testenv.SearchHeadSHCPod, deployment.GetName(), 2)
+			testenv.VerifyLMConfiguredOnPod(deployment, searchHeadPodName)
+		})
+	})
+
+	Context("Multisite cluster deployment (M13 - Multisite indexer cluster, Search head cluster)", func() {
+		It("licensemaster: can deploy indexers and search head cluster", func() {
+
+			// Download License File
+			licenseFilePath, err := testenv.DownloadFromS3Bucket()
+			Expect(err).To(Succeed(), "Unable to download license file")
+
+			// Create License Config Map
+			testenvInstance.CreateLicenseConfigMap(licenseFilePath)
+
+			siteCount := 3
+			err = deployment.DeployMultisiteClusterWithSearchHead(deployment.GetName(), 1, siteCount)
+			Expect(err).To(Succeed(), "Unable to deploy cluster")
+
+			// Ensure that the cluster-master goes to Ready phase
+			testenv.ClusterMasterReady(deployment, testenvInstance)
+
+			// Ensure the indexers of all sites go to Ready phase
+			testenv.IndexersReady(deployment, testenvInstance, siteCount)
+
+			// Ensure cluster configured as multisite
+			testenv.IndexerClusterMultisiteStatus(deployment, testenvInstance, siteCount)
+
+			// Ensure search head cluster go to Ready phase
+			testenv.SearchHeadClusterReady(deployment, testenvInstance)
+
+			// Verify MC Pod is Ready
+			testenv.MCPodReady(testenvInstance.GetName(), deployment)
+
+			// Verify RF SF is met
+			testenv.VerifyRFSFMet(deployment, testenvInstance)
+
+			// Verify LM is configured on indexers
+			indexerPodName := fmt.Sprintf(testenv.IndexerMultisitePod, deployment.GetName(), 1, 0)
+			testenv.VerifyLMConfiguredOnPod(deployment, indexerPodName)
+			indexerPodName = fmt.Sprintf(testenv.IndexerMultisitePod, deployment.GetName(), 2, 0)
+			testenv.VerifyLMConfiguredOnPod(deployment, indexerPodName)
+			indexerPodName = fmt.Sprintf(testenv.IndexerMultisitePod, deployment.GetName(), 3, 0)
+			testenv.VerifyLMConfiguredOnPod(deployment, indexerPodName)
+
+			// Verify LM is configured on SHs
+			searchHeadPodName := fmt.Sprintf(testenv.SearchHeadSHCPod, deployment.GetName(), 0)
+			testenv.VerifyLMConfiguredOnPod(deployment, searchHeadPodName)
+			searchHeadPodName = fmt.Sprintf(testenv.SearchHeadSHCPod, deployment.GetName(), 1)
+			testenv.VerifyLMConfiguredOnPod(deployment, searchHeadPodName)
+			searchHeadPodName = fmt.Sprintf(testenv.SearchHeadSHCPod, deployment.GetName(), 2)
+			testenv.VerifyLMConfiguredOnPod(deployment, searchHeadPodName)
+		})
+	})
+
+	Context("Multisite cluster deployment (M1 - multisite indexer cluster)", func() {
+		It("licensemaster: can deploy multisite indexers cluster", func() {
+
+			// Download License File
+			licenseFilePath, err := testenv.DownloadFromS3Bucket()
+			Expect(err).To(Succeed(), "Unable to download license file")
+
+			// Create License Config Map
+			testenvInstance.CreateLicenseConfigMap(licenseFilePath)
+
+			siteCount := 3
+			err = deployment.DeployMultisiteCluster(deployment.GetName(), 1, siteCount)
+			Expect(err).To(Succeed(), "Unable to deploy cluster")
+
+			// Ensure that the cluster-master goes to Ready phase
+			testenv.ClusterMasterReady(deployment, testenvInstance)
+
+			// Ensure the indexers of all sites go to Ready phase
+			testenv.IndexersReady(deployment, testenvInstance, siteCount)
+
+			// Ensure cluster configured as multisite
+			testenv.IndexerClusterMultisiteStatus(deployment, testenvInstance, siteCount)
+
+			// Verify MC Pod is Ready
+			testenv.MCPodReady(testenvInstance.GetName(), deployment)
+
+			// Verify RF SF is met
+			testenv.VerifyRFSFMet(deployment, testenvInstance)
+
+			// Verify LM is configured on indexers
+			indexerPodName := fmt.Sprintf(testenv.IndexerMultisitePod, deployment.GetName(), 1, 0)
+			testenv.VerifyLMConfiguredOnPod(deployment, indexerPodName)
+			indexerPodName = fmt.Sprintf(testenv.IndexerMultisitePod, deployment.GetName(), 2, 0)
+			testenv.VerifyLMConfiguredOnPod(deployment, indexerPodName)
+			indexerPodName = fmt.Sprintf(testenv.IndexerMultisitePod, deployment.GetName(), 3, 0)
+			testenv.VerifyLMConfiguredOnPod(deployment, indexerPodName)
+		})
+	})
+
+	Context("Standalone deployment (S1) with LM", func() {
+		It("licensemaster: can deploy a standalone instance and a License Master", func() {
+
+			// Download License File
+			licenseFilePath, err := testenv.DownloadFromS3Bucket()
+			Expect(err).To(Succeed(), "Unable to download license file")
+
+			// Create License Config Map
+			testenvInstance.CreateLicenseConfigMap(licenseFilePath)
+
+			// Create standalone Deployment with License Master
+			standalone, err := deployment.DeployStandaloneWithLM(deployment.GetName())
+			Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM")
+
+			// Wait for License Master to be in READY status
+			testenv.LicenseMasterReady(deployment, testenvInstance)
+
+			// Wait for Standalone to be in READY status
+			testenv.StandaloneReady(deployment, deployment.GetName(), standalone, testenvInstance)
+
+			// Verify MC Pod is Ready
+			testenv.MCPodReady(testenvInstance.GetName(), deployment)
+
+			// Verify LM is configured on standalone instance
+			standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)
+			testenv.VerifyLMConfiguredOnPod(deployment, standalonePodName)
+		})
+	})
+
+	Context("Clustered deployment (C3 - clustered indexer) with LM", func() {
+		It("licensemaster: can deploy indexer cluster with LM", func() {
+
+			// Download License File
+			licenseFilePath, err := testenv.DownloadFromS3Bucket()
+			Expect(err).To(Succeed(), "Unable to download license file")
+
+			// Create License Config Map
+			testenvInstance.CreateLicenseConfigMap(licenseFilePath)
+
+			// Create Cluster Master with LicenseMasterRef, IndexerCluster without LicenseMasterRef
+			err = deployment.DeploySingleSiteCluster(deployment.GetName(), 3, true /*shc*/)
+			Expect(err).To(Succeed(), "Unable to deploy cluster")
+
+			// Ensure that the cluster-master goes to Ready phase
+			testenv.ClusterMasterReady(deployment, testenvInstance)
+
+			// Ensure indexers go to Ready phase
+			testenv.SingleSiteIndexersReady(deployment, testenvInstance)
+
+			// Verify MC Pod is Ready
+			testenv.MCPodReady(testenvInstance.GetName(), deployment)
+
+			// Verify RF SF is met
+			testenv.VerifyRFSFMet(deployment, testenvInstance)
+
+			// Verify LM is configured on indexers
+			indexerPodName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), 0)
+			testenv.VerifyLMConfiguredOnPod(deployment, indexerPodName)
+			indexerPodName = fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), 1)
+			testenv.VerifyLMConfiguredOnPod(deployment, indexerPodName)
+			indexerPodName = fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), 2)
+			testenv.VerifyLMConfiguredOnPod(deployment, indexerPodName)
+		})
+	})
+})

test/smoke/smoke_test.go

@@ -143,38 +143,6 @@ var _ = Describe("Smoke test", func() {
 		})
 	})
 
-	Context("Standalone deployment (S1) with LM", func() {
-		It("smoke: can deploy a standalone instance and a License Master", func() {
-			// Download License File
-			licenseFilePath, err := testenv.DownloadFromS3Bucket()
-			Expect(err).To(Succeed(), "Unable to downlaod license file")
-
-			// Create License Config Map
-			testenvInstance.CreateLicenseConfigMap(licenseFilePath)
-
-			// Create standalone Deployment with License Master
-			standalone, err := deployment.DeployStandaloneWithLM(deployment.GetName())
-			Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM")
-
-			// Wait for License Master to be in READY status
-			testenv.LicenseMasterReady(deployment, testenvInstance)
-
-			// Wait for Standalone to be in READY status
-			testenv.StandaloneReady(deployment, deployment.GetName(), standalone, testenvInstance)
-
-			// Verify MC Pod is Ready
-			testenv.MCPodReady(testenvInstance.GetName(), deployment)
-
-			// Verify LM is configured on standalone instance
-			standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)
-			testenv.VerifyLMConfiguredOnPod(deployment, standalonePodName)
-
-			// Verify LM is configured on MC Pod
-			mcPodName := fmt.Sprintf(testenv.MonitoringConsolePod, testenvInstance.GetName(), 0)
-			testenv.VerifyLMConfiguredOnPod(deployment, mcPodName)
-		})
-	})
-
 	Context("Standalone deployment (S1) with Service Account", func() {
 		It("smoke: can deploy a standalone instance attached to a service account", func() {
 			// Create Service Account
@@ -206,36 +174,4 @@ var _ = Describe("Smoke test", func() {
 			testenv.VerifyServiceAccountConfiguredOnPod(deployment, testenvInstance.GetName(), standalonePodName, serviceAccountName)
 		})
 	})
-
-	Context("Clustered deployment (C3 - clustered indexer) with LM", func() {
-		It("smoke: can deploy indexer cluster with LM", func() {
-			// Download License File
-			licenseFilePath, err := testenv.DownloadFromS3Bucket()
-			Expect(err).To(Succeed(), "Unable to downlaod license file")
-
-			// Create License Config Map
-			testenvInstance.CreateLicenseConfigMap(licenseFilePath)
-
-			// Create Cluster Master with LicenseMasterRef, IndexerCluster without LicenseMasterRef
-			err = deployment.DeploySingleSiteCluster(deployment.GetName(), 3, true /*shc*/)
-			Expect(err).To(Succeed(), "Unable to deploy cluster")
-
-			// Ensure that the cluster-master goes to Ready phase
-			testenv.ClusterMasterReady(deployment, testenvInstance)
-
-			// Ensure indexers go to Ready phase
-			testenv.SingleSiteIndexersReady(deployment, testenvInstance)
-
-			// Verify MC Pod is Ready
-			testenv.MCPodReady(testenvInstance.GetName(), deployment)
-
-			// Verify RF SF is met
-			testenv.VerifyRFSFMet(deployment, testenvInstance)
-
-			// Verify LM is configured on indexers
-			indexerPodName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), 0)
-			testenv.VerifyLMConfiguredOnPod(deployment, indexerPodName)
-
-		})
-	})
 })