feat: use samples in the e2e (helm)

hack/generate/samples/generate_testdata.go

@@ -21,9 +21,9 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
-	"github.com/operator-framework/operator-sdk/hack/generate/samples/internal/ansible"
-	gosamples "github.com/operator-framework/operator-sdk/hack/generate/samples/internal/go"
-	"github.com/operator-framework/operator-sdk/hack/generate/samples/internal/helm"
+	"github.com/operator-framework/operator-sdk/internal/samples/ansible"
+	gosamples "github.com/operator-framework/operator-sdk/internal/samples/go"
+	"github.com/operator-framework/operator-sdk/internal/samples/helm"
 	"github.com/operator-framework/operator-sdk/internal/testutils"
 )
 

hack/generate/samples/molecule/generate.go

@@ -22,7 +22,7 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
-	"github.com/operator-framework/operator-sdk/hack/generate/samples/internal/ansible"
+	"github.com/operator-framework/operator-sdk/internal/samples/ansible"
 	"github.com/operator-framework/operator-sdk/internal/testutils"
 )
 

hack/generate/samples/internal/ansible/memcached.go → internal/samples/ansible/memcached.go

@@ -22,7 +22,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	kbtestutils "sigs.k8s.io/kubebuilder/test/e2e/utils"
 
-	"github.com/operator-framework/operator-sdk/hack/generate/samples/internal/pkg"
+	"github.com/operator-framework/operator-sdk/internal/samples/pkg"
 	"github.com/operator-framework/operator-sdk/internal/testutils"
 )
 

hack/generate/samples/internal/ansible/molecule.go → internal/samples/ansible/molecule.go

@@ -22,7 +22,7 @@ import (
 
 	kbtestutils "sigs.k8s.io/kubebuilder/test/e2e/utils"
 
-	"github.com/operator-framework/operator-sdk/hack/generate/samples/internal/pkg"
+	"github.com/operator-framework/operator-sdk/internal/samples/pkg"
 	"github.com/operator-framework/operator-sdk/internal/testutils"
 	log "github.com/sirupsen/logrus"
 )

hack/generate/samples/internal/go/memcached_with_webhooks.go → internal/samples/go/memcached_with_webhooks.go

@@ -23,7 +23,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	kbtestutils "sigs.k8s.io/kubebuilder/test/e2e/utils"
 
-	"github.com/operator-framework/operator-sdk/hack/generate/samples/internal/pkg"
+	"github.com/operator-framework/operator-sdk/internal/samples/pkg"
 	"github.com/operator-framework/operator-sdk/internal/testutils"
 )
 
@@ -436,7 +436,7 @@ func (r *Memcached) ValidateDelete() error {
 }
 func validateOdd(n int32) error {
 	if n%2 == 0 {
-		return errors.New("Cluster size must be an odd number")
+		return errors.New("cluster size must be an odd number")
 	}
 	return nil
 }

hack/generate/samples/internal/helm/memcached.go → internal/samples/helm/memcached.go

@@ -21,10 +21,12 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
-	"github.com/operator-framework/operator-sdk/hack/generate/samples/internal/pkg"
+	"github.com/operator-framework/operator-sdk/internal/samples/pkg"
 	"github.com/operator-framework/operator-sdk/internal/testutils"
 )
 
+const chartPath = "internal/samples/helm/testdata/memcached-0.0.1.tgz"
+
 // MemcachedHelm defines the Memcached Sample in Helm
 type MemcachedHelm struct {
 	ctx *pkg.SampleContext
@@ -70,7 +72,11 @@ func (mh *MemcachedHelm) Run() {
 	// required to make Helm project generation idempotent because contributors and CI environments
 	// can all have slightly different environments that can affect the content of the generated
 	// role and cause sanity testing to fail.
-	os.Setenv("KUBECONFIG", "broken_so_we_generate_static_default_rules")
+	if !mh.isRunningEe2() {
+		// Set the env var only when it is running to gen the sample
+		// For the e2e test the following should not be set
+		os.Setenv("KUBECONFIG", "broken_so_we_generate_static_default_rules")
+	}
 
 	log.Infof("creating the project")
 	err = mh.ctx.Init(
@@ -79,9 +85,12 @@ func (mh *MemcachedHelm) Run() {
 	pkg.CheckError("creating the project", err)
 
 	log.Infof("handling work path to get helm chart mock data")
-	projectPath := strings.Split(current, "operator-sdk/")[0]
-	projectPath = strings.Replace(projectPath, "operator-sdk", "", 1)
-	helmChartPath := filepath.Join(projectPath, "operator-sdk/hack/generate/samples/internal/helm/testdata/memcached-0.0.1.tgz")
+	helmChartPath := filepath.Join(current, chartPath)
+	if mh.isRunningEe2() {
+		// the current path for the e2e test is not the same to gen the samples
+		helmChartPath = filepath.Join(strings.Split(current, "operator-sdk/")[0],
+			"internal/samples/helm/testdata/memcached-0.0.1.tgz")
+	}
 	log.Infof("using the helm chart in: (%v)", helmChartPath)
 
 	err = mh.ctx.CreateAPI(
@@ -106,6 +115,11 @@ func (mh *MemcachedHelm) Run() {
 	pkg.RunOlmIntegration(mh.ctx)
 }
 
+// isRunningEe2 return true when context dir iss
+func (mh *MemcachedHelm) isRunningEe2() bool {
+	return strings.Contains(mh.ctx.Dir, "e2e-helm")
+}
+
 // GenerateMemcachedHelmSample will call all actions to create the directory and generate the sample
 // The Context to run the samples are not the same in the e2e test. In this way, note that it should NOT
 // be called in the e2e tests since it will call the Prepare() to set the sample context and generate the files
@@ -121,7 +135,7 @@ func GenerateMemcachedHelmSample(samplesPath string) {
 
 const policyRolesFragment = `
 ##
-## Rules customized for cache.example.com/v1alpha1, Kind: Memcached
+## Rules customized
 ##
 - apiGroups:
   - policy

hack/generate/samples/internal/pkg/context.go → internal/samples/pkg/context.go

@@ -44,3 +44,10 @@ func NewSampleContext(binary string, path string, env ...string) (s SampleContex
 
 	return s, err
 }
+
+// NewSampleContextWithTestContext returns a SampleContext containing the kubebuilder TestContext informed
+// It is useful to allow the samples code be re-used in the e2e tests.
+func NewSampleContextWithTestContext(tc *testutils.TestContext) (s SampleContext, err error) {
+	s.TestContext = *tc
+	return s, err
+}

hack/generate/samples/internal/pkg/utils.go → internal/samples/pkg/utils.go

@@ -23,14 +23,14 @@ import (
 // CheckError will check the error and exit with 1 when as errors
 func CheckError(msg string, err error) {
 	if err != nil {
-		log.Errorf("error %s: %s", msg, err)
+		log.Errorf("Error %s: %s", msg, err)
 		os.Exit(1)
 	}
 }
 
 // RunOlmIntegration runs all commands to integrate the project with OLM
 func RunOlmIntegration(ctx *SampleContext) {
-	log.Infof("integrating project with OLM")
+	log.Infof("Integrating project with OLM")
 	err := ctx.DisableOLMBundleInteractiveMode()
 	CheckError("disabling the OLM bundle", err)
 

test/e2e-helm/e2e_helm_cluster_test.go

@@ -33,11 +33,6 @@ var _ = Describe("Running Helm projects", func() {
 
 	Context("built with operator-sdk", func() {
 		BeforeEach(func() {
-			By("enabling Prometheus via the kustomization.yaml")
-			Expect(kbtestutils.UncommentCode(
-				filepath.Join(tc.Dir, "config", "default", "kustomization.yaml"),
-				"#- ../prometheus", "#")).To(Succeed())
-
 			By("deploying project on the cluster")
 			err := tc.Make("deploy", "IMG="+tc.ImageName)
 			Expect(err).NotTo(HaveOccurred())
@@ -141,14 +136,15 @@ var _ = Describe("Running Helm projects", func() {
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(releaseName)).NotTo(BeIdenticalTo(0))
 
-			By("checking the release(CR) deployment status")
+			statefulsetName := fmt.Sprintf("%s-%s", releaseName, "memcached")
+			By("checking the release(CR) statefulset status")
 			verifyReleaseUp := func() string {
 				output, err := tc.Kubectl.Command(
-					"rollout", "status", "deployment", releaseName)
+					"rollout", "status", "statefulset", statefulsetName)
 				Expect(err).NotTo(HaveOccurred())
 				return output
 			}
-			Eventually(verifyReleaseUp, time.Minute, time.Second).Should(ContainSubstring("successfully rolled out"))
+			Eventually(verifyReleaseUp, time.Minute, time.Second).Should(ContainSubstring("statefulset rolling update complete 3 pods"))
 
 			By("ensuring the created Service for the release(CR)")
 			crServiceName, err := tc.Kubectl.Get(
@@ -158,26 +154,26 @@ var _ = Describe("Running Helm projects", func() {
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(crServiceName)).NotTo(BeIdenticalTo(0))
 
-			By("scaling deployment replicas to 2")
+			By("scaling statefulset replicas to 2")
 			_, err = tc.Kubectl.Command(
-				"scale", "deployment", releaseName, "--replicas", "2")
+				"scale", "statefulset", statefulsetName, "--replicas", "2")
 			Expect(err).NotTo(HaveOccurred())
 
-			By("verifying the deployment automatically scales back down to 1")
+			By("verifying the statefulset automatically scales back to 3")
 			verifyRelease := func() error {
 				replicas, err := tc.Kubectl.Get(
 					false,
-					"deployment", releaseName, "-o", "jsonpath={..spec.replicas}")
+					"statefulset", statefulsetName, "-o", "jsonpath={..spec.replicas}")
 				Expect(err).NotTo(HaveOccurred())
-				if replicas != "1" {
-					return fmt.Errorf("release(CR) deployment with %s replicas", replicas)
+				if replicas != "3" {
+					return fmt.Errorf("release(CR) statefulset with %s replicas", replicas)
 				}
 				return nil
 			}
-			Eventually(verifyRelease, time.Minute, time.Second).Should(Succeed())
+			Eventually(verifyRelease, 4*time.Minute, 3*time.Second).Should(Succeed())
 
 			By("updating replicaCount to 2 in the CR manifest")
-			err = testutils.ReplaceInFile(filepath.Join(tc.Dir, sampleFile), "replicaCount: 1", "replicaCount: 2")
+			err = testutils.ReplaceInFile(filepath.Join(tc.Dir, sampleFile), "replicaCount: 3", "replicaCount: 2")
 			Expect(err).NotTo(HaveOccurred())
 
 			By("applying CR manifest with replicaCount: 2")
@@ -193,14 +189,14 @@ var _ = Describe("Running Helm projects", func() {
 			Eventually(managerContainerLogsAfterUpdateCR, time.Minute, time.Second).Should(
 				ContainSubstring("Upgraded release"))
 
-			By("checking Deployment replicas spec is equals 2")
+			By("checking statefulset replicas spec is equals 2")
 			verifyReleaseUpgrade := func() error {
 				replicas, err := tc.Kubectl.Get(
 					false,
-					"deployment", releaseName, "-o", "jsonpath={..spec.replicas}")
+					"statefulset", statefulsetName, "-o", "jsonpath={..spec.replicas}")
 				Expect(err).NotTo(HaveOccurred())
 				if replicas != "2" {
-					return fmt.Errorf("release(CR) deployment with %s replicas", replicas)
+					return fmt.Errorf("release(CR) statefulset with %s replicas", replicas)
 				}
 				return nil
 			}

test/e2e-helm/e2e_helm_suite_test.go

@@ -21,6 +21,8 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
+	"github.com/operator-framework/operator-sdk/internal/samples/helm"
+	"github.com/operator-framework/operator-sdk/internal/samples/pkg"
 	"github.com/operator-framework/operator-sdk/internal/testutils"
 )
 
@@ -55,33 +57,16 @@ var _ = BeforeSuite(func() {
 	By("preparing the prerequisites on cluster")
 	tc.InstallPrerequisites()
 
-	By("initializing a Helm project")
-	err = tc.Init(
-		"--plugins", "helm",
-		"--project-version", "3-alpha",
-		"--domain", tc.Domain)
-	Expect(err).NotTo(HaveOccurred())
-
-	By("using dev image for scorecard-test")
-	err = tc.ReplaceScorecardImagesForDev()
-	Expect(err).NotTo(HaveOccurred())
-
-	By("creating an API definition")
-	err = tc.CreateAPI(
-		"--group", tc.Group,
-		"--version", tc.Version,
-		"--kind", tc.Kind)
-	Expect(err).NotTo(HaveOccurred())
+	By("mocking Helm Memcached Sample")
+	ctx, err := pkg.NewSampleContextWithTestContext(&tc)
+	Expect(err).Should(Succeed())
+	sample := helm.NewMemcachedHelm(&ctx)
+	sample.Run()
 
 	By("replacing project Dockerfile to use Helm base image with the dev tag")
 	err = testutils.ReplaceRegexInFile(filepath.Join(tc.Dir, "Dockerfile"), "quay.io/operator-framework/helm-operator:.*", "quay.io/operator-framework/helm-operator:dev")
 	Expect(err).Should(Succeed())
 
-	By("turning off interactive prompts for all generation tasks.")
-	replace := "operator-sdk generate kustomize manifests"
-	err = testutils.ReplaceInFile(filepath.Join(tc.Dir, "Makefile"), replace, replace+" --interactive=false")
-	Expect(err).Should(Succeed())
-
 	By("checking the kustomize setup")
 	err = tc.Make("kustomize")
 	Expect(err).NotTo(HaveOccurred())
@@ -96,6 +81,10 @@ var _ = BeforeSuite(func() {
 		Expect(tc.LoadImageToKindClusterWithName("quay.io/operator-framework/scorecard-test:dev")).To(Succeed())
 	}
 
+	By("using dev image for scorecard-test")
+	err = tc.ReplaceScorecardImagesForDev()
+	Expect(err).NotTo(HaveOccurred())
+
 	By("generating the operator bundle")
 	err = tc.Make("bundle", "IMG="+tc.ImageName)
 	Expect(err).NotTo(HaveOccurred())

testdata/go/memcached-operator/api/v1alpha1/memcached_webhook.go

@@ -72,7 +72,7 @@ func (r *Memcached) ValidateDelete() error {
 }
 func validateOdd(n int32) error {
 	if n%2 == 0 {
-		return errors.New("Cluster size must be an odd number")
+		return errors.New("cluster size must be an odd number")
 	}
 	return nil
 }

testdata/helm/memcached-operator/config/rbac/role.yaml

@@ -82,7 +82,7 @@ rules:
 
 
 ##
-## Rules customized for cache.example.com/v1alpha1, Kind: Memcached
+## Rules customized
 ##
 - apiGroups:
   - policy