cleanup go e2e tests and perform improvements (#3499)

**Description of the change:**
- Update the go e2e test to have the suffix `_test` which has [special meaning for the Go compiler](golang/go#25223).
- Cleanup the test in order to make all following the same standard
- Ensure that the test are done using the makefile targets scaffolds as we suggest in our docs. 
- Add tests to ensure the metrics, run the project locally and the bundle sdk features
- fix the target in the doc https://master.sdk.operatorframework.io/docs/olm-integration/generation/#package-manifests-format. Users should use `$(KUSTOMIZE)` instead of kustomize directly.
- Fix Prometheus typo issue 

**Motivation for the change:**

- maintainability, readability and reusability
- ensure that our e2e tests are using the targets commands as we suggested to the users in the docs

NOTE: The first goal is to keep all tests following the same standard. However, it still has a hall for improvements that ought to make horizontally in follow-ups.

hack/tests/e2e-go.sh

@@ -8,14 +8,11 @@ source ./hack/lib/test_lib.sh
 source ./hack/lib/image_lib.sh
 
 test_dir=./test
-tests=$test_dir/e2e
+tests=$test_dir/e2e-go
 
 export TRACE=1
 export GO111MODULE=on
 
 setup_envs $tmp_sdk_root
 
-docker pull gcr.io/kubebuilder/kube-rbac-proxy:v0.5.0
-load_image_if_kind gcr.io/kubebuilder/kube-rbac-proxy:v0.5.0
-
-go test -v $tests
+go test $tests -v -ginkgo.v

test/e2e-ansible/e2e_ansible_suite_test.go

@@ -38,8 +38,8 @@ func TestE2EAnsible(t *testing.T) {
 
 var (
 	tc testutils.TestContext
-	// isPromethuesManagedBySuite is true when the suite tests is installing/uninstalling the Prometheus
-	isPromethuesManagedBySuite = true
+	// isPrometheusManagedBySuite is true when the suite tests is installing/uninstalling the Prometheus
+	isPrometheusManagedBySuite = true
 	// isOLMManagedBySuite is true when the suite tests is installing/uninstalling the OLM
 	isOLMManagedBySuite = true
 	// kubectx stores the k8s context from where the tests are running
@@ -66,13 +66,13 @@ var _ = BeforeSuite(func(done Done) {
 	output, err := tc.Kubectl.Command("api-resources")
 	Expect(err).NotTo(HaveOccurred())
 	if strings.Contains(output, "servicemonitors") {
-		isPromethuesManagedBySuite = false
+		isPrometheusManagedBySuite = false
 	}
 	if strings.Contains(output, "clusterserviceversions") {
 		isOLMManagedBySuite = false
 	}
 
-	if isPromethuesManagedBySuite {
+	if isPrometheusManagedBySuite {
 		By("installing Prometheus")
 		Expect(tc.InstallPrometheusOperManager()).To(Succeed())
 
@@ -176,7 +176,7 @@ var _ = BeforeSuite(func(done Done) {
 // AfterSuite run after all the specs have run, regardless of whether any tests have failed to ensures that
 // all be cleaned up
 var _ = AfterSuite(func() {
-	if isPromethuesManagedBySuite {
+	if isPrometheusManagedBySuite {
 		By("uninstalling Prometheus")
 		tc.UninstallPrometheusOperManager()
 	}

test/e2e-go/e2e_go_cluster_test.go

@@ -0,0 +1,191 @@
+// Copyright 2020 The Operator-SDK Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Modified from https://github.com/kubernetes-sigs/kubebuilder/tree/39224f0/test/e2e/v3
+
+package e2e_go_test
+
+import (
+	"encoding/base64"
+	"fmt"
+	"path/filepath"
+	"strings"
+	"time"
+
+	. "github.com/onsi/ginkgo" //nolint:golint
+	. "github.com/onsi/gomega" //nolint:golint
+	kbtestutils "sigs.k8s.io/kubebuilder/test/e2e/utils"
+)
+
+var _ = Describe("operator-sdk", func() {
+	var controllerPodName string
+
+	Context("built with operator-sdk", func() {
+		BeforeEach(func() {
+			By("enabling Prometheus via the kustomization.yaml")
+			Expect(kbtestutils.UncommentCode(
+				filepath.Join(tc.Dir, "config", "default", "kustomization.yaml"),
+				"#- ../prometheus", "#")).To(Succeed())
+
+			By("deploying project on the cluster")
+			err := tc.Make("deploy", "IMG="+tc.ImageName)
+			Expect(err).Should(Succeed())
+		})
+		AfterEach(func() {
+			By("cleaning up the operator and resources")
+			defaultOutput, err := tc.KustomizeBuild(filepath.Join("config", "default"))
+			Expect(err).NotTo(HaveOccurred())
+			_, err = tc.Kubectl.WithInput(string(defaultOutput)).Command("delete", "-f", "-")
+			Expect(err).NotTo(HaveOccurred())
+
+			By("deleting Curl Pod created")
+			_, _ = tc.Kubectl.Delete(true, "pod", "curl")
+
+			By("cleaning up permissions")
+			_, _ = tc.Kubectl.Command("delete", "clusterrolebinding",
+				fmt.Sprintf("metrics-%s", tc.TestSuffix))
+
+			By("undeploy project")
+			_ = tc.Make("undeploy")
+
+			By("ensuring that the namespace was deleted")
+			verifyNamespaceDeleted := func() error {
+				_, err := tc.Kubectl.Command("get", "namespace", tc.Kubectl.Namespace)
+				if strings.Contains(err.Error(), "(NotFound): namespaces") {
+					return err
+				}
+				return nil
+			}
+			Eventually(verifyNamespaceDeleted, 2*time.Minute, time.Second).ShouldNot(Succeed())
+		})
+
+		It("should run correctly in a cluster", func() {
+			By("checking if the Operator project Pod is running")
+			verifyControllerUp := func() error {
+				By("getting the controller-manager pod name")
+				podOutput, err := tc.Kubectl.Get(
+					true,
+					"pods", "-l", "control-plane=controller-manager",
+					"-o", "go-template={{ range .items }}{{ if not .metadata.deletionTimestamp }}{{ .metadata.name }}"+
+						"{{ \"\\n\" }}{{ end }}{{ end }}")
+				Expect(err).NotTo(HaveOccurred())
+
+				By("ensuring the created controller-manager Pod")
+				podNames := kbtestutils.GetNonEmptyLines(podOutput)
+				if len(podNames) != 1 {
+					return fmt.Errorf("expect 1 controller pods running, but got %d", len(podNames))
+				}
+				controllerPodName = podNames[0]
+				Expect(controllerPodName).Should(ContainSubstring("controller-manager"))
+
+				By("checking the controller-manager Pod is running")
+				status, err := tc.Kubectl.Get(
+					true,
+					"pods", controllerPodName, "-o", "jsonpath={.status.phase}")
+				Expect(err).NotTo(HaveOccurred())
+				if status != "Running" {
+					return fmt.Errorf("controller pod in %s status", status)
+				}
+				return nil
+			}
+			Eventually(verifyControllerUp, 2*time.Minute, time.Second).Should(Succeed())
+
+			By("ensuring the created ServiceMonitor for the manager")
+			_, err := tc.Kubectl.Get(
+				true,
+				"ServiceMonitor",
+				fmt.Sprintf("e2e-%s-controller-manager-metrics-monitor", tc.TestSuffix))
+			Expect(err).NotTo(HaveOccurred())
+
+			By("ensuring the created metrics Service for the manager")
+			_, err = tc.Kubectl.Get(
+				true,
+				"Service",
+				fmt.Sprintf("e2e-%s-controller-manager-metrics-service", tc.TestSuffix))
+			Expect(err).NotTo(HaveOccurred())
+
+			By("creating an instance of CR")
+			// currently controller-runtime doesn't provide a readiness probe, we retry a few times
+			// we can change it to probe the readiness endpoint after CR supports it.
+			sampleFile := filepath.Join("config", "samples",
+				fmt.Sprintf("%s_%s_%s.yaml", tc.Group, tc.Version, strings.ToLower(tc.Kind)))
+			Eventually(func() error {
+				_, err = tc.Kubectl.Apply(true, "-f", sampleFile)
+				return err
+			}, time.Minute, time.Second).Should(Succeed())
+
+			By("ensuring the created resource object gets reconciled in controller")
+			managerContainerLogs := func() string {
+				logOutput, err := tc.Kubectl.Logs(controllerPodName, "-c", "manager")
+				Expect(err).NotTo(HaveOccurred())
+				return logOutput
+			}
+			Eventually(managerContainerLogs, time.Minute, time.Second).Should(ContainSubstring("Successfully Reconciled"))
+
+			By("granting permissions to access the metrics and read the token")
+			_, err = tc.Kubectl.Command(
+				"create",
+				"clusterrolebinding",
+				fmt.Sprintf("metrics-%s", tc.TestSuffix),
+				fmt.Sprintf("--clusterrole=e2e-%s-metrics-reader", tc.TestSuffix),
+				fmt.Sprintf("--serviceaccount=%s:default", tc.Kubectl.Namespace))
+			Expect(err).NotTo(HaveOccurred())
+
+			By("getting the token")
+			b64Token, err := tc.Kubectl.Get(
+				true,
+				"secrets",
+				"-o=jsonpath={.items[0].data.token}")
+			Expect(err).NotTo(HaveOccurred())
+			token, err := base64.StdEncoding.DecodeString(strings.TrimSpace(b64Token))
+			Expect(err).NotTo(HaveOccurred())
+			Expect(token).NotTo(HaveLen(0))
+
+			By("creating a pod with curl image")
+			// todo: the flag --generator=run-pod/v1 is deprecated, however, shows that besides
+			// it should not make any difference and work locally successfully when the flag is removed
+			// travis has been failing and the curl pod is not found when the flag is not used
+			cmdOpts := []string{
+				"run", "--generator=run-pod/v1", "curl", "--image=curlimages/curl:7.68.0", "--restart=OnFailure", "--",
+				"curl", "-v", "-k", "-H", fmt.Sprintf(`Authorization: Bearer %s`, token),
+				fmt.Sprintf("https://e2e-%v-controller-manager-metrics-service.e2e-%v-system.svc:8443/metrics",
+					tc.TestSuffix, tc.TestSuffix),
+			}
+			_, err = tc.Kubectl.CommandInNamespace(cmdOpts...)
+			Expect(err).NotTo(HaveOccurred())
+
+			By("validating the curl pod running as expected")
+			verifyCurlUp := func() error {
+				// Validate pod status
+				status, err := tc.Kubectl.Get(
+					true,
+					"pods", "curl", "-o", "jsonpath={.status.phase}")
+				Expect(err).NotTo(HaveOccurred())
+				if status != "Completed" && status != "Succeeded" {
+					return fmt.Errorf("curl pod in %s status", status)
+				}
+				return nil
+			}
+			Eventually(verifyCurlUp, 4*time.Minute, time.Second).Should(Succeed())
+
+			By("checking metrics endpoint serving as expected")
+			getCurlLogs := func() string {
+				logOutput, err := tc.Kubectl.Logs("curl")
+				Expect(err).NotTo(HaveOccurred())
+				return logOutput
+			}
+			Eventually(getCurlLogs, time.Minute, time.Second).Should(ContainSubstring("< HTTP/2 200"))
+		})
+	})
+})

test/e2e-go/e2e_go_local_test.go

@@ -0,0 +1,50 @@
+// Copyright 2020 The Operator-SDK Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e_go_test
+
+import (
+	"os/exec"
+
+	. "github.com/onsi/ginkgo" //nolint:golint
+	. "github.com/onsi/gomega" //nolint:golint
+)
+
+var _ = Describe("Running Go projects", func() {
+	Context("built with operator-sdk", func() {
+
+		BeforeEach(func() {
+			By("installing CRD's")
+			err := tc.Make("install")
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		AfterEach(func() {
+			By("uninstalling CRD's")
+			err := tc.Make("uninstall")
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("should run correctly locally", func() {
+			By("running the project")
+			cmd := exec.Command("make", "run")
+			err := cmd.Start()
+			Expect(err).NotTo(HaveOccurred())
+
+			By("killing the project")
+			err = cmd.Process.Kill()
+			Expect(err).NotTo(HaveOccurred())
+		})
+	})
+})