From 2a7503e6d86a4b8c73f34e00f298c7bfa987c8b7 Mon Sep 17 00:00:00 2001 From: assaf-admi Date: Tue, 18 Oct 2022 14:56:10 +0300 Subject: [PATCH] Test for operator custom metrics Signed-off-by: assaf-admi --- .../memcached-with-webhooks/e2e_test_code.go | 335 +++++++++++++++--- .../memcached-operator/test/e2e/e2e_test.go | 309 +++++++++++++--- .../v3/memcached-operator/test/utils/utils.go | 24 ++ .../memcached-operator/test/e2e/e2e_test.go | 309 +++++++++++++--- .../memcached-operator/test/utils/utils.go | 24 ++ 5 files changed, 832 insertions(+), 169 deletions(-) diff --git a/hack/generate/samples/internal/go/memcached-with-webhooks/e2e_test_code.go b/hack/generate/samples/internal/go/memcached-with-webhooks/e2e_test_code.go index 8e7e35cb18f..2c478929b6c 100644 --- a/hack/generate/samples/internal/go/memcached-with-webhooks/e2e_test_code.go +++ b/hack/generate/samples/internal/go/memcached-with-webhooks/e2e_test_code.go @@ -142,9 +142,13 @@ limitations under the License. package e2e import ( + "bufio" + "encoding/json" "fmt" + "os" "os/exec" "path/filepath" + "strconv" "strings" "time" @@ -159,77 +163,105 @@ import ( "github.com/example/memcached-operator/test/utils" ) -// namespace store the ns where the Operator and Operand will be executed -const namespace = "memcached-operator-system" - -var _ = Describe("memcached", func() { - - Context("ensure that Operator and Operand(s) can run in restricted namespaces", func() { - BeforeEach(func() { - // The prometheus and the certmanager are installed in this test - // because the Memcached sample has this option enable and - // when we try to apply the manifests both will be required to be installed - By("installing prometheus operator") - Expect(utils.InstallPrometheusOperator()).To(Succeed()) - - By("installing the cert-manager") - Expect(utils.InstallCertManager()).To(Succeed()) - - // The namespace can be created when we run make install - // However, in this test we want to ensure that the solution - // can run in a ns labeled as restricted. Therefore, we are - // creating the namespace and labeling it. - By("creating manager namespace") - cmd := exec.Command("kubectl", "create", "ns", namespace) - _, _ = utils.Run(cmd) - - // Now, let's ensure that all namespaces can raise a Warn when we apply the manifests - // and that the namespace where the Operator and Operand will run are enforced as - // restricted so that we can ensure that both can be admitted and run with the enforcement - By("labeling all namespaces to warn when we apply the manifest if it would violate the PodStandards") - cmd = exec.Command("kubectl", "label", "--overwrite", "ns", "--all", - "pod-security.kubernetes.io/audit=restricted", - "pod-security.kubernetes.io/enforce-version=v1.24", - "pod-security.kubernetes.io/warn=restricted") - _, err := utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) +// constant parts of the file +const ( + namespace = "memcached-operator-system" + memcachedDeploymentSizeUndesiredCountTotalName = "memcached_deployment_size_undesired_count_total" + tokenRequestRawString = "{\"apiVersion\": \"authentication.k8s.io/v1\", \"kind\": \"TokenRequest\"}" +) - By("labeling enforce the namespace where the Operator and Operand(s) will run") - cmd = exec.Command("kubectl", "label", "--overwrite", "ns", namespace, - "pod-security.kubernetes.io/audit=restricted", - "pod-security.kubernetes.io/enforce-version=v1.24", - "pod-security.kubernetes.io/enforce=restricted") - _, err = utils.Run(cmd) - Expect(err).To(Not(HaveOccurred())) - }) +// tokenRequest is a trimmed down version of the authentication.k8s.io/v1/TokenRequest Type +// that we want to use for extracting the token. +type tokenRequest struct { + Status struct { + Token string "json:\"token\"" + } "json:\"status\"" +} - AfterEach(func() { - By("uninstalling the Prometheus manager bundle") - utils.UninstallPrometheusOperator() +var _ = Describe("memcached", Ordered, func() { + BeforeAll(func() { + // The prometheus and the certmanager are installed in this test + // because the Memcached sample has this option enable and + // when we try to apply the manifests both will be required to be installed + By("installing prometheus operator") + Expect(utils.InstallPrometheusOperator()).To(Succeed()) + + By("installing the cert-manager") + Expect(utils.InstallCertManager()).To(Succeed()) + + // The namespace can be created when we run make install + // However, in this test we want ensure that the solution + // can run in a ns labeled as restricted. Therefore, we are + // creating the namespace an lebeling it. + By("creating manager namespace") + cmd := exec.Command("kubectl", "create", "ns", namespace) + _, _ = utils.Run(cmd) + + // Now, let's ensure that all namespaces can raise an Warn when we apply the manifests + // and that the namespace where the Operator and Operand will run are enforced as + // restricted so that we can ensure that both can be admitted and run with the enforcement + By("labeling all namespaces to warn when we apply the manifest if would violate the PodStandards") + cmd = exec.Command("kubectl", "label", "--overwrite", "ns", "--all", + "pod-security.kubernetes.io/audit=restricted", + "pod-security.kubernetes.io/enforce-version=v1.24", + "pod-security.kubernetes.io/warn=restricted") + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("labeling enforce the namespace where the Operator and Operand(s) will run") + cmd = exec.Command("kubectl", "label", "--overwrite", "ns", namespace, + "pod-security.kubernetes.io/audit=restricted", + "pod-security.kubernetes.io/enforce-version=v1.24", + "pod-security.kubernetes.io/enforce=restricted") + _, err = utils.Run(cmd) + Expect(err).To(Not(HaveOccurred())) + + By("uncommenting all sections with 'monitoring' to enable operator custom metrics") + err = utils.ReplaceInFile("controllers/memcached_controller.go", + "//"+monitoringImportFragment, monitoringImportFragment) + Expect(err).To(Not(HaveOccurred())) + + err = utils.ReplaceInFile("controllers/memcached_controller.go", + "//"+incMemcachedDeploymentSizeUndesiredCountTotalFragment, incMemcachedDeploymentSizeUndesiredCountTotalFragment) + Expect(err).To(Not(HaveOccurred())) + + err = utils.ReplaceInFile("main.go", + "//"+monitoringImportFragment, monitoringImportFragment) + Expect(err).To(Not(HaveOccurred())) + + err = utils.ReplaceInFile("main.go", + "//"+registerMetricsFragment, registerMetricsFragment) + Expect(err).To(Not(HaveOccurred())) + }) - By("uninstalling the cert-manager bundle") - utils.UninstallCertManager() + AfterAll(func() { + By("uninstalling the Prometheus manager bundle") + utils.UninstallPrometheusOperator() - By("removing manager namespace") - cmd := exec.Command("kubectl", "create", "ns", namespace) - _, _ = utils.Run(cmd) - }) + By("uninstalling the cert-manager bundle") + utils.UninstallCertManager() + + By("removing manager namespace") + cmd := exec.Command("kubectl", "create", "ns", namespace) + _, _ = utils.Run(cmd) + }) - It("should successfully run the Memcached Operator", func() { + Context("Memcached Operator", func() { + It("should run successfully", func() { var controllerPodName string var err error projectDir, _ := utils.GetProjectDir() - // operatorImage store the name of the imahe used in the example - const operatorImage = "example.com/memcached-operator:v0.0.1" + // operatorImage stores the name of the image used in the example + var operatorImage = "example.com/memcached-operator:v0.0.1" By("building the manager(Operator) image") - cmd := exec.Command("make", "docker-build", "IMG=example.com/memcached-operator:v0.0.1") + cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", operatorImage)) _, err = utils.Run(cmd) ExpectWithOffset(1, err).NotTo(HaveOccurred()) By("loading the the manager(Operator) image on Kind") - err = utils.LoadImageToKindClusterWithName("example.com/memcached-operator:v0.0.1") + err = utils.LoadImageToKindClusterWithName(operatorImage) ExpectWithOffset(1, err).NotTo(HaveOccurred()) By("installing CRDs") @@ -287,7 +319,7 @@ var _ = Describe("memcached", func() { By("validating that pod(s) status.phase=Running") getMemcachedPodStatus := func() error { - cmd = exec.Command("kubectl", "get", + cmd = exec.Command("kubectl", "get", "pods", "-l", "app.kubernetes.io/name=Memcached", "-o", "jsonpath={.items[*].status}", "-n", namespace, ) @@ -318,7 +350,172 @@ var _ = Describe("memcached", func() { Eventually(getStatus, time.Minute, time.Second).Should(Succeed()) }) }) + + Context("Memcached Operator metrics", Ordered, func() { + BeforeAll(func() { + By("granting permissions to access the metrics") + cmd := exec.Command("kubectl", + "create", "clusterrolebinding", "metrics-memcached-operator", + "--clusterrole=memcached-operator-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:memcached-operator-controller-manager", namespace)) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterAll(func() { + By("removing permissions to access the metrics") + cmd := exec.Command("kubectl", "delete", + "clusterrolebinding", "metrics-memcached-operator") + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + }) + + It("MemcachedDeploymentSizeUndesiredCountTotal should be increased when scaling the Memcached deployment", func() { + initialMetricValue := getMetricValue(memcachedDeploymentSizeUndesiredCountTotalName) + + numberOfScales := 5 + By(fmt.Sprintf("scaling memcached-samle deployment %d times", numberOfScales)) + scaleMemcachedSampleDeployment(numberOfScales) + + By(fmt.Sprintf("validating MemcachedDeploymentSizeUndesiredCountTotal has increased by %d", numberOfScales)) + finalMetricValue := getMetricValue(memcachedDeploymentSizeUndesiredCountTotalName) + Expect(finalMetricValue).To(Equal(initialMetricValue + numberOfScales)) + }) + }) }) + +// getMetricValue will reach the Memcached operator metrics endpoint, validate the metric and extract its value +func getMetricValue(metricName string) int { + // reach the metrics endpoint and validate the metric exists + metricsEndpoint := curlMetrics() + ExpectWithOffset(1, metricsEndpoint).Should(ContainSubstring(metricName)) + + // extract the metric value + metricValue, err := strconv.Atoi(parseMetricValue(metricsEndpoint, metricName)) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + return metricValue +} + +// curlMetrics curl's the /metrics endpoint, returning all logs once a 200 status is returned. +func curlMetrics() string { + By("reading the metrics token") + // Filter token query by service account in case more than one exists in a namespace. + token, err := serviceAccountToken() + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + ExpectWithOffset(2, len(token)).To(BeNumerically(">", 0)) + + By("creating a curl pod") + cmd := exec.Command("kubectl", "run", "curl", "--image=curlimages/curl:7.68.0", + "--restart=OnFailure", "-n", "default", "--", "curl", "-v", "-k", "-H", + fmt.Sprintf("Authorization: Bearer %s", strings.TrimSpace(token)), + fmt.Sprintf("https://memcached-operator-controller-manager-metrics-service.%s.svc:8443/metrics", namespace)) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + + By("validating that the curl pod is running as expected") + verifyCurlUp := func() error { + // Validate pod status + cmd := exec.Command("kubectl", "get", "pods", "curl", + "-o", "jsonpath={.status.phase}", "-n", "default") + statusOutput, err := utils.Run(cmd) + status := string(statusOutput) + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + if status != "Completed" && status != "Succeeded" { + return fmt.Errorf("curl pod in %s status", status) + } + return nil + } + EventuallyWithOffset(2, verifyCurlUp, 240*time.Second, time.Second).Should(Succeed()) + + By("validating that the metrics endpoint is serving as expected") + var metricsEndpoint string + getCurlLogs := func() string { + cmd = exec.Command("kubectl", "logs", "curl", "-n", "default") + metricsEndpointOutput, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + metricsEndpoint = string(metricsEndpointOutput) + return metricsEndpoint + } + EventuallyWithOffset(2, getCurlLogs, 10*time.Second, time.Second).Should(ContainSubstring("< HTTP/2 200")) + + By("cleaning up the curl pod") + cmd = exec.Command("kubectl", "delete", + "pods/curl", "-n", "default") + _, err = utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + + return metricsEndpoint +} + +// serviceAccountToken provides a helper function that can provide you with a service account +// token that you can use to interact with the service. This function leverages the k8s' +// TokenRequest API in raw format in order to make it generic for all version of the k8s that +// is currently being supported in kubebuilder test infra. +// TokenRequest API returns the token in raw JWT format itself. There is no conversion required. +func serviceAccountToken() (out string, err error) { + By("Creating the ServiceAccount token") + secretName := "memcached-operator-controller-manager-token-request" + projectDir, _ := utils.GetProjectDir() + tokenRequestFile := filepath.Join(projectDir, "/test/e2e/", secretName) + err = os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o755)) + if err != nil { + return out, err + } + var rawJson string + Eventually(func() error { + // Output of this is already a valid JWT token. No need to covert this from base64 to string format + cmd := exec.Command("kubectl", "create", "--raw", + fmt.Sprintf("/api/v1/namespaces/%s/serviceaccounts/memcached-operator-controller-manager/token", namespace), + "-f", tokenRequestFile, + ) + rawJsonOutput, err := utils.Run(cmd) + rawJson = string(rawJsonOutput) + if err != nil { + return err + } + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + if err != nil { + return err + } + out = token.Status.Token + return nil + }, time.Minute, time.Second).Should(Succeed()) + + return out, err +} + +// parseMetricValue will parse the metric value from the metrics endpoint +func parseMetricValue(metricsEndpoint string, metricName string) string { + r := strings.NewReader(metricsEndpoint) + scan := bufio.NewScanner(r) + for scan.Scan() { + metricLine := scan.Text() + if strings.HasPrefix(metricLine, metricName) { + split := strings.Split(metricLine, " ") + return split[1] + } + } + return "" +} + +// scaleMemcachedSampleDeployment will scale memcached-sample deployment 'numberOfScales' times +func scaleMemcachedSampleDeployment(numberOfScales int) { + for i := 1; i <= numberOfScales; i++ { + cmd := exec.Command("kubectl", "scale", "--replicas=3", + "deployment", "memcached-sample", "-n", namespace) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + time.Sleep(10 * time.Second) + } +} + +const monitoringImportFragment = "\"github.com/example/memcached-operator/monitoring\"" + +const incMemcachedDeploymentSizeUndesiredCountTotalFragment = "monitoring.MemcachedDeploymentSizeUndesiredCountTotal.Inc()" + +const registerMetricsFragment = "monitoring.RegisterMetrics()" ` const utilsTemplate = `/* @@ -340,6 +537,7 @@ limitations under the License. package utils import ( + "errors" "fmt" "os" "os/exec" @@ -464,6 +662,29 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + +// ReplaceInFile replaces all instances of old with new in the file at path. +func ReplaceInFile(path, old, new string) error { + info, err := os.Stat(path) + if err != nil { + return err + } + // false positive + // nolint:gosec + b, err := os.ReadFile(path) + if err != nil { + return err + } + if !strings.Contains(string(b), old) { + return errors.New("unable to find the content to be replaced") + } + s := strings.Replace(string(b), old, new, -1) + err = os.WriteFile(path, []byte(s), info.Mode()) + if err != nil { + return err + } + return nil +} ` const targetTemplate = ` diff --git a/testdata/go/v3/memcached-operator/test/e2e/e2e_test.go b/testdata/go/v3/memcached-operator/test/e2e/e2e_test.go index 7fca7e41c5f..204575e7984 100644 --- a/testdata/go/v3/memcached-operator/test/e2e/e2e_test.go +++ b/testdata/go/v3/memcached-operator/test/e2e/e2e_test.go @@ -17,9 +17,13 @@ limitations under the License. package e2e import ( + "bufio" + "encoding/json" "fmt" + "os" "os/exec" "path/filepath" + "strconv" "strings" "time" @@ -34,77 +38,105 @@ import ( "github.com/example/memcached-operator/test/utils" ) -// namespace store the ns where the Operator and Operand will be executed -const namespace = "memcached-operator-system" - -var _ = Describe("memcached", func() { - - Context("ensure that Operator and Operand(s) can run in restricted namespaces", func() { - BeforeEach(func() { - // The prometheus and the certmanager are installed in this test - // because the Memcached sample has this option enable and - // when we try to apply the manifests both will be required to be installed - By("installing prometheus operator") - Expect(utils.InstallPrometheusOperator()).To(Succeed()) - - By("installing the cert-manager") - Expect(utils.InstallCertManager()).To(Succeed()) - - // The namespace can be created when we run make install - // However, in this test we want to ensure that the solution - // can run in a ns labeled as restricted. Therefore, we are - // creating the namespace and labeling it. - By("creating manager namespace") - cmd := exec.Command("kubectl", "create", "ns", namespace) - _, _ = utils.Run(cmd) - - // Now, let's ensure that all namespaces can raise a Warn when we apply the manifests - // and that the namespace where the Operator and Operand will run are enforced as - // restricted so that we can ensure that both can be admitted and run with the enforcement - By("labeling all namespaces to warn when we apply the manifest if it would violate the PodStandards") - cmd = exec.Command("kubectl", "label", "--overwrite", "ns", "--all", - "pod-security.kubernetes.io/audit=restricted", - "pod-security.kubernetes.io/enforce-version=v1.24", - "pod-security.kubernetes.io/warn=restricted") - _, err := utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) +// constant parts of the file +const ( + namespace = "memcached-operator-system" + memcachedDeploymentSizeUndesiredCountTotalName = "memcached_deployment_size_undesired_count_total" + tokenRequestRawString = "{\"apiVersion\": \"authentication.k8s.io/v1\", \"kind\": \"TokenRequest\"}" +) - By("labeling enforce the namespace where the Operator and Operand(s) will run") - cmd = exec.Command("kubectl", "label", "--overwrite", "ns", namespace, - "pod-security.kubernetes.io/audit=restricted", - "pod-security.kubernetes.io/enforce-version=v1.24", - "pod-security.kubernetes.io/enforce=restricted") - _, err = utils.Run(cmd) - Expect(err).To(Not(HaveOccurred())) - }) +// tokenRequest is a trimmed down version of the authentication.k8s.io/v1/TokenRequest Type +// that we want to use for extracting the token. +type tokenRequest struct { + Status struct { + Token string "json:\"token\"" + } "json:\"status\"" +} - AfterEach(func() { - By("uninstalling the Prometheus manager bundle") - utils.UninstallPrometheusOperator() +var _ = Describe("memcached", Ordered, func() { + BeforeAll(func() { + // The prometheus and the certmanager are installed in this test + // because the Memcached sample has this option enable and + // when we try to apply the manifests both will be required to be installed + By("installing prometheus operator") + Expect(utils.InstallPrometheusOperator()).To(Succeed()) - By("uninstalling the cert-manager bundle") - utils.UninstallCertManager() + By("installing the cert-manager") + Expect(utils.InstallCertManager()).To(Succeed()) - By("removing manager namespace") - cmd := exec.Command("kubectl", "create", "ns", namespace) - _, _ = utils.Run(cmd) - }) + // The namespace can be created when we run make install + // However, in this test we want ensure that the solution + // can run in a ns labeled as restricted. Therefore, we are + // creating the namespace an lebeling it. + By("creating manager namespace") + cmd := exec.Command("kubectl", "create", "ns", namespace) + _, _ = utils.Run(cmd) + + // Now, let's ensure that all namespaces can raise an Warn when we apply the manifests + // and that the namespace where the Operator and Operand will run are enforced as + // restricted so that we can ensure that both can be admitted and run with the enforcement + By("labeling all namespaces to warn when we apply the manifest if would violate the PodStandards") + cmd = exec.Command("kubectl", "label", "--overwrite", "ns", "--all", + "pod-security.kubernetes.io/audit=restricted", + "pod-security.kubernetes.io/enforce-version=v1.24", + "pod-security.kubernetes.io/warn=restricted") + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("labeling enforce the namespace where the Operator and Operand(s) will run") + cmd = exec.Command("kubectl", "label", "--overwrite", "ns", namespace, + "pod-security.kubernetes.io/audit=restricted", + "pod-security.kubernetes.io/enforce-version=v1.24", + "pod-security.kubernetes.io/enforce=restricted") + _, err = utils.Run(cmd) + Expect(err).To(Not(HaveOccurred())) + + By("uncommenting all sections with 'monitoring' to enable operator custom metrics") + err = utils.ReplaceInFile("controllers/memcached_controller.go", + "//"+monitoringImportFragment, monitoringImportFragment) + Expect(err).To(Not(HaveOccurred())) + + err = utils.ReplaceInFile("controllers/memcached_controller.go", + "//"+incMemcachedDeploymentSizeUndesiredCountTotalFragment, incMemcachedDeploymentSizeUndesiredCountTotalFragment) + Expect(err).To(Not(HaveOccurred())) + + err = utils.ReplaceInFile("main.go", + "//"+monitoringImportFragment, monitoringImportFragment) + Expect(err).To(Not(HaveOccurred())) - It("should successfully run the Memcached Operator", func() { + err = utils.ReplaceInFile("main.go", + "//"+registerMetricsFragment, registerMetricsFragment) + Expect(err).To(Not(HaveOccurred())) + }) + + AfterAll(func() { + By("uninstalling the Prometheus manager bundle") + utils.UninstallPrometheusOperator() + + By("uninstalling the cert-manager bundle") + utils.UninstallCertManager() + + By("removing manager namespace") + cmd := exec.Command("kubectl", "create", "ns", namespace) + _, _ = utils.Run(cmd) + }) + + Context("Memcached Operator", func() { + It("should run successfully", func() { var controllerPodName string var err error projectDir, _ := utils.GetProjectDir() - // operatorImage store the name of the imahe used in the example - const operatorImage = "example.com/memcached-operator:v0.0.1" + // operatorImage stores the name of the image used in the example + var operatorImage = "example.com/memcached-operator:v0.0.1" By("building the manager(Operator) image") - cmd := exec.Command("make", "docker-build", "IMG=example.com/memcached-operator:v0.0.1") + cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", operatorImage)) _, err = utils.Run(cmd) ExpectWithOffset(1, err).NotTo(HaveOccurred()) By("loading the the manager(Operator) image on Kind") - err = utils.LoadImageToKindClusterWithName("example.com/memcached-operator:v0.0.1") + err = utils.LoadImageToKindClusterWithName(operatorImage) ExpectWithOffset(1, err).NotTo(HaveOccurred()) By("installing CRDs") @@ -193,4 +225,169 @@ var _ = Describe("memcached", func() { Eventually(getStatus, time.Minute, time.Second).Should(Succeed()) }) }) + + Context("Memcached Operator metrics", Ordered, func() { + BeforeAll(func() { + By("granting permissions to access the metrics") + cmd := exec.Command("kubectl", + "create", "clusterrolebinding", "metrics-memcached-operator", + "--clusterrole=memcached-operator-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:memcached-operator-controller-manager", namespace)) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterAll(func() { + By("removing permissions to access the metrics") + cmd := exec.Command("kubectl", "delete", + "clusterrolebinding", "metrics-memcached-operator") + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + }) + + It("MemcachedDeploymentSizeUndesiredCountTotal should be increased when scaling the Memcached deployment", func() { + initialMetricValue := getMetricValue(memcachedDeploymentSizeUndesiredCountTotalName) + + numberOfScales := 5 + By(fmt.Sprintf("scaling memcached-samle deployment %d times", numberOfScales)) + scaleMemcachedSampleDeployment(numberOfScales) + + By(fmt.Sprintf("validating MemcachedDeploymentSizeUndesiredCountTotal has increased by %d", numberOfScales)) + finalMetricValue := getMetricValue(memcachedDeploymentSizeUndesiredCountTotalName) + Expect(finalMetricValue).To(Equal(initialMetricValue + numberOfScales)) + }) + }) }) + +// getMetricValue will reach the Memcached operator metrics endpoint, validate the metric and extract its value +func getMetricValue(metricName string) int { + // reach the metrics endpoint and validate the metric exists + metricsEndpoint := curlMetrics() + ExpectWithOffset(1, metricsEndpoint).Should(ContainSubstring(metricName)) + + // extract the metric value + metricValue, err := strconv.Atoi(parseMetricValue(metricsEndpoint, metricName)) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + return metricValue +} + +// curlMetrics curl's the /metrics endpoint, returning all logs once a 200 status is returned. +func curlMetrics() string { + By("reading the metrics token") + // Filter token query by service account in case more than one exists in a namespace. + token, err := serviceAccountToken() + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + ExpectWithOffset(2, len(token)).To(BeNumerically(">", 0)) + + By("creating a curl pod") + cmd := exec.Command("kubectl", "run", "curl", "--image=curlimages/curl:7.68.0", + "--restart=OnFailure", "-n", "default", "--", "curl", "-v", "-k", "-H", + fmt.Sprintf("Authorization: Bearer %s", strings.TrimSpace(token)), + fmt.Sprintf("https://memcached-operator-controller-manager-metrics-service.%s.svc:8443/metrics", namespace)) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + + By("validating that the curl pod is running as expected") + verifyCurlUp := func() error { + // Validate pod status + cmd := exec.Command("kubectl", "get", "pods", "curl", + "-o", "jsonpath={.status.phase}", "-n", "default") + statusOutput, err := utils.Run(cmd) + status := string(statusOutput) + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + if status != "Completed" && status != "Succeeded" { + return fmt.Errorf("curl pod in %s status", status) + } + return nil + } + EventuallyWithOffset(2, verifyCurlUp, 240*time.Second, time.Second).Should(Succeed()) + + By("validating that the metrics endpoint is serving as expected") + var metricsEndpoint string + getCurlLogs := func() string { + cmd = exec.Command("kubectl", "logs", "curl", "-n", "default") + metricsEndpointOutput, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + metricsEndpoint = string(metricsEndpointOutput) + return metricsEndpoint + } + EventuallyWithOffset(2, getCurlLogs, 10*time.Second, time.Second).Should(ContainSubstring("< HTTP/2 200")) + + By("cleaning up the curl pod") + cmd = exec.Command("kubectl", "delete", + "pods/curl", "-n", "default") + _, err = utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + + return metricsEndpoint +} + +// serviceAccountToken provides a helper function that can provide you with a service account +// token that you can use to interact with the service. This function leverages the k8s' +// TokenRequest API in raw format in order to make it generic for all version of the k8s that +// is currently being supported in kubebuilder test infra. +// TokenRequest API returns the token in raw JWT format itself. There is no conversion required. +func serviceAccountToken() (out string, err error) { + By("Creating the ServiceAccount token") + secretName := "memcached-operator-controller-manager-token-request" + projectDir, _ := utils.GetProjectDir() + tokenRequestFile := filepath.Join(projectDir, "/test/e2e/", secretName) + err = os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o755)) + if err != nil { + return out, err + } + var rawJson string + Eventually(func() error { + // Output of this is already a valid JWT token. No need to covert this from base64 to string format + cmd := exec.Command("kubectl", "create", "--raw", + fmt.Sprintf("/api/v1/namespaces/%s/serviceaccounts/memcached-operator-controller-manager/token", namespace), + "-f", tokenRequestFile, + ) + rawJsonOutput, err := utils.Run(cmd) + rawJson = string(rawJsonOutput) + if err != nil { + return err + } + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + if err != nil { + return err + } + out = token.Status.Token + return nil + }, time.Minute, time.Second).Should(Succeed()) + + return out, err +} + +// parseMetricValue will parse the metric value from the metrics endpoint +func parseMetricValue(metricsEndpoint string, metricName string) string { + r := strings.NewReader(metricsEndpoint) + scan := bufio.NewScanner(r) + for scan.Scan() { + metricLine := scan.Text() + if strings.HasPrefix(metricLine, metricName) { + split := strings.Split(metricLine, " ") + return split[1] + } + } + return "" +} + +// scaleMemcachedSampleDeployment will scale memcached-sample deployment 'numberOfScales' times +func scaleMemcachedSampleDeployment(numberOfScales int) { + for i := 1; i <= numberOfScales; i++ { + cmd := exec.Command("kubectl", "scale", "--replicas=3", + "deployment", "memcached-sample", "-n", namespace) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + time.Sleep(10 * time.Second) + } +} + +const monitoringImportFragment = "\"github.com/example/memcached-operator/monitoring\"" + +const incMemcachedDeploymentSizeUndesiredCountTotalFragment = "monitoring.MemcachedDeploymentSizeUndesiredCountTotal.Inc()" + +const registerMetricsFragment = "monitoring.RegisterMetrics()" diff --git a/testdata/go/v3/memcached-operator/test/utils/utils.go b/testdata/go/v3/memcached-operator/test/utils/utils.go index da7dc942b3d..5c53ebe4804 100644 --- a/testdata/go/v3/memcached-operator/test/utils/utils.go +++ b/testdata/go/v3/memcached-operator/test/utils/utils.go @@ -17,6 +17,7 @@ limitations under the License. package utils import ( + "errors" "fmt" "os" "os/exec" @@ -141,3 +142,26 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + +// ReplaceInFile replaces all instances of old with new in the file at path. +func ReplaceInFile(path, old, new string) error { + info, err := os.Stat(path) + if err != nil { + return err + } + // false positive + // nolint:gosec + b, err := os.ReadFile(path) + if err != nil { + return err + } + if !strings.Contains(string(b), old) { + return errors.New("unable to find the content to be replaced") + } + s := strings.Replace(string(b), old, new, -1) + err = os.WriteFile(path, []byte(s), info.Mode()) + if err != nil { + return err + } + return nil +} diff --git a/testdata/go/v4-alpha/memcached-operator/test/e2e/e2e_test.go b/testdata/go/v4-alpha/memcached-operator/test/e2e/e2e_test.go index 7fca7e41c5f..204575e7984 100644 --- a/testdata/go/v4-alpha/memcached-operator/test/e2e/e2e_test.go +++ b/testdata/go/v4-alpha/memcached-operator/test/e2e/e2e_test.go @@ -17,9 +17,13 @@ limitations under the License. package e2e import ( + "bufio" + "encoding/json" "fmt" + "os" "os/exec" "path/filepath" + "strconv" "strings" "time" @@ -34,77 +38,105 @@ import ( "github.com/example/memcached-operator/test/utils" ) -// namespace store the ns where the Operator and Operand will be executed -const namespace = "memcached-operator-system" - -var _ = Describe("memcached", func() { - - Context("ensure that Operator and Operand(s) can run in restricted namespaces", func() { - BeforeEach(func() { - // The prometheus and the certmanager are installed in this test - // because the Memcached sample has this option enable and - // when we try to apply the manifests both will be required to be installed - By("installing prometheus operator") - Expect(utils.InstallPrometheusOperator()).To(Succeed()) - - By("installing the cert-manager") - Expect(utils.InstallCertManager()).To(Succeed()) - - // The namespace can be created when we run make install - // However, in this test we want to ensure that the solution - // can run in a ns labeled as restricted. Therefore, we are - // creating the namespace and labeling it. - By("creating manager namespace") - cmd := exec.Command("kubectl", "create", "ns", namespace) - _, _ = utils.Run(cmd) - - // Now, let's ensure that all namespaces can raise a Warn when we apply the manifests - // and that the namespace where the Operator and Operand will run are enforced as - // restricted so that we can ensure that both can be admitted and run with the enforcement - By("labeling all namespaces to warn when we apply the manifest if it would violate the PodStandards") - cmd = exec.Command("kubectl", "label", "--overwrite", "ns", "--all", - "pod-security.kubernetes.io/audit=restricted", - "pod-security.kubernetes.io/enforce-version=v1.24", - "pod-security.kubernetes.io/warn=restricted") - _, err := utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) +// constant parts of the file +const ( + namespace = "memcached-operator-system" + memcachedDeploymentSizeUndesiredCountTotalName = "memcached_deployment_size_undesired_count_total" + tokenRequestRawString = "{\"apiVersion\": \"authentication.k8s.io/v1\", \"kind\": \"TokenRequest\"}" +) - By("labeling enforce the namespace where the Operator and Operand(s) will run") - cmd = exec.Command("kubectl", "label", "--overwrite", "ns", namespace, - "pod-security.kubernetes.io/audit=restricted", - "pod-security.kubernetes.io/enforce-version=v1.24", - "pod-security.kubernetes.io/enforce=restricted") - _, err = utils.Run(cmd) - Expect(err).To(Not(HaveOccurred())) - }) +// tokenRequest is a trimmed down version of the authentication.k8s.io/v1/TokenRequest Type +// that we want to use for extracting the token. +type tokenRequest struct { + Status struct { + Token string "json:\"token\"" + } "json:\"status\"" +} - AfterEach(func() { - By("uninstalling the Prometheus manager bundle") - utils.UninstallPrometheusOperator() +var _ = Describe("memcached", Ordered, func() { + BeforeAll(func() { + // The prometheus and the certmanager are installed in this test + // because the Memcached sample has this option enable and + // when we try to apply the manifests both will be required to be installed + By("installing prometheus operator") + Expect(utils.InstallPrometheusOperator()).To(Succeed()) - By("uninstalling the cert-manager bundle") - utils.UninstallCertManager() + By("installing the cert-manager") + Expect(utils.InstallCertManager()).To(Succeed()) - By("removing manager namespace") - cmd := exec.Command("kubectl", "create", "ns", namespace) - _, _ = utils.Run(cmd) - }) + // The namespace can be created when we run make install + // However, in this test we want ensure that the solution + // can run in a ns labeled as restricted. Therefore, we are + // creating the namespace an lebeling it. + By("creating manager namespace") + cmd := exec.Command("kubectl", "create", "ns", namespace) + _, _ = utils.Run(cmd) + + // Now, let's ensure that all namespaces can raise an Warn when we apply the manifests + // and that the namespace where the Operator and Operand will run are enforced as + // restricted so that we can ensure that both can be admitted and run with the enforcement + By("labeling all namespaces to warn when we apply the manifest if would violate the PodStandards") + cmd = exec.Command("kubectl", "label", "--overwrite", "ns", "--all", + "pod-security.kubernetes.io/audit=restricted", + "pod-security.kubernetes.io/enforce-version=v1.24", + "pod-security.kubernetes.io/warn=restricted") + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("labeling enforce the namespace where the Operator and Operand(s) will run") + cmd = exec.Command("kubectl", "label", "--overwrite", "ns", namespace, + "pod-security.kubernetes.io/audit=restricted", + "pod-security.kubernetes.io/enforce-version=v1.24", + "pod-security.kubernetes.io/enforce=restricted") + _, err = utils.Run(cmd) + Expect(err).To(Not(HaveOccurred())) + + By("uncommenting all sections with 'monitoring' to enable operator custom metrics") + err = utils.ReplaceInFile("controllers/memcached_controller.go", + "//"+monitoringImportFragment, monitoringImportFragment) + Expect(err).To(Not(HaveOccurred())) + + err = utils.ReplaceInFile("controllers/memcached_controller.go", + "//"+incMemcachedDeploymentSizeUndesiredCountTotalFragment, incMemcachedDeploymentSizeUndesiredCountTotalFragment) + Expect(err).To(Not(HaveOccurred())) + + err = utils.ReplaceInFile("main.go", + "//"+monitoringImportFragment, monitoringImportFragment) + Expect(err).To(Not(HaveOccurred())) - It("should successfully run the Memcached Operator", func() { + err = utils.ReplaceInFile("main.go", + "//"+registerMetricsFragment, registerMetricsFragment) + Expect(err).To(Not(HaveOccurred())) + }) + + AfterAll(func() { + By("uninstalling the Prometheus manager bundle") + utils.UninstallPrometheusOperator() + + By("uninstalling the cert-manager bundle") + utils.UninstallCertManager() + + By("removing manager namespace") + cmd := exec.Command("kubectl", "create", "ns", namespace) + _, _ = utils.Run(cmd) + }) + + Context("Memcached Operator", func() { + It("should run successfully", func() { var controllerPodName string var err error projectDir, _ := utils.GetProjectDir() - // operatorImage store the name of the imahe used in the example - const operatorImage = "example.com/memcached-operator:v0.0.1" + // operatorImage stores the name of the image used in the example + var operatorImage = "example.com/memcached-operator:v0.0.1" By("building the manager(Operator) image") - cmd := exec.Command("make", "docker-build", "IMG=example.com/memcached-operator:v0.0.1") + cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", operatorImage)) _, err = utils.Run(cmd) ExpectWithOffset(1, err).NotTo(HaveOccurred()) By("loading the the manager(Operator) image on Kind") - err = utils.LoadImageToKindClusterWithName("example.com/memcached-operator:v0.0.1") + err = utils.LoadImageToKindClusterWithName(operatorImage) ExpectWithOffset(1, err).NotTo(HaveOccurred()) By("installing CRDs") @@ -193,4 +225,169 @@ var _ = Describe("memcached", func() { Eventually(getStatus, time.Minute, time.Second).Should(Succeed()) }) }) + + Context("Memcached Operator metrics", Ordered, func() { + BeforeAll(func() { + By("granting permissions to access the metrics") + cmd := exec.Command("kubectl", + "create", "clusterrolebinding", "metrics-memcached-operator", + "--clusterrole=memcached-operator-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:memcached-operator-controller-manager", namespace)) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterAll(func() { + By("removing permissions to access the metrics") + cmd := exec.Command("kubectl", "delete", + "clusterrolebinding", "metrics-memcached-operator") + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + }) + + It("MemcachedDeploymentSizeUndesiredCountTotal should be increased when scaling the Memcached deployment", func() { + initialMetricValue := getMetricValue(memcachedDeploymentSizeUndesiredCountTotalName) + + numberOfScales := 5 + By(fmt.Sprintf("scaling memcached-samle deployment %d times", numberOfScales)) + scaleMemcachedSampleDeployment(numberOfScales) + + By(fmt.Sprintf("validating MemcachedDeploymentSizeUndesiredCountTotal has increased by %d", numberOfScales)) + finalMetricValue := getMetricValue(memcachedDeploymentSizeUndesiredCountTotalName) + Expect(finalMetricValue).To(Equal(initialMetricValue + numberOfScales)) + }) + }) }) + +// getMetricValue will reach the Memcached operator metrics endpoint, validate the metric and extract its value +func getMetricValue(metricName string) int { + // reach the metrics endpoint and validate the metric exists + metricsEndpoint := curlMetrics() + ExpectWithOffset(1, metricsEndpoint).Should(ContainSubstring(metricName)) + + // extract the metric value + metricValue, err := strconv.Atoi(parseMetricValue(metricsEndpoint, metricName)) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + return metricValue +} + +// curlMetrics curl's the /metrics endpoint, returning all logs once a 200 status is returned. +func curlMetrics() string { + By("reading the metrics token") + // Filter token query by service account in case more than one exists in a namespace. + token, err := serviceAccountToken() + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + ExpectWithOffset(2, len(token)).To(BeNumerically(">", 0)) + + By("creating a curl pod") + cmd := exec.Command("kubectl", "run", "curl", "--image=curlimages/curl:7.68.0", + "--restart=OnFailure", "-n", "default", "--", "curl", "-v", "-k", "-H", + fmt.Sprintf("Authorization: Bearer %s", strings.TrimSpace(token)), + fmt.Sprintf("https://memcached-operator-controller-manager-metrics-service.%s.svc:8443/metrics", namespace)) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + + By("validating that the curl pod is running as expected") + verifyCurlUp := func() error { + // Validate pod status + cmd := exec.Command("kubectl", "get", "pods", "curl", + "-o", "jsonpath={.status.phase}", "-n", "default") + statusOutput, err := utils.Run(cmd) + status := string(statusOutput) + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + if status != "Completed" && status != "Succeeded" { + return fmt.Errorf("curl pod in %s status", status) + } + return nil + } + EventuallyWithOffset(2, verifyCurlUp, 240*time.Second, time.Second).Should(Succeed()) + + By("validating that the metrics endpoint is serving as expected") + var metricsEndpoint string + getCurlLogs := func() string { + cmd = exec.Command("kubectl", "logs", "curl", "-n", "default") + metricsEndpointOutput, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + metricsEndpoint = string(metricsEndpointOutput) + return metricsEndpoint + } + EventuallyWithOffset(2, getCurlLogs, 10*time.Second, time.Second).Should(ContainSubstring("< HTTP/2 200")) + + By("cleaning up the curl pod") + cmd = exec.Command("kubectl", "delete", + "pods/curl", "-n", "default") + _, err = utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + + return metricsEndpoint +} + +// serviceAccountToken provides a helper function that can provide you with a service account +// token that you can use to interact with the service. This function leverages the k8s' +// TokenRequest API in raw format in order to make it generic for all version of the k8s that +// is currently being supported in kubebuilder test infra. +// TokenRequest API returns the token in raw JWT format itself. There is no conversion required. +func serviceAccountToken() (out string, err error) { + By("Creating the ServiceAccount token") + secretName := "memcached-operator-controller-manager-token-request" + projectDir, _ := utils.GetProjectDir() + tokenRequestFile := filepath.Join(projectDir, "/test/e2e/", secretName) + err = os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o755)) + if err != nil { + return out, err + } + var rawJson string + Eventually(func() error { + // Output of this is already a valid JWT token. No need to covert this from base64 to string format + cmd := exec.Command("kubectl", "create", "--raw", + fmt.Sprintf("/api/v1/namespaces/%s/serviceaccounts/memcached-operator-controller-manager/token", namespace), + "-f", tokenRequestFile, + ) + rawJsonOutput, err := utils.Run(cmd) + rawJson = string(rawJsonOutput) + if err != nil { + return err + } + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + if err != nil { + return err + } + out = token.Status.Token + return nil + }, time.Minute, time.Second).Should(Succeed()) + + return out, err +} + +// parseMetricValue will parse the metric value from the metrics endpoint +func parseMetricValue(metricsEndpoint string, metricName string) string { + r := strings.NewReader(metricsEndpoint) + scan := bufio.NewScanner(r) + for scan.Scan() { + metricLine := scan.Text() + if strings.HasPrefix(metricLine, metricName) { + split := strings.Split(metricLine, " ") + return split[1] + } + } + return "" +} + +// scaleMemcachedSampleDeployment will scale memcached-sample deployment 'numberOfScales' times +func scaleMemcachedSampleDeployment(numberOfScales int) { + for i := 1; i <= numberOfScales; i++ { + cmd := exec.Command("kubectl", "scale", "--replicas=3", + "deployment", "memcached-sample", "-n", namespace) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + time.Sleep(10 * time.Second) + } +} + +const monitoringImportFragment = "\"github.com/example/memcached-operator/monitoring\"" + +const incMemcachedDeploymentSizeUndesiredCountTotalFragment = "monitoring.MemcachedDeploymentSizeUndesiredCountTotal.Inc()" + +const registerMetricsFragment = "monitoring.RegisterMetrics()" diff --git a/testdata/go/v4-alpha/memcached-operator/test/utils/utils.go b/testdata/go/v4-alpha/memcached-operator/test/utils/utils.go index da7dc942b3d..5c53ebe4804 100644 --- a/testdata/go/v4-alpha/memcached-operator/test/utils/utils.go +++ b/testdata/go/v4-alpha/memcached-operator/test/utils/utils.go @@ -17,6 +17,7 @@ limitations under the License. package utils import ( + "errors" "fmt" "os" "os/exec" @@ -141,3 +142,26 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + +// ReplaceInFile replaces all instances of old with new in the file at path. +func ReplaceInFile(path, old, new string) error { + info, err := os.Stat(path) + if err != nil { + return err + } + // false positive + // nolint:gosec + b, err := os.ReadFile(path) + if err != nil { + return err + } + if !strings.Contains(string(b), old) { + return errors.New("unable to find the content to be replaced") + } + s := strings.Replace(string(b), old, new, -1) + err = os.WriteFile(path, []byte(s), info.Mode()) + if err != nil { + return err + } + return nil +}