Integrate LitmusChaos - Pod Memory Hog experiment (#2014)

* updating readme Signed-off-by: Mahfuza Humayra Mohona <mhmohona@gmail.com> Signed-off-by: Mahfuza Mohona <mahfuza.mohona@LEADSOFT.BIZ> * Updating GetWithRetry function Signed-off-by: Mahfuza Humayra Mohona <mhmohona@gmail.com> Signed-off-by: Mahfuza Mohona <mahfuza.mohona@LEADSOFT.BIZ> * Updating GetWithRetry function Signed-off-by: Mahfuza Humayra Mohona <mhmohona@gmail.com> Signed-off-by: Mahfuza Mohona <mahfuza.mohona@LEADSOFT.BIZ> * Updating GetWithRetry function Signed-off-by: Mahfuza Humayra Mohona <mhmohona@gmail.com> Signed-off-by: Mahfuza Mohona <mahfuza.mohona@LEADSOFT.BIZ> * removing update Signed-off-by: Mahfuza Mohona <mahfuza.mohona@LEADSOFT.BIZ> * Update utils.go Signed-off-by: Mahfuza Mohona <mahfuza.mohona@LEADSOFT.BIZ>
kyverno · Jun 23, 2021 · 9e769d1 · 9e769d1
1 parent 9efd58f
commit 9e769d1
Show file tree

Hide file tree

Showing 4 changed files with 471 additions and 0 deletions.
diff --git a/litmuschaos/README.md b/litmuschaos/README.md
@@ -0,0 +1,29 @@
+# Integration of Kyverno with Litmus
+
+Kyverno is a policy engine designed for Kubernetes. It can validate, mutate, and generate configurations using admission controls and background scans. Litmus provides a large number of experiments for testing containers, pods, nodes, as well as specific platforms and tools. The advantage of chaos engineering is that one can quickly figure out issues that other testing layers cannot easily capture. This can save a lot of time in the future, and will help to find the loopholes in the system and fix them.
+
+
+## Steps to Execute LitmusChaos Experiment
+
+### Prerequisites
+ * Ensure that Kubernetes Version > 1.15
+ * Ensure that the Kyverno is running by executing `kubectl get pods` in operator namespace (typically, `kyverno`). If not, install from [here](https://kyverno.io/docs/installation/).
+* Update Kyverno Deployment to use `ghcr.io/kyverno/kyverno:test-litmuschaos` image. Note that this image is built specifically to run Litmuschaos experiments per this request,  [CHAOS_KILL_COMMAND](https://docs.litmuschaos.io/docs/pod-cpu-hog/#prepare-chaosengine). The official Kyverno images will adopt this soon.
+ * Ensure that the Litmus Chaos Operator is running by executing `kubectl get pods` in operator namespace (typically, `litmus`). If not, install from [here](https://docs.litmuschaos.io/docs/getstarted/#install-litmus).
+
+
+### Running experiment
+Aftr setting up the docker images, for running a LitmusChaos experiment following steps need to be followed - 
+- Firstly, exicute ` eval export E2E="ok" `
+- Run the Chaos Experiment Test Command - ` go test ./litmuschaos/pod_cpu_hog -v `.
+
+The test passes if the enforce policy shows it's expected behaviour. 
+
+# Experiments
+
+| Experiment name  | LitmusChaos experiment - Pod CPU Hog |
+| :-------------: | ------------- |
+| Test command  | ` go test ./litmuschaos/pod_cpu_hog -v ` |
+| Goal  | Seeing how the overall application stack behaves when Kyverno pods experience CPU spikes either due to expected/undesired processes  |
+| Performed tests |  <li> Deploy enforce policy. </li><li>Run the chaos test to consume CPU resources on the application container. </li><li> Verify the enforce policy behaviour.  </li></li>|
+| Expected result  | Kyverno pod is responding after running Pod CPU Hog Experiment |
diff --git a/litmuschaos/pod_cpu_hog/config.go b/litmuschaos/pod_cpu_hog/config.go
@@ -0,0 +1,26 @@
+package e2e
+
+type testData struct {
+	testResourceName, group, version, resource, namespace string
+	manifest                                              []byte
+}
+
+// Pod CPU hog test
+var PodCPUHogTest = struct {
+	//TestName - Name of the Test
+	TestName string
+	TestData []testData
+}{
+
+	TestName: "test-litmus-chaos-experiment",
+	TestData: []testData{
+		{
+			testResourceName: "add-new-capabilities",
+			group:            "",
+			version:          "v1",
+			resource:         "Pod",
+			namespace:        "test-litmus",
+			manifest:         KyvernoTestResourcesYaml,
+		},
+	},
+}
diff --git a/litmuschaos/pod_cpu_hog/pod_cpu_hog_test.go b/litmuschaos/pod_cpu_hog/pod_cpu_hog_test.go
@@ -0,0 +1,147 @@
+package e2e
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/kyverno/kyverno/test/e2e"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var (
+	// Namespace GVR
+	nsGVR = e2e.GetGVR("", "v1", "namespaces")
+	// Chaos service account GVR
+	saGVR = e2e.GetGVR("", "v1", "serviceaccounts")
+	// Role GVR
+	rGVR = e2e.GetGVR("rbac.authorization.k8s.io", "v1", "roles")
+	// RoleBinding GVR
+	rbGVR = e2e.GetGVR("rbac.authorization.k8s.io", "v1", "rolebindings")
+	// PodCPUHogExperiment GVR
+	cpuGVR = e2e.GetGVR("litmuschaos.io", "v1alpha1", "chaosexperiments")
+	// ChaosEngine GVR
+	ceGVR = e2e.GetGVR("litmuschaos.io", "v1alpha1", "chaosengines")
+	// Chaos Result GVR
+	crGVR = e2e.GetGVR("litmuschaos.io", "v1alpha1", "chaosresults")
+	// Cluster Policy GVR
+	clPolGVR = e2e.GetGVR("kyverno.io", "v1", "clusterpolicies")
+	// Kyverno disallow_cri_sock_mount Policy GVR
+	dcsmPolGVR = e2e.GetGVR("", "v1", "pods")
+
+	// ClusterPolicy Namespace
+	clPolNS = ""
+	// Namespace Name
+	// Hardcoded in YAML Definition
+	nspace = "test-litmus"
+)
+
+func Test_Pod_CPU_Hog(t *testing.T) {
+	RegisterTestingT(t)
+	if os.Getenv("E2E") == "" {
+		t.Skip("Skipping E2E Test")
+	}
+
+	// Generate E2E Client
+	e2eClient, err := e2e.NewE2EClient()
+	Expect(err).To(BeNil())
+
+	for _, resource := range PodCPUHogTest.TestData {
+
+		// CleanUp Resources
+		By(fmt.Sprintf("Cleaning Cluster Policies in %s", nspace))
+		e2eClient.CleanClusterPolicies(clPolGVR) //Clean Cluster Policy
+		By(fmt.Sprintf("Deleting Namespace : %s", nspace))
+		e2eClient.DeleteClusteredResource(nsGVR, nspace) // Clear Namespace
+		e2eClient.DeleteNamespacedResource(dcsmPolGVR, nspace, resource.testResourceName)
+		e2e.GetWithRetry(time.Duration(1), 15, func() error { // Wait Till Deletion of Namespace
+			_, err := e2eClient.GetClusteredResource(nsGVR, nspace)
+			if err != nil {
+				return nil
+			}
+			return errors.New("Deleting Namespace")
+		})
+
+		// Create Namespace
+		By(fmt.Sprintf("Creating Namespace %s", saGVR))
+		_, err = e2eClient.CreateClusteredResourceYaml(nsGVR, LitmusChaosnamespaceYaml)
+		Expect(err).NotTo(HaveOccurred())
+		e2e.GetWithRetry(time.Duration(1), 15, func() error { // Wait Till Creation of Namespace
+			_, err := e2eClient.GetClusteredResource(nsGVR, resource.namespace)
+			if err != nil {
+				return err
+			}
+			return nil
+		})
+
+		// ================== Litmus Chaos Experiment ==================
+		// Prepare chaosServiceAccount
+		By(fmt.Sprintf("\nPrepareing Chaos Service Account in %s", nspace))
+		_, err = e2eClient.CreateNamespacedResourceYaml(saGVR, nspace, ChaosServiceAccountYaml)
+		Expect(err).NotTo(HaveOccurred())
+		_, err = e2eClient.CreateNamespacedResourceYaml(rGVR, nspace, ChaosRoleYaml)
+		Expect(err).NotTo(HaveOccurred())
+		_, err = e2eClient.CreateNamespacedResourceYaml(rbGVR, nspace, ChaosRoleBindingYaml)
+		Expect(err).NotTo(HaveOccurred())
+
+		// Deploy Pod CPU Hog Experiment
+		By(fmt.Sprintf("\nInstalling Litmus Chaos Experiment in %s", nspace))
+		_, err = e2eClient.CreateNamespacedResourceYaml(cpuGVR, nspace, PodCPUHogExperimentYaml)
+		Expect(err).NotTo(HaveOccurred())
+
+		// Prepare Chaos Engine
+		By(fmt.Sprintf("\nCreating ChaosEngine Resource in %s", nspace))
+		_, err = e2eClient.CreateNamespacedResourceYaml(ceGVR, nspace, ChaosEngineYaml)
+		Expect(err).NotTo(HaveOccurred())
+
+		By(fmt.Sprintf("\nMonitoring status from ChaosResult in %s", nspace))
+
+		e2e.GetWithRetry(time.Duration(30), 5, func() error { // Wait Till preparing Chaos engine
+			chaosresult, err := e2eClient.GetNamespacedResource(crGVR, nspace, "kind-chaos-pod-cpu-hog")
+			if err != nil {
+				return fmt.Errorf("Unable to fatch ChaosResult: %v", err)
+			}
+			chaosVerdict, _, err := unstructured.NestedString(chaosresult.UnstructuredContent(), "status", "experimentStatus", "verdict")
+			if err != nil {
+				By(fmt.Sprintf("\nUnable to fatch the status.verdict from ChaosResult: %v", err))
+			}
+
+			By(fmt.Sprintf("\nChaos verdict %s", chaosVerdict))
+
+			if chaosVerdict == "Pass" {
+				return nil
+			}
+			return errors.New("Chaos result is not passed")
+		})
+
+		// Create disallow_cri_sock_mount policy
+		By(fmt.Sprintf("\nCreating Enforce Policy in %s", clPolNS))
+		_, err = e2eClient.CreateNamespacedResourceYaml(clPolGVR, clPolNS, DisallowAddingCapabilitiesYaml)
+		Expect(err).NotTo(HaveOccurred())
+
+		// Deploy disallow_cri_sock_mount policy
+		By(fmt.Sprintf("\nDeploying Enforce Policy in %s", nspace))
+		_, err = e2eClient.CreateNamespacedResourceYaml(dcsmPolGVR, nspace, resource.manifest)
+		Expect(err).To(HaveOccurred())
+
+		//CleanUp Resources
+		e2eClient.CleanClusterPolicies(clPolGVR) //Clean Cluster Policy
+		e2eClient.CleanClusterPolicies(saGVR)
+		e2eClient.DeleteClusteredResource(nsGVR, nspace)      // Clear Namespace
+		e2e.GetWithRetry(time.Duration(1), 15, func() error { // Wait Till Deletion of Namespace
+			_, err := e2eClient.GetClusteredResource(nsGVR, nspace)
+			if err != nil {
+				return nil
+			}
+			return errors.New("Deleting Namespace")
+		})
+
+		By(fmt.Sprintf("Test %s Completed. \n\n\n", PodCPUHogTest.TestName))
+	}
+
+}