e2e_node: add tests for 2 Kubelet plugins

bart0sh · May 3, 2024 · 60321bb · 60321bb
1 parent e93a0dd
commit 60321bb
Showing 1 changed file with 156 additions and 1 deletion.
diff --git a/test/e2e_node/dra_test.go b/test/e2e_node/dra_test.go
@@ -54,6 +54,8 @@ import (
 
 const (
 	driverName                = "test-driver.cdi.k8s.io"
+	kubeletPlugin1Name        = "test-driver1.cdi.k8s.io"
+	kubeletPlugin2Name        = "test-driver2.cdi.k8s.io"
 	cdiDir                    = "/var/run/cdi"
 	endpointTemplate          = "/var/lib/kubelet/plugins/%s/dra.sock"
 	pluginRegistrationPath    = "/var/lib/kubelet/plugins_registry"
@@ -65,7 +67,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
 	f := framework.NewDefaultFramework("dra-node")
 	f.NamespacePodSecurityLevel = admissionapi.LevelBaseline
 
-	var kubeletPlugin *testdriver.ExamplePlugin
+	var kubeletPlugin, kubeletPlugin1, kubeletPlugin2 *testdriver.ExamplePlugin
 
 	f.Context("Resource Kubelet Plugin", f.WithSerial(), func() {
 		ginkgo.BeforeEach(func(ctx context.Context) {
@@ -308,6 +310,159 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
 			gomega.Consistently(kubeletPlugin.CountCalls("/NodePrepareResources")).WithTimeout(dra.PluginClientTimeout).Should(gomega.Equal(calls))
 		})
 	})
+
+	f.Context("Two resource Kubelet Plugins", f.WithSerial(), func() {
+		ginkgo.BeforeEach(func(ctx context.Context) {
+			kubeletPlugin1 = newKubeletPlugin(ctx, getNodeName(ctx, f), kubeletPlugin1Name)
+			kubeletPlugin2 = newKubeletPlugin(ctx, getNodeName(ctx, f), kubeletPlugin2Name)
+
+			ginkgo.By("wait for Kubelet plugin registration")
+			gomega.Eventually(kubeletPlugin1.GetGRPCCalls()).WithTimeout(pluginRegistrationTimeout).Should(testdriver.BeRegistered)
+			gomega.Eventually(kubeletPlugin2.GetGRPCCalls()).WithTimeout(pluginRegistrationTimeout).Should(testdriver.BeRegistered)
+		})
+
+		ginkgo.It("must prepare and unprepare resources", func(ctx context.Context) {
+			pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", 0, true, []string{kubeletPlugin1Name, kubeletPlugin2Name})
+
+			ginkgo.By("wait for pod to succeed")
+			err := e2epod.WaitForPodSuccessInNamespace(ctx, f.ClientSet, pod.Name, f.Namespace.Name)
+			framework.ExpectNoError(err)
+
+			ginkgo.By("wait for NodePrepareResources calls to succeed")
+			gomega.Eventually(kubeletPlugin1.GetGRPCCalls).WithTimeout(dra.PluginClientTimeout * 2).Should(testdriver.NodePrepareResourcesSucceeded)
+			gomega.Eventually(kubeletPlugin2.GetGRPCCalls).WithTimeout(dra.PluginClientTimeout * 2).Should(testdriver.NodePrepareResourcesSucceeded)
+
+			ginkgo.By("wait for NodeUnprepareResources calls to succeed")
+			gomega.Eventually(kubeletPlugin1.GetGRPCCalls).WithTimeout(dra.PluginClientTimeout * 2).Should(testdriver.NodeUnprepareResourcesSucceeded)
+			gomega.Eventually(kubeletPlugin2.GetGRPCCalls).WithTimeout(dra.PluginClientTimeout * 2).Should(testdriver.NodeUnprepareResourcesSucceeded)
+		})
+
+		ginkgo.It("must run pod if NodePrepareResources fails for one plugin and then succeeds", func(ctx context.Context) {
+			ginkgo.By("set failure mode for plugin2")
+			kubeletPlugin2.SetFailure(errors.New("Simulated failure"))
+			pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", 0, true, []string{kubeletPlugin1Name, kubeletPlugin2Name})
+
+			ginkgo.By("wait for pod to be in Pending state")
+			err := e2epod.WaitForPodCondition(ctx, f.ClientSet, f.Namespace.Name, pod.Name, "Pending", framework.PodStartShortTimeout, func(pod *v1.Pod) (bool, error) {
+				return pod.Status.Phase == v1.PodPending, nil
+			})
+			framework.ExpectNoError(err)
+
+			ginkgo.By("wait for plugin1 NodePrepareResources call to succeed")
+			gomega.Eventually(kubeletPlugin1.GetGRPCCalls).WithTimeout(dra.PluginClientTimeout * 2).Should(testdriver.NodePrepareResourcesSucceeded)
+
+			ginkgo.By("wait for plugin2 NodePrepareResources call to fail")
+			gomega.Eventually(kubeletPlugin2.GetGRPCCalls).WithTimeout(dra.PluginClientTimeout * 2).Should(testdriver.NodePrepareResourcesErrored)
+
+			ginkgo.By("set success mode for plugin2")
+			kubeletPlugin2.SetFailure(nil)
+
+			ginkgo.By("wait for plugin2 NodePrepareResources call to succeed")
+			gomega.Eventually(kubeletPlugin2.GetGRPCCalls).WithTimeout(dra.PluginClientTimeout * 2).Should(testdriver.NodePrepareResourcesSucceeded)
+
+			ginkgo.By("wait for pod to succeed")
+			err = e2epod.WaitForPodSuccessInNamespace(ctx, f.ClientSet, pod.Name, f.Namespace.Name)
+			framework.ExpectNoError(err)
+		})
+
+		ginkgo.It("must run pod if NodeUnprepareResources fails for one plugin and then succeeds", func(ctx context.Context) {
+			pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", 10, true, []string{kubeletPlugin1Name, kubeletPlugin2Name})
+
+			ginkgo.By("wait for pod to run")
+			err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
+			framework.ExpectNoError(err)
+
+			ginkgo.By("wait for plugin1 NodePrepareResources call to succeed")
+			gomega.Eventually(kubeletPlugin1.GetGRPCCalls).WithTimeout(dra.PluginClientTimeout * 2).Should(testdriver.NodePrepareResourcesSucceeded)
+
+			ginkgo.By("wait for plugin2 NodePrepareResources call to succeed")
+			gomega.Eventually(kubeletPlugin2.GetGRPCCalls).WithTimeout(dra.PluginClientTimeout * 2).Should(testdriver.NodePrepareResourcesSucceeded)
+
+			ginkgo.By("set failure mode for plugin2")
+			kubeletPlugin2.SetFailure(errors.New("Simulated failure"))
+
+			ginkgo.By("wait for plugin2 NodeUnprepareResources call to fail")
+			gomega.Eventually(kubeletPlugin2.GetGRPCCalls).WithTimeout(dra.PluginClientTimeout * 2).Should(testdriver.NodeUnprepareResourcesErrored)
+
+			ginkgo.By("wait for plugin1 NodeUnprepareResources call to succeed")
+			gomega.Eventually(kubeletPlugin1.GetGRPCCalls).WithTimeout(dra.PluginClientTimeout * 2).Should(testdriver.NodeUnprepareResourcesSucceeded)
+
+			ginkgo.By("set success mode for plugin2")
+			kubeletPlugin2.SetFailure(nil)
+
+			ginkgo.By("wait for plugin2 NodeUnprepareResources call to succeed")
+			gomega.Eventually(kubeletPlugin2.GetGRPCCalls).WithTimeout(dra.PluginClientTimeout * 2).Should(testdriver.NodeUnprepareResourcesSucceeded)
+
+			ginkgo.By("wait for pod to succeed")
+			err = e2epod.WaitForPodSuccessInNamespace(ctx, f.ClientSet, pod.Name, f.Namespace.Name)
+			framework.ExpectNoError(err)
+		})
+
+		ginkgo.It("must run pod if NodePrepareResources is in progress for one plugin when Kubelet restarts", func(ctx context.Context) {
+			ginkgo.By("block plugin2")
+			kubeletPlugin2.Block()
+			pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", 0, true, []string{kubeletPlugin1Name, kubeletPlugin2Name})
+
+			ginkgo.By("wait for pod to be in Pending state")
+			err := e2epod.WaitForPodCondition(ctx, f.ClientSet, f.Namespace.Name, pod.Name, "Pending", framework.PodStartShortTimeout, func(pod *v1.Pod) (bool, error) {
+				return pod.Status.Phase == v1.PodPending, nil
+			})
+			framework.ExpectNoError(err)
+
+			ginkgo.By("wait for plugin1 NodePrepareResources call to succeed")
+			gomega.Eventually(kubeletPlugin1.GetGRPCCalls).WithTimeout(dra.PluginClientTimeout * 2).Should(testdriver.NodePrepareResourcesSucceeded)
+
+			ginkgo.By("restart Kubelet")
+			restartKubelet(true)
+
+			ginkgo.By("unblock plugin2")
+			kubeletPlugin2.Unblock()
+
+			ginkgo.By("wait for plugin2 NodePrepareResources call to succeed")
+			gomega.Eventually(kubeletPlugin2.GetGRPCCalls).WithTimeout(dra.PluginClientTimeout * 2).Should(testdriver.NodePrepareResourcesSucceeded)
+
+			ginkgo.By("wait for pod to succeed")
+			err = e2epod.WaitForPodSuccessInNamespace(ctx, f.ClientSet, pod.Name, f.Namespace.Name)
+			framework.ExpectNoError(err)
+		})
+
+		ginkgo.It("must call NodeUnprepareResources again if it's in progress for one plugin when Kubelet restarts", func(ctx context.Context) {
+			pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", 10, true, []string{kubeletPlugin1Name, kubeletPlugin2Name})
+
+			ginkgo.By("wait for pod to run")
+			err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
+			framework.ExpectNoError(err)
+
+			ginkgo.By("wait for plugin1 NodePrepareResources call to succeed")
+			gomega.Eventually(kubeletPlugin1.GetGRPCCalls).WithTimeout(dra.PluginClientTimeout * 2).Should(testdriver.NodePrepareResourcesSucceeded)
+
+			ginkgo.By("wait for plugin2 NodePrepareResources call to succeed")
+			gomega.Eventually(kubeletPlugin2.GetGRPCCalls).WithTimeout(dra.PluginClientTimeout * 2).Should(testdriver.NodePrepareResourcesSucceeded)
+
+			ginkgo.By("block plugin2")
+			kubeletPlugin2.Block()
+			ginkgo.By("set failure mode for plugin2")
+			kubeletPlugin2.SetFailure(errors.New("Simulated failure"))
+
+			ginkgo.By("wait for plugin1 NodeUnprepareResources call to succeed")
+			gomega.Eventually(kubeletPlugin1.GetGRPCCalls).WithTimeout(dra.PluginClientTimeout * 2).Should(testdriver.NodeUnprepareResourcesSucceeded)
+
+			ginkgo.By("restart Kubelet")
+			restartKubelet(true)
+
+			ginkgo.By("set success mode for plugin2")
+			kubeletPlugin2.SetFailure(nil)
+			ginkgo.By("unblock plugin2")
+			kubeletPlugin2.Unblock()
+
+			ginkgo.By("wait for plugin2 NodeUnprepareResources call to succeed")
+			gomega.Eventually(kubeletPlugin2.GetGRPCCalls).WithTimeout(dra.PluginClientTimeout * 2).Should(testdriver.NodeUnprepareResourcesSucceeded)
+
+			ginkgo.By("wait for pod to succeed")
+			err = e2epod.WaitForPodSuccessInNamespace(ctx, f.ClientSet, pod.Name, f.Namespace.Name)
+			framework.ExpectNoError(err)
+		})
+	})
 })
 
 // Run Kubelet plugin and wait until it's registered