diff --git a/test/e2e/framework/node/BUILD b/test/e2e/framework/node/BUILD index a4a2d4cf032b..c05930b48a4c 100644 --- a/test/e2e/framework/node/BUILD +++ b/test/e2e/framework/node/BUILD @@ -10,6 +10,7 @@ go_library( importpath = "k8s.io/kubernetes/test/e2e/framework/node", visibility = ["//visibility:public"], deps = [ + "//pkg/controller:go_default_library", "//pkg/scheduler/framework/v1alpha1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", @@ -22,6 +23,7 @@ go_library( "//test/e2e/system:go_default_library", "//test/utils:go_default_library", "//test/utils/image:go_default_library", + "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", "//vendor/k8s.io/utils/pointer:go_default_library", ], diff --git a/test/e2e/framework/node/resource.go b/test/e2e/framework/node/resource.go index d8f05ea72649..5a9c70d1dd58 100644 --- a/test/e2e/framework/node/resource.go +++ b/test/e2e/framework/node/resource.go @@ -23,6 +23,7 @@ import ( "strings" "time" + "github.com/onsi/ginkgo" "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" @@ -30,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/util/rand" "k8s.io/apimachinery/pkg/util/sets" clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/pkg/controller" schedfwk "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" e2elog "k8s.io/kubernetes/test/e2e/framework/log" "k8s.io/kubernetes/test/e2e/system" @@ -556,3 +558,33 @@ func CreatePodsPerNodeForSimpleApp(c clientset.Interface, namespace, appName str } return podLabels } + +// RemoveTaintOffNode removes the given taint from the given node. +func RemoveTaintOffNode(c clientset.Interface, nodeName string, taint v1.Taint) { + err := controller.RemoveTaintOffNode(c, nodeName, nil, &taint) + + // TODO use wrapper methods in expect.go after removing core e2e dependency on node + gomega.ExpectWithOffset(2, err).NotTo(gomega.HaveOccurred()) + verifyThatTaintIsGone(c, nodeName, &taint) +} + +func verifyThatTaintIsGone(c clientset.Interface, nodeName string, taint *v1.Taint) { + ginkgo.By("verifying the node doesn't have the taint " + taint.ToString()) + nodeUpdated, err := c.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) + + // TODO use wrapper methods in expect.go after removing core e2e dependency on node + gomega.ExpectWithOffset(2, err).NotTo(gomega.HaveOccurred()) + if taintExists(nodeUpdated.Spec.Taints, taint) { + e2elog.Failf("Failed removing taint " + taint.ToString() + " of the node " + nodeName) + } +} + +// taintExists checks if the given taint exists in list of taints. Returns true if exists false otherwise. +func taintExists(taints []v1.Taint, taintToFind *v1.Taint) bool { + for _, taint := range taints { + if taint.MatchTaint(taintToFind) { + return true + } + } + return false +} diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index b18e943bf733..bc31b8c5bde1 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -978,12 +978,6 @@ func ExpectNodeHasLabel(c clientset.Interface, nodeName string, labelKey string, ExpectEqual(node.Labels[labelKey], labelValue) } -// RemoveTaintOffNode removes the given taint from the given node. -func RemoveTaintOffNode(c clientset.Interface, nodeName string, taint v1.Taint) { - ExpectNoError(controller.RemoveTaintOffNode(c, nodeName, nil, &taint)) - verifyThatTaintIsGone(c, nodeName, &taint) -} - // AddOrUpdateTaintOnNode adds the given taint to the given node or updates taint. func AddOrUpdateTaintOnNode(c clientset.Interface, nodeName string, taint v1.Taint) { ExpectNoError(controller.AddOrUpdateTaintOnNode(c, nodeName, &taint)) @@ -999,15 +993,6 @@ func RemoveLabelOffNode(c clientset.Interface, nodeName string, labelKey string) ExpectNoError(testutils.VerifyLabelsRemoved(c, nodeName, []string{labelKey})) } -func verifyThatTaintIsGone(c clientset.Interface, nodeName string, taint *v1.Taint) { - ginkgo.By("verifying the node doesn't have the taint " + taint.ToString()) - nodeUpdated, err := c.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) - ExpectNoError(err) - if taintExists(nodeUpdated.Spec.Taints, taint) { - Failf("Failed removing taint " + taint.ToString() + " of the node " + nodeName) - } -} - // ExpectNodeHasTaint expects that the node has the given taint. func ExpectNodeHasTaint(c clientset.Interface, nodeName string, taint *v1.Taint) { ginkgo.By("verifying the node has the taint " + taint.ToString()) diff --git a/test/e2e/kubectl/BUILD b/test/e2e/kubectl/BUILD index 250a9717a275..567c010c6be2 100644 --- a/test/e2e/kubectl/BUILD +++ b/test/e2e/kubectl/BUILD @@ -35,6 +35,7 @@ go_library( "//test/e2e/framework/auth:go_default_library", "//test/e2e/framework/endpoints:go_default_library", "//test/e2e/framework/kubectl:go_default_library", + "//test/e2e/framework/node:go_default_library", "//test/e2e/framework/pod:go_default_library", "//test/e2e/framework/service:go_default_library", "//test/e2e/framework/testfiles:go_default_library", diff --git a/test/e2e/kubectl/kubectl.go b/test/e2e/kubectl/kubectl.go index 365e5650a948..b85d9e94a2fe 100644 --- a/test/e2e/kubectl/kubectl.go +++ b/test/e2e/kubectl/kubectl.go @@ -65,6 +65,7 @@ import ( e2eauth "k8s.io/kubernetes/test/e2e/framework/auth" e2eendpoints "k8s.io/kubernetes/test/e2e/framework/endpoints" e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl" + e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2eservice "k8s.io/kubernetes/test/e2e/framework/service" e2etestfiles "k8s.io/kubernetes/test/e2e/framework/testfiles" @@ -1604,7 +1605,7 @@ metadata: ginkgo.By("adding the taint " + testTaint.ToString() + " to a node") runKubectlRetryOrDie(ns, "taint", "nodes", nodeName, testTaint.ToString()) - defer framework.RemoveTaintOffNode(f.ClientSet, nodeName, testTaint) + defer e2enode.RemoveTaintOffNode(f.ClientSet, nodeName, testTaint) ginkgo.By("verifying the node has the taint " + testTaint.ToString()) output := runKubectlRetryOrDie(ns, "describe", "node", nodeName) @@ -1635,7 +1636,7 @@ metadata: ginkgo.By("adding the taint " + testTaint.ToString() + " to a node") runKubectlRetryOrDie(ns, "taint", "nodes", nodeName, testTaint.ToString()) - defer framework.RemoveTaintOffNode(f.ClientSet, nodeName, testTaint) + defer e2enode.RemoveTaintOffNode(f.ClientSet, nodeName, testTaint) ginkgo.By("verifying the node has the taint " + testTaint.ToString()) output := runKubectlRetryOrDie(ns, "describe", "node", nodeName) @@ -1653,7 +1654,7 @@ metadata: } ginkgo.By("adding another taint " + newTestTaint.ToString() + " to the node") runKubectlRetryOrDie(ns, "taint", "nodes", nodeName, newTestTaint.ToString()) - defer framework.RemoveTaintOffNode(f.ClientSet, nodeName, newTestTaint) + defer e2enode.RemoveTaintOffNode(f.ClientSet, nodeName, newTestTaint) ginkgo.By("verifying the node has the taint " + newTestTaint.ToString()) output = runKubectlRetryOrDie(ns, "describe", "node", nodeName) @@ -1671,7 +1672,7 @@ metadata: } ginkgo.By("adding NoExecute taint " + noExecuteTaint.ToString() + " to the node") runKubectlRetryOrDie(ns, "taint", "nodes", nodeName, noExecuteTaint.ToString()) - defer framework.RemoveTaintOffNode(f.ClientSet, nodeName, noExecuteTaint) + defer e2enode.RemoveTaintOffNode(f.ClientSet, nodeName, noExecuteTaint) ginkgo.By("verifying the node has the taint " + noExecuteTaint.ToString()) output = runKubectlRetryOrDie(ns, "describe", "node", nodeName) diff --git a/test/e2e/node/runtimeclass.go b/test/e2e/node/runtimeclass.go index f60aa5900db5..42f3a17e3ee9 100644 --- a/test/e2e/node/runtimeclass.go +++ b/test/e2e/node/runtimeclass.go @@ -92,7 +92,7 @@ var _ = ginkgo.Describe("[sig-node] RuntimeClass", func() { } framework.AddOrUpdateTaintOnNode(f.ClientSet, nodeName, taint) framework.ExpectNodeHasTaint(f.ClientSet, nodeName, &taint) - defer framework.RemoveTaintOffNode(f.ClientSet, nodeName, taint) + defer e2enode.RemoveTaintOffNode(f.ClientSet, nodeName, taint) ginkgo.By("Trying to create runtimeclass and pod") runtimeClass := newRuntimeClass(f.Namespace.Name, "non-conflict-runtimeclass", framework.TestContext.ContainerRuntime) diff --git a/test/e2e/node/taints.go b/test/e2e/node/taints.go index b22062d72fab..0e9b810d9746 100644 --- a/test/e2e/node/taints.go +++ b/test/e2e/node/taints.go @@ -190,7 +190,7 @@ var _ = SIGDescribe("NoExecuteTaintManager Single Pod [Serial]", func() { testTaint := getTestTaint() framework.AddOrUpdateTaintOnNode(cs, nodeName, testTaint) framework.ExpectNodeHasTaint(cs, nodeName, &testTaint) - defer framework.RemoveTaintOffNode(cs, nodeName, testTaint) + defer e2enode.RemoveTaintOffNode(cs, nodeName, testTaint) // Wait a bit ginkgo.By("Waiting for Pod to be deleted") @@ -222,7 +222,7 @@ var _ = SIGDescribe("NoExecuteTaintManager Single Pod [Serial]", func() { testTaint := getTestTaint() framework.AddOrUpdateTaintOnNode(cs, nodeName, testTaint) framework.ExpectNodeHasTaint(cs, nodeName, &testTaint) - defer framework.RemoveTaintOffNode(cs, nodeName, testTaint) + defer e2enode.RemoveTaintOffNode(cs, nodeName, testTaint) // Wait a bit ginkgo.By("Waiting for Pod to be deleted") @@ -255,7 +255,7 @@ var _ = SIGDescribe("NoExecuteTaintManager Single Pod [Serial]", func() { testTaint := getTestTaint() framework.AddOrUpdateTaintOnNode(cs, nodeName, testTaint) framework.ExpectNodeHasTaint(cs, nodeName, &testTaint) - defer framework.RemoveTaintOffNode(cs, nodeName, testTaint) + defer e2enode.RemoveTaintOffNode(cs, nodeName, testTaint) // Wait a bit ginkgo.By("Waiting to see if a Pod won't be deleted") @@ -305,7 +305,7 @@ var _ = SIGDescribe("NoExecuteTaintManager Single Pod [Serial]", func() { taintRemoved := false defer func() { if !taintRemoved { - framework.RemoveTaintOffNode(cs, nodeName, testTaint) + e2enode.RemoveTaintOffNode(cs, nodeName, testTaint) } }() @@ -322,7 +322,7 @@ var _ = SIGDescribe("NoExecuteTaintManager Single Pod [Serial]", func() { // 4. Remove the taint framework.Logf("Removing taint from Node") - framework.RemoveTaintOffNode(cs, nodeName, testTaint) + e2enode.RemoveTaintOffNode(cs, nodeName, testTaint) taintRemoved = true // 5. See if Pod won't be evicted. @@ -376,11 +376,11 @@ var _ = SIGDescribe("NoExecuteTaintManager Multiple Pods [Serial]", func() { testTaint := getTestTaint() framework.AddOrUpdateTaintOnNode(cs, nodeName1, testTaint) framework.ExpectNodeHasTaint(cs, nodeName1, &testTaint) - defer framework.RemoveTaintOffNode(cs, nodeName1, testTaint) + defer e2enode.RemoveTaintOffNode(cs, nodeName1, testTaint) if nodeName2 != nodeName1 { framework.AddOrUpdateTaintOnNode(cs, nodeName2, testTaint) framework.ExpectNodeHasTaint(cs, nodeName2, &testTaint) - defer framework.RemoveTaintOffNode(cs, nodeName2, testTaint) + defer e2enode.RemoveTaintOffNode(cs, nodeName2, testTaint) } // Wait a bit @@ -449,7 +449,7 @@ var _ = SIGDescribe("NoExecuteTaintManager Multiple Pods [Serial]", func() { testTaint := getTestTaint() framework.AddOrUpdateTaintOnNode(cs, nodeName, testTaint) framework.ExpectNodeHasTaint(cs, nodeName, &testTaint) - defer framework.RemoveTaintOffNode(cs, nodeName, testTaint) + defer e2enode.RemoveTaintOffNode(cs, nodeName, testTaint) // 3. Wait to see if both pods get evicted in between [5, 25] seconds ginkgo.By("Waiting for Pod1 and Pod2 to be deleted") diff --git a/test/e2e/scheduling/predicates.go b/test/e2e/scheduling/predicates.go index 462646e177d5..da7caccfe60c 100644 --- a/test/e2e/scheduling/predicates.go +++ b/test/e2e/scheduling/predicates.go @@ -587,7 +587,7 @@ var _ = SIGDescribe("SchedulerPredicates [Serial]", func() { } framework.AddOrUpdateTaintOnNode(cs, nodeName, testTaint) framework.ExpectNodeHasTaint(cs, nodeName, &testTaint) - defer framework.RemoveTaintOffNode(cs, nodeName, testTaint) + defer e2enode.RemoveTaintOffNode(cs, nodeName, testTaint) ginkgo.By("Trying to apply a random label on the found node.") labelKey := fmt.Sprintf("kubernetes.io/e2e-label-key-%s", string(uuid.NewUUID())) @@ -630,7 +630,7 @@ var _ = SIGDescribe("SchedulerPredicates [Serial]", func() { } framework.AddOrUpdateTaintOnNode(cs, nodeName, testTaint) framework.ExpectNodeHasTaint(cs, nodeName, &testTaint) - defer framework.RemoveTaintOffNode(cs, nodeName, testTaint) + defer e2enode.RemoveTaintOffNode(cs, nodeName, testTaint) ginkgo.By("Trying to apply a random label on the found node.") labelKey := fmt.Sprintf("kubernetes.io/e2e-label-key-%s", string(uuid.NewUUID())) @@ -904,7 +904,7 @@ func getRequestedStorageEphemeralStorage(pod v1.Pod) int64 { // from the given node upon invocation. func removeTaintFromNodeAction(cs clientset.Interface, nodeName string, testTaint v1.Taint) Action { return func() error { - framework.RemoveTaintOffNode(cs, nodeName, testTaint) + e2enode.RemoveTaintOffNode(cs, nodeName, testTaint) return nil } } diff --git a/test/e2e/scheduling/priorities.go b/test/e2e/scheduling/priorities.go index 4c90fbc32fd4..30b46a87f634 100644 --- a/test/e2e/scheduling/priorities.go +++ b/test/e2e/scheduling/priorities.go @@ -314,14 +314,14 @@ var _ = SIGDescribe("SchedulerPriorities [Serial]", func() { for i := 0; i < 10; i++ { testTaint := addRandomTaintToNode(cs, nodeName) tolerations = append(tolerations, v1.Toleration{Key: testTaint.Key, Value: testTaint.Value, Effect: testTaint.Effect}) - defer framework.RemoveTaintOffNode(cs, nodeName, *testTaint) + defer e2enode.RemoveTaintOffNode(cs, nodeName, *testTaint) } ginkgo.By("Adding 10 intolerable taints to all other nodes") for i := 1; i < len(nodeList.Items); i++ { node := nodeList.Items[i] for i := 0; i < 10; i++ { testTaint := addRandomTaintToNode(cs, node.Name) - defer framework.RemoveTaintOffNode(cs, node.Name, *testTaint) + defer e2enode.RemoveTaintOffNode(cs, node.Name, *testTaint) } }