taint_based_evictions.go

/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package scheduling

import (
	"errors"
	"fmt"
	"time"

	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/fields"
	clientset "k8s.io/client-go/kubernetes"
	schedulerapi "k8s.io/kubernetes/pkg/scheduler/api"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
)

func newUnreachableNoExecuteTaint() *v1.Taint {
	return &v1.Taint{
		Key:    schedulerapi.TaintNodeUnreachable,
		Effect: v1.TaintEffectNoExecute,
	}
}

func getTolerationSeconds(tolerations []v1.Toleration) (int64, error) {
	for _, t := range tolerations {
		if t.Key == schedulerapi.TaintNodeUnreachable && t.Effect == v1.TaintEffectNoExecute && t.Operator == v1.TolerationOpExists {
			return *t.TolerationSeconds, nil
		}
	}
	return 0, errors.New("cannot find toleration")
}

var _ = SIGDescribe("TaintBasedEvictions [Serial]", func() {
	f := framework.NewDefaultFramework("sched-taint-based-evictions")
	var cs clientset.Interface
	var ns string

	BeforeEach(func() {
		cs = f.ClientSet
		ns = f.Namespace.Name
		// skip if TaintBasedEvictions is not enabled
		// TODO(Huang-Wei): remove this when TaintBasedEvictions is GAed
		framework.SkipUnlessTaintBasedEvictionsEnabled()
		// it's required to run on a cluster that has more than 1 node
		// otherwise node lifecycle manager enters a fully disruption mode
		framework.SkipUnlessNodeCountIsAtLeast(2)
	})

	// This test verifies that when a node becomes unreachable
	// 1. node lifecycle manager generate a status change: [NodeReady=true, status=ConditionUnknown]
	// 1. it's applied with node.kubernetes.io/unreachable=:NoExecute taint
	// 2. pods without toleration are applied with toleration with tolerationSeconds=300
	// 3. pods with toleration and without tolerationSeconds won't be modifed, and won't be evicted
	// 4. pods with toleration and with tolerationSeconds won't be modified, and will be evicted after tolerationSeconds
	// When network issue recovers, it's expected to see:
	// 5. node lifecycle manager generate a status change: [NodeReady=true, status=ConditionTrue]
	// 6. node.kubernetes.io/unreachable=:NoExecute taint is taken off the node
	It("Checks that the node becomes unreachable", func() {
		// find an available node
		nodeName := GetNodeThatCanRunPod(f)
		By("Finding an available node " + nodeName)

		// pod0 is a pod with unschedulable=:NoExecute toleration, and tolerationSeconds=0s
		// pod1 is a pod with unschedulable=:NoExecute toleration, and tolerationSeconds=200s
		// pod2 is a pod without any toleration
		base := "taint-based-eviction"
		tolerationSeconds := []int64{0, 200}
		numPods := len(tolerationSeconds) + 1
		By(fmt.Sprintf("Preparing %v pods", numPods))
		pods := make([]*v1.Pod, numPods)
		zero := int64(0)
		// build pod0, pod1
		for i := 0; i < numPods-1; i++ {
			pods[i] = createPausePod(f, pausePodConfig{
				Name:     fmt.Sprintf("%v-%v", base, i),
				NodeName: nodeName,
				Tolerations: []v1.Toleration{
					{
						Key:               schedulerapi.TaintNodeUnreachable,
						Operator:          v1.TolerationOpExists,
						Effect:            v1.TaintEffectNoExecute,
						TolerationSeconds: &tolerationSeconds[i],
					},
				},
				DeletionGracePeriodSeconds: &zero,
			})
		}
		// build pod2
		pods[numPods-1] = createPausePod(f, pausePodConfig{
			Name:     fmt.Sprintf("%v-%v", base, numPods-1),
			NodeName: nodeName,
		})

		By("Verifying all pods are running properly")
		for _, pod := range pods {
			framework.ExpectNoError(framework.WaitForPodRunningInNamespace(cs, pod))
		}

		// get the node API object
		nodeSelector := fields.OneTermEqualSelector("metadata.name", nodeName)
		nodeList, err := cs.CoreV1().Nodes().List(metav1.ListOptions{FieldSelector: nodeSelector.String()})
		if err != nil || len(nodeList.Items) != 1 {
			framework.Failf("expected no err, got %v; expected len(nodes) = 1, got %v", err, len(nodeList.Items))
		}
		node := nodeList.Items[0]

		By(fmt.Sprintf("Blocking traffic from node %s to the master", nodeName))
		host, err := framework.GetNodeExternalIP(&node)
		// TODO(Huang-Wei): make this case work for local provider
		// if err != nil {
		// 	host, err = framework.GetNodeInternalIP(&node)
		// }
		framework.ExpectNoError(err)
		masterAddresses := framework.GetAllMasterAddresses(cs)
		taint := newUnreachableNoExecuteTaint()

		defer func() {
			By(fmt.Sprintf("Unblocking traffic from node %s to the master", node.Name))
			for _, masterAddress := range masterAddresses {
				framework.UnblockNetwork(host, masterAddress)
			}

			if CurrentGinkgoTestDescription().Failed {
				framework.Failf("Current e2e test has failed, so return from here.")
				return
			}

			By(fmt.Sprintf("Expecting to see node %q becomes Ready", nodeName))
			framework.WaitForNodeToBeReady(cs, nodeName, time.Minute*1)
			By("Expecting to see unreachable=:NoExecute taint is taken off")
			err := framework.WaitForNodeHasTaintOrNot(cs, nodeName, taint, false, time.Second*30)
			framework.ExpectNoError(err)
		}()

		for _, masterAddress := range masterAddresses {
			framework.BlockNetwork(host, masterAddress)
		}

		By(fmt.Sprintf("Expecting to see node %q becomes NotReady", nodeName))
		if !framework.WaitForNodeToBeNotReady(cs, nodeName, time.Minute*3) {
			framework.Failf("node %q doesn't turn to NotReady after 3 minutes", nodeName)
		}
		By("Expecting to see unreachable=:NoExecute taint is applied")
		err = framework.WaitForNodeHasTaintOrNot(cs, nodeName, taint, true, time.Second*30)
		framework.ExpectNoError(err)

		By("Expecting pod0 to be evicted immediately")
		err = framework.WaitForPodCondition(cs, ns, pods[0].Name, "pod0 terminating", time.Second*15, func(pod *v1.Pod) (bool, error) {
			// as node is unreachable, pod0 is expected to be in Terminating status
			// rather than getting deleted
			if pod.DeletionTimestamp != nil {
				return true, nil
			}
			return false, nil
		})
		framework.ExpectNoError(err)

		By("Expecting pod2 to be updated with a toleration with tolerationSeconds=300")
		err = framework.WaitForPodCondition(cs, ns, pods[2].Name, "pod2 updated with tolerationSeconds=300", time.Second*15, func(pod *v1.Pod) (bool, error) {
			if seconds, err := getTolerationSeconds(pod.Spec.Tolerations); err == nil {
				return seconds == 300, nil
			}
			return false, nil
		})
		framework.ExpectNoError(err)

		By("Expecting pod1 to be unchanged")
		livePod1, err := cs.CoreV1().Pods(pods[1].Namespace).Get(pods[1].Name, metav1.GetOptions{})
		framework.ExpectNoError(err)
		seconds, err := getTolerationSeconds(livePod1.Spec.Tolerations)
		framework.ExpectNoError(err)
		if seconds != 200 {
			framework.Failf("expect tolerationSeconds of pod1 is 200, but got %v", seconds)
		}
	})
})