Skip to content
Permalink
Browse files

feat: update taint nodes by condition to GA

  • Loading branch information
draveness committed Sep 13, 2019
1 parent 534051a commit d79abe85fde8ea2dc86c92071a8be8f76bfee771
@@ -179,7 +179,6 @@ func startNodeLifecycleController(ctx ControllerContext) (http.Handler, bool, er
ctx.ComponentConfig.NodeLifecycleController.UnhealthyZoneThreshold,
ctx.ComponentConfig.NodeLifecycleController.EnableTaintManager,
utilfeature.DefaultFeatureGate.Enabled(features.TaintBasedEvictions),
utilfeature.DefaultFeatureGate.Enabled(features.TaintNodesByCondition),
)
if err != nil {
return nil, true, err
@@ -26,7 +26,7 @@ import (
"time"

apps "k8s.io/api/apps/v1"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
@@ -1804,8 +1804,6 @@ func TestTaintPressureNodeDaemonLaunchesPod(t *testing.T) {
}
manager.nodeStore.Add(node)

// Enabling critical pod and taint nodes by condition feature gate should create critical pod
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.TaintNodesByCondition, true)()
manager.dsStore.Add(ds)
syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0)
}
@@ -296,10 +296,6 @@ type Controller struct {
// taints instead of evicting Pods itself.
useTaintBasedEvictions bool

// if set to true, NodeController will taint Nodes based on its condition for 'NetworkUnavailable',
// 'MemoryPressure', 'PIDPressure' and 'DiskPressure'.
taintNodeByCondition bool

nodeUpdateQueue workqueue.Interface
}

@@ -320,7 +316,7 @@ func NewNodeLifecycleController(
unhealthyZoneThreshold float32,
runTaintManager bool,
useTaintBasedEvictions bool,
taintNodeByCondition bool) (*Controller, error) {
) (*Controller, error) {

if kubeClient == nil {
klog.Fatalf("kubeClient is nil when starting Controller")
@@ -359,7 +355,6 @@ func NewNodeLifecycleController(
unhealthyZoneThreshold: unhealthyZoneThreshold,
runTaintManager: runTaintManager,
useTaintBasedEvictions: useTaintBasedEvictions && runTaintManager,
taintNodeByCondition: taintNodeByCondition,
nodeUpdateQueue: workqueue.NewNamed("node_lifecycle_controller"),
}
if useTaintBasedEvictions {
@@ -469,10 +464,6 @@ func NewNodeLifecycleController(
}),
})

if nc.taintNodeByCondition {
klog.Infof("Controller will taint node by condition.")
}

nc.leaseLister = leaseInformer.Lister()
if utilfeature.DefaultFeatureGate.Enabled(features.NodeLease) {
nc.leaseInformerSynced = leaseInformer.Informer().HasSynced
@@ -547,11 +538,9 @@ func (nc *Controller) doNodeProcessingPassWorker() {
return
}
nodeName := obj.(string)
if nc.taintNodeByCondition {
if err := nc.doNoScheduleTaintingPass(nodeName); err != nil {
klog.Errorf("Failed to taint NoSchedule on node <%s>, requeue it: %v", nodeName, err)
// TODO(k82cn): Add nodeName back to the queue
}
if err := nc.doNoScheduleTaintingPass(nodeName); err != nil {
klog.Errorf("Failed to taint NoSchedule on node <%s>, requeue it: %v", nodeName, err)
// TODO(k82cn): Add nodeName back to the queue
}
// TODO: re-evaluate whether there are any labels that need to be
// reconcile in 1.19. Remove this function if it's no longer necessary.
@@ -182,7 +182,6 @@ func newNodeLifecycleControllerFromClient(
unhealthyZoneThreshold,
useTaints,
useTaints,
useTaints,
)
if err != nil {
return nil, err
@@ -127,6 +127,7 @@ const (

// owner: @k82cn
// beta: v1.12
// GA: v1.17
//
// Taint nodes based on their condition status for 'NetworkUnavailable',
// 'MemoryPressure', 'PIDPressure' and 'DiskPressure'.
@@ -512,7 +513,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
EphemeralContainers: {Default: false, PreRelease: featuregate.Alpha},
PodShareProcessNamespace: {Default: true, PreRelease: featuregate.Beta},
PodPriority: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.18
TaintNodesByCondition: {Default: true, PreRelease: featuregate.Beta},
TaintNodesByCondition: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.18
QOSReserved: {Default: false, PreRelease: featuregate.Alpha},
ExpandPersistentVolumes: {Default: true, PreRelease: featuregate.Beta},
ExpandInUsePersistentVolumes: {Default: true, PreRelease: featuregate.Beta},
@@ -142,12 +142,9 @@ func DefaultOffAdmissionPlugins() sets.String {
resourcequota.PluginName, //ResourceQuota
storageobjectinuseprotection.PluginName, //StorageObjectInUseProtection
podpriority.PluginName, //PodPriority
nodetaint.PluginName, //TaintNodesByCondition
)

if utilfeature.DefaultFeatureGate.Enabled(features.TaintNodesByCondition) {
defaultOnPlugins.Insert(nodetaint.PluginName) //TaintNodesByCondition
}

if utilfeature.DefaultFeatureGate.Enabled(features.RuntimeClass) {
defaultOnPlugins.Insert(runtimeclass.PluginName) //RuntimeClass
}
@@ -245,7 +245,6 @@ go_test(
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
"//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library",
"//staging/src/k8s.io/client-go/util/testing:go_default_library",
"//staging/src/k8s.io/component-base/featuregate:go_default_library",
"//staging/src/k8s.io/component-base/featuregate/testing:go_default_library",
"//staging/src/k8s.io/component-base/version:go_default_library",
"//vendor/github.com/google/cadvisor/info/v1:go_default_library",
@@ -144,13 +144,12 @@ func (m *managerImpl) Admit(attrs *lifecycle.PodAdmitAttributes) lifecycle.PodAd
return lifecycle.PodAdmitResult{Admit: true}
}

// When node has memory pressure and TaintNodesByCondition is enabled, check BestEffort Pod's toleration:
// When node has memory pressure, check BestEffort Pod's toleration:
// admit it if tolerates memory pressure taint, fail for other tolerations, e.g. DiskPressure.
if utilfeature.DefaultFeatureGate.Enabled(features.TaintNodesByCondition) &&
v1helper.TolerationsTolerateTaint(attrs.Pod.Spec.Tolerations, &v1.Taint{
Key: schedulerapi.TaintNodeMemoryPressure,
Effect: v1.TaintEffectNoSchedule,
}) {
if v1helper.TolerationsTolerateTaint(attrs.Pod.Spec.Tolerations, &v1.Taint{
Key: schedulerapi.TaintNodeMemoryPressure,
Effect: v1.TaintEffectNoSchedule,
}) {
return lifecycle.PodAdmitResult{Admit: true}
}
}
@@ -25,7 +25,7 @@ import (
"strings"
"time"

"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
apiequality "k8s.io/apimachinery/pkg/api/equality"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
@@ -246,13 +246,11 @@ func (kl *Kubelet) initialNode() (*v1.Node, error) {
Effect: v1.TaintEffectNoSchedule,
}

// If TaintNodesByCondition enabled, taint node with TaintNodeUnschedulable when initializing
// Taint node with TaintNodeUnschedulable when initializing
// node to avoid race condition; refer to #63897 for more detail.
if utilfeature.DefaultFeatureGate.Enabled(features.TaintNodesByCondition) {
if node.Spec.Unschedulable &&
!taintutil.TaintExists(nodeTaints, &unschedulableTaint) {
nodeTaints = append(nodeTaints, unschedulableTaint)
}
if node.Spec.Unschedulable &&
!taintutil.TaintExists(nodeTaints, &unschedulableTaint) {
nodeTaints = append(nodeTaints, unschedulableTaint)
}

if kl.externalCloudProvider {
@@ -48,7 +48,6 @@ import (
"k8s.io/client-go/kubernetes/fake"
"k8s.io/client-go/rest"
core "k8s.io/client-go/testing"
"k8s.io/component-base/featuregate"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/component-base/version"
"k8s.io/kubernetes/pkg/features"
@@ -1983,25 +1982,23 @@ func TestRegisterWithApiServerWithTaint(t *testing.T) {
// Make node to be unschedulable.
kubelet.registerSchedulable = false

forEachFeatureGate(t, []featuregate.Feature{features.TaintNodesByCondition}, func(t *testing.T) {
// Reset kubelet status for each test.
kubelet.registrationCompleted = false
// Reset kubelet status for each test.
kubelet.registrationCompleted = false

// Register node to apiserver.
kubelet.registerWithAPIServer()
// Register node to apiserver.
kubelet.registerWithAPIServer()

// Check the unschedulable taint.
got := gotNode.(*v1.Node)
unschedulableTaint := &v1.Taint{
Key: schedulerapi.TaintNodeUnschedulable,
Effect: v1.TaintEffectNoSchedule,
}
// Check the unschedulable taint.
got := gotNode.(*v1.Node)
unschedulableTaint := &v1.Taint{
Key: schedulerapi.TaintNodeUnschedulable,
Effect: v1.TaintEffectNoSchedule,
}

require.Equal(t,
utilfeature.DefaultFeatureGate.Enabled(features.TaintNodesByCondition),
taintutil.TaintExists(got.Spec.Taints, unschedulableTaint),
"test unschedulable taint for TaintNodesByCondition")
})
require.Equal(t,
true,
taintutil.TaintExists(got.Spec.Taints, unschedulableTaint),
"test unschedulable taint for TaintNodesByCondition")
}

func TestNodeStatusHasChanged(t *testing.T) {
@@ -35,12 +35,9 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/kubernetes/fake"
"k8s.io/client-go/tools/record"
"k8s.io/client-go/util/flowcontrol"
"k8s.io/component-base/featuregate"
featuregatetesting "k8s.io/component-base/featuregate/testing"
cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
"k8s.io/kubernetes/pkg/kubelet/cm"
"k8s.io/kubernetes/pkg/kubelet/config"
@@ -2031,17 +2028,6 @@ func runVolumeManager(kubelet *Kubelet) chan struct{} {
return stopCh
}

func forEachFeatureGate(t *testing.T, fs []featuregate.Feature, tf func(t *testing.T)) {
for _, fg := range fs {
for _, f := range []bool{true, false} {
func() {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, fg, f)()
t.Run(fmt.Sprintf("%v(%t)", fg, f), tf)
}()
}
}
}

// Sort pods by UID.
type podsByUID []*v1.Pod

@@ -47,12 +47,9 @@ func defaultPredicates() sets.String {
predicates.MatchInterPodAffinityPred,
predicates.NoDiskConflictPred,
predicates.GeneralPred,
predicates.CheckNodeMemoryPressurePred,
predicates.CheckNodeDiskPressurePred,
predicates.CheckNodePIDPressurePred,
predicates.CheckNodeConditionPred,
predicates.PodToleratesNodeTaintsPred,
predicates.CheckVolumeBindingPred,
predicates.CheckNodeUnschedulablePred,
)
}

@@ -62,34 +59,6 @@ func defaultPredicates() sets.String {
// of a feature gate temporarily.
func ApplyFeatureGates() (restore func()) {
snapshot := scheduler.RegisteredPredicatesAndPrioritiesSnapshot()
if utilfeature.DefaultFeatureGate.Enabled(features.TaintNodesByCondition) {
// Remove "CheckNodeCondition", "CheckNodeMemoryPressure", "CheckNodePIDPressure"
// and "CheckNodeDiskPressure" predicates
scheduler.RemoveFitPredicate(predicates.CheckNodeConditionPred)
scheduler.RemoveFitPredicate(predicates.CheckNodeMemoryPressurePred)
scheduler.RemoveFitPredicate(predicates.CheckNodeDiskPressurePred)
scheduler.RemoveFitPredicate(predicates.CheckNodePIDPressurePred)
// Remove key "CheckNodeCondition", "CheckNodeMemoryPressure", "CheckNodePIDPressure" and "CheckNodeDiskPressure"
// from ALL algorithm provider
// The key will be removed from all providers which in algorithmProviderMap[]
// if you just want remove specific provider, call func RemovePredicateKeyFromAlgoProvider()
scheduler.RemovePredicateKeyFromAlgorithmProviderMap(predicates.CheckNodeConditionPred)
scheduler.RemovePredicateKeyFromAlgorithmProviderMap(predicates.CheckNodeMemoryPressurePred)
scheduler.RemovePredicateKeyFromAlgorithmProviderMap(predicates.CheckNodeDiskPressurePred)
scheduler.RemovePredicateKeyFromAlgorithmProviderMap(predicates.CheckNodePIDPressurePred)

// Fit is determined based on whether a pod can tolerate all of the node's taints
scheduler.RegisterMandatoryFitPredicate(predicates.PodToleratesNodeTaintsPred, predicates.PodToleratesNodeTaints)
// Fit is determined based on whether a pod can tolerate unschedulable of node
scheduler.RegisterMandatoryFitPredicate(predicates.CheckNodeUnschedulablePred, predicates.CheckNodeUnschedulablePredicate)
// Insert Key "PodToleratesNodeTaints" and "CheckNodeUnschedulable" To All Algorithm Provider
// The key will insert to all providers which in algorithmProviderMap[]
// if you just want insert to specific provider, call func InsertPredicateKeyToAlgoProvider()
scheduler.InsertPredicateKeyToAlgorithmProviderMap(predicates.PodToleratesNodeTaintsPred)
scheduler.InsertPredicateKeyToAlgorithmProviderMap(predicates.CheckNodeUnschedulablePred)

klog.Infof("TaintNodesByCondition is enabled, PodToleratesNodeTaints predicate is mandatory")
}

// Only register EvenPodsSpread predicate & priority if the feature is enabled
if utilfeature.DefaultFeatureGate.Enabled(features.EvenPodsSpread) {
@@ -78,12 +78,9 @@ func TestDefaultPredicates(t *testing.T) {
predicates.MatchInterPodAffinityPred,
predicates.NoDiskConflictPred,
predicates.GeneralPred,
predicates.CheckNodeMemoryPressurePred,
predicates.CheckNodeDiskPressurePred,
predicates.CheckNodePIDPressurePred,
predicates.CheckNodeConditionPred,
predicates.PodToleratesNodeTaintsPred,
predicates.CheckVolumeBindingPred,
predicates.CheckNodeUnschedulablePred,
)

if expected := defaultPredicates(); !result.Equal(expected) {
@@ -113,10 +113,13 @@ func init() {
scheduler.RegisterFitPredicate(predicates.CheckNodePIDPressurePred, predicates.CheckNodePIDPressurePredicate)

// Fit is determined by node conditions: not ready, network unavailable or out of disk.
scheduler.RegisterMandatoryFitPredicate(predicates.CheckNodeConditionPred, predicates.CheckNodeConditionPredicate)
scheduler.RegisterFitPredicate(predicates.CheckNodeConditionPred, predicates.CheckNodeConditionPredicate)

// Fit is determined based on whether a pod can tolerate all of the node's taints
scheduler.RegisterFitPredicate(predicates.PodToleratesNodeTaintsPred, predicates.PodToleratesNodeTaints)
scheduler.RegisterMandatoryFitPredicate(predicates.PodToleratesNodeTaintsPred, predicates.PodToleratesNodeTaints)

// Fit is determined based on whether a pod can tolerate unschedulable of node
scheduler.RegisterMandatoryFitPredicate(predicates.CheckNodeUnschedulablePred, predicates.CheckNodeUnschedulablePredicate)

// Fit is determined by volume topology requirements.
scheduler.RegisterFitPredicateFactory(
@@ -20,9 +20,6 @@ import (
"fmt"
"testing"

utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/scheduler"
)

@@ -81,8 +78,8 @@ func TestApplyFeatureGates(t *testing.T) {
t.Fatalf("Error retrieving provider: %v", err)
}

if !p.FitPredicateKeys.Has("CheckNodeCondition") {
t.Fatalf("Failed to find predicate: 'CheckNodeCondition'")
if p.FitPredicateKeys.Has("CheckNodeCondition") {
t.Fatalf("Unexpected predicate: 'CheckNodeCondition'")
}

if !p.FitPredicateKeys.Has("PodToleratesNodeTaints") {
@@ -91,9 +88,6 @@ func TestApplyFeatureGates(t *testing.T) {
})
}

// Apply features for algorithm providers.
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.TaintNodesByCondition, true)()

defer ApplyFeatureGates()()

for _, pn := range algorithmProviderNames {
@@ -80,6 +80,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
{Name: "NodeAffinity"},
{Name: "NodeResources"},
{Name: "VolumeRestrictions"},
{Name: "TaintToleration"},
},
},
},
@@ -126,6 +127,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
{Name: "NodeAffinity"},
{Name: "NodeResources"},
{Name: "VolumeRestrictions"},
{Name: "TaintToleration"},
},
},
},
@@ -182,6 +184,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
{Name: "NodeAffinity"},
{Name: "NodeResources"},
{Name: "VolumeRestrictions"},
{Name: "TaintToleration"},
{Name: "VolumeZone"},
},
"ScorePlugin": {
@@ -1203,7 +1206,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
registeredPriorities := sets.NewString(scheduler.ListRegisteredPriorityFunctions()...)
seenPredicates := sets.NewString()
seenPriorities := sets.NewString()
mandatoryPredicates := sets.NewString("CheckNodeCondition")
mandatoryPredicates := sets.NewString("CheckNodeUnschedulable")
filterToPredicateMap := map[string]string{
"TaintToleration": "PodToleratesNodeTaints",
"NodeName": "HostName",

0 comments on commit d79abe8

Please sign in to comment.
You can’t perform that action at this time.