Skip to content
Permalink
Browse files

feat: remove TaintNodesByCondition feature gate

  • Loading branch information
draveness committed Sep 15, 2019
1 parent 9e75714 commit 32e67c2e90fd5f25227992a421949001aa6f8fae
@@ -178,7 +178,6 @@ func startNodeLifecycleController(ctx ControllerContext) (http.Handler, bool, er
ctx.ComponentConfig.NodeLifecycleController.UnhealthyZoneThreshold,
ctx.ComponentConfig.NodeLifecycleController.EnableTaintManager,
utilfeature.DefaultFeatureGate.Enabled(features.TaintBasedEvictions),
utilfeature.DefaultFeatureGate.Enabled(features.TaintNodesByCondition),
)
if err != nil {
return nil, true, err
@@ -26,7 +26,7 @@ import (
"time"

apps "k8s.io/api/apps/v1"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
@@ -1804,8 +1804,6 @@ func TestTaintPressureNodeDaemonLaunchesPod(t *testing.T) {
}
manager.nodeStore.Add(node)

// Enabling critical pod and taint nodes by condition feature gate should create critical pod
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.TaintNodesByCondition, true)()
manager.dsStore.Add(ds)
syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0)
}
@@ -294,10 +294,6 @@ type Controller struct {
// taints instead of evicting Pods itself.
useTaintBasedEvictions bool

// if set to true, NodeController will taint Nodes based on its condition for 'NetworkUnavailable',
// 'MemoryPressure', 'PIDPressure' and 'DiskPressure'.
taintNodeByCondition bool

nodeUpdateQueue workqueue.Interface
}

@@ -318,7 +314,7 @@ func NewNodeLifecycleController(
unhealthyZoneThreshold float32,
runTaintManager bool,
useTaintBasedEvictions bool,
taintNodeByCondition bool) (*Controller, error) {
) (*Controller, error) {

if kubeClient == nil {
klog.Fatalf("kubeClient is nil when starting Controller")
@@ -357,7 +353,6 @@ func NewNodeLifecycleController(
unhealthyZoneThreshold: unhealthyZoneThreshold,
runTaintManager: runTaintManager,
useTaintBasedEvictions: useTaintBasedEvictions && runTaintManager,
taintNodeByCondition: taintNodeByCondition,
nodeUpdateQueue: workqueue.NewNamed("node_lifecycle_controller"),
}
if useTaintBasedEvictions {
@@ -467,9 +462,7 @@ func NewNodeLifecycleController(
}),
})

if nc.taintNodeByCondition {
klog.Infof("Controller will taint node by condition.")
}
klog.Infof("Controller will taint node by condition.")

nc.leaseLister = leaseInformer.Lister()
if utilfeature.DefaultFeatureGate.Enabled(features.NodeLease) {
@@ -545,11 +538,9 @@ func (nc *Controller) doNodeProcessingPassWorker() {
return
}
nodeName := obj.(string)
if nc.taintNodeByCondition {
if err := nc.doNoScheduleTaintingPass(nodeName); err != nil {
klog.Errorf("Failed to taint NoSchedule on node <%s>, requeue it: %v", nodeName, err)
// TODO(k82cn): Add nodeName back to the queue
}
if err := nc.doNoScheduleTaintingPass(nodeName); err != nil {
klog.Errorf("Failed to taint NoSchedule on node <%s>, requeue it: %v", nodeName, err)
// TODO(k82cn): Add nodeName back to the queue
}
// TODO: re-evaluate whether there are any labels that need to be
// reconcile in 1.19. Remove this function if it's no longer necessary.
@@ -157,7 +157,6 @@ func newNodeLifecycleControllerFromClient(
unhealthyZoneThreshold,
useTaints,
useTaints,
useTaints,
)
if err != nil {
return nil, err
@@ -133,14 +133,6 @@ const (
// Add priority to pods. Priority affects scheduling and preemption of pods.
PodPriority featuregate.Feature = "PodPriority"

// owner: @k82cn
// beta: v1.12
// GA: v1.17
//
// Taint nodes based on their condition status for 'NetworkUnavailable',
// 'MemoryPressure', 'PIDPressure' and 'DiskPressure'.
TaintNodesByCondition featuregate.Feature = "TaintNodesByCondition"

// owner: @sjenning
// alpha: v1.11
//
@@ -527,7 +519,6 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
EphemeralContainers: {Default: false, PreRelease: featuregate.Alpha},
PodShareProcessNamespace: {Default: true, PreRelease: featuregate.Beta},
PodPriority: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.18
TaintNodesByCondition: {Default: true, PreRelease: featuregate.GA},
QOSReserved: {Default: false, PreRelease: featuregate.Alpha},
ExpandPersistentVolumes: {Default: true, PreRelease: featuregate.Beta},
ExpandInUsePersistentVolumes: {Default: true, PreRelease: featuregate.Beta},
@@ -142,12 +142,9 @@ func DefaultOffAdmissionPlugins() sets.String {
resourcequota.PluginName, //ResourceQuota
storageobjectinuseprotection.PluginName, //StorageObjectInUseProtection
podpriority.PluginName, //PodPriority
nodetaint.PluginName, //TaintNodesByCondition
)

if utilfeature.DefaultFeatureGate.Enabled(features.TaintNodesByCondition) {
defaultOnPlugins.Insert(nodetaint.PluginName) //TaintNodesByCondition
}

if utilfeature.DefaultFeatureGate.Enabled(features.RuntimeClass) {
defaultOnPlugins.Insert(runtimeclass.PluginName) //RuntimeClass
}
@@ -144,13 +144,12 @@ func (m *managerImpl) Admit(attrs *lifecycle.PodAdmitAttributes) lifecycle.PodAd
return lifecycle.PodAdmitResult{Admit: true}
}

// When node has memory pressure and TaintNodesByCondition is enabled, check BestEffort Pod's toleration:
// When node has memory pressure, check BestEffort Pod's toleration:
// admit it if tolerates memory pressure taint, fail for other tolerations, e.g. DiskPressure.
if utilfeature.DefaultFeatureGate.Enabled(features.TaintNodesByCondition) &&
v1helper.TolerationsTolerateTaint(attrs.Pod.Spec.Tolerations, &v1.Taint{
Key: schedulerapi.TaintNodeMemoryPressure,
Effect: v1.TaintEffectNoSchedule,
}) {
if v1helper.TolerationsTolerateTaint(attrs.Pod.Spec.Tolerations, &v1.Taint{
Key: schedulerapi.TaintNodeMemoryPressure,
Effect: v1.TaintEffectNoSchedule,
}) {
return lifecycle.PodAdmitResult{Admit: true}
}
}
@@ -25,7 +25,7 @@ import (
"strings"
"time"

"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
apiequality "k8s.io/apimachinery/pkg/api/equality"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
@@ -246,13 +246,11 @@ func (kl *Kubelet) initialNode() (*v1.Node, error) {
Effect: v1.TaintEffectNoSchedule,
}

// If TaintNodesByCondition enabled, taint node with TaintNodeUnschedulable when initializing
// Taint node with TaintNodeUnschedulable when initializing
// node to avoid race condition; refer to #63897 for more detail.
if utilfeature.DefaultFeatureGate.Enabled(features.TaintNodesByCondition) {
if node.Spec.Unschedulable &&
!taintutil.TaintExists(nodeTaints, &unschedulableTaint) {
nodeTaints = append(nodeTaints, unschedulableTaint)
}
if node.Spec.Unschedulable &&
!taintutil.TaintExists(nodeTaints, &unschedulableTaint) {
nodeTaints = append(nodeTaints, unschedulableTaint)
}

if kl.externalCloudProvider {
@@ -48,7 +48,6 @@ import (
"k8s.io/client-go/kubernetes/fake"
"k8s.io/client-go/rest"
core "k8s.io/client-go/testing"
"k8s.io/component-base/featuregate"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/kubernetes/pkg/features"
kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
@@ -1983,25 +1982,23 @@ func TestRegisterWithApiServerWithTaint(t *testing.T) {
// Make node to be unschedulable.
kubelet.registerSchedulable = false

forEachFeatureGate(t, []featuregate.Feature{features.TaintNodesByCondition}, func(t *testing.T) {
// Reset kubelet status for each test.
kubelet.registrationCompleted = false
// Reset kubelet status for each test.
kubelet.registrationCompleted = false

// Register node to apiserver.
kubelet.registerWithAPIServer()
// Register node to apiserver.
kubelet.registerWithAPIServer()

// Check the unschedulable taint.
got := gotNode.(*v1.Node)
unschedulableTaint := &v1.Taint{
Key: schedulerapi.TaintNodeUnschedulable,
Effect: v1.TaintEffectNoSchedule,
}
// Check the unschedulable taint.
got := gotNode.(*v1.Node)
unschedulableTaint := &v1.Taint{
Key: schedulerapi.TaintNodeUnschedulable,
Effect: v1.TaintEffectNoSchedule,
}

require.Equal(t,
utilfeature.DefaultFeatureGate.Enabled(features.TaintNodesByCondition),
taintutil.TaintExists(got.Spec.Taints, unschedulableTaint),
"test unschedulable taint for TaintNodesByCondition")
})
require.Equal(t,
true,
taintutil.TaintExists(got.Spec.Taints, unschedulableTaint),
"test unschedulable taint for TaintNodesByCondition")
}

func TestNodeStatusHasChanged(t *testing.T) {
@@ -35,12 +35,9 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/kubernetes/fake"
"k8s.io/client-go/tools/record"
"k8s.io/client-go/util/flowcontrol"
"k8s.io/component-base/featuregate"
featuregatetesting "k8s.io/component-base/featuregate/testing"
cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
"k8s.io/kubernetes/pkg/kubelet/cm"
"k8s.io/kubernetes/pkg/kubelet/config"
@@ -2031,17 +2028,6 @@ func runVolumeManager(kubelet *Kubelet) chan struct{} {
return stopCh
}

func forEachFeatureGate(t *testing.T, fs []featuregate.Feature, tf func(t *testing.T)) {
for _, fg := range fs {
for _, f := range []bool{true, false} {
func() {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, fg, f)()
t.Run(fmt.Sprintf("%v(%t)", fg, f), tf)
}()
}
}
}

// Sort pods by UID.
type podsByUID []*v1.Pod

@@ -17,12 +17,7 @@ go_test(
name = "go_default_test",
srcs = ["plugins_test.go"],
embed = [":go_default_library"],
deps = [
"//pkg/features:go_default_library",
"//pkg/scheduler/factory:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//staging/src/k8s.io/component-base/featuregate/testing:go_default_library",
],
deps = ["//pkg/scheduler/factory:go_default_library"],
)

filegroup(
@@ -62,34 +62,30 @@ func defaultPredicates() sets.String {
// of a feature gate temporarily.
func ApplyFeatureGates() (restore func()) {
snapshot := factory.Copy()
if utilfeature.DefaultFeatureGate.Enabled(features.TaintNodesByCondition) {
// Remove "CheckNodeCondition", "CheckNodeMemoryPressure", "CheckNodePIDPressure"
// and "CheckNodeDiskPressure" predicates
factory.RemoveFitPredicate(predicates.CheckNodeConditionPred)
factory.RemoveFitPredicate(predicates.CheckNodeMemoryPressurePred)
factory.RemoveFitPredicate(predicates.CheckNodeDiskPressurePred)
factory.RemoveFitPredicate(predicates.CheckNodePIDPressurePred)
// Remove key "CheckNodeCondition", "CheckNodeMemoryPressure", "CheckNodePIDPressure" and "CheckNodeDiskPressure"
// from ALL algorithm provider
// The key will be removed from all providers which in algorithmProviderMap[]
// if you just want remove specific provider, call func RemovePredicateKeyFromAlgoProvider()
factory.RemovePredicateKeyFromAlgorithmProviderMap(predicates.CheckNodeConditionPred)
factory.RemovePredicateKeyFromAlgorithmProviderMap(predicates.CheckNodeMemoryPressurePred)
factory.RemovePredicateKeyFromAlgorithmProviderMap(predicates.CheckNodeDiskPressurePred)
factory.RemovePredicateKeyFromAlgorithmProviderMap(predicates.CheckNodePIDPressurePred)

// Fit is determined based on whether a pod can tolerate all of the node's taints
factory.RegisterMandatoryFitPredicate(predicates.PodToleratesNodeTaintsPred, predicates.PodToleratesNodeTaints)
// Fit is determined based on whether a pod can tolerate unschedulable of node
factory.RegisterMandatoryFitPredicate(predicates.CheckNodeUnschedulablePred, predicates.CheckNodeUnschedulablePredicate)
// Insert Key "PodToleratesNodeTaints" and "CheckNodeUnschedulable" To All Algorithm Provider
// The key will insert to all providers which in algorithmProviderMap[]
// if you just want insert to specific provider, call func InsertPredicateKeyToAlgoProvider()
factory.InsertPredicateKeyToAlgorithmProviderMap(predicates.PodToleratesNodeTaintsPred)
factory.InsertPredicateKeyToAlgorithmProviderMap(predicates.CheckNodeUnschedulablePred)

klog.Infof("TaintNodesByCondition is enabled, PodToleratesNodeTaints predicate is mandatory")
}
// Remove "CheckNodeCondition", "CheckNodeMemoryPressure", "CheckNodePIDPressure"
// and "CheckNodeDiskPressure" predicates
factory.RemoveFitPredicate(predicates.CheckNodeConditionPred)
factory.RemoveFitPredicate(predicates.CheckNodeMemoryPressurePred)
factory.RemoveFitPredicate(predicates.CheckNodeDiskPressurePred)
factory.RemoveFitPredicate(predicates.CheckNodePIDPressurePred)
// Remove key "CheckNodeCondition", "CheckNodeMemoryPressure", "CheckNodePIDPressure" and "CheckNodeDiskPressure"
// from ALL algorithm provider
// The key will be removed from all providers which in algorithmProviderMap[]
// if you just want remove specific provider, call func RemovePredicateKeyFromAlgoProvider()
factory.RemovePredicateKeyFromAlgorithmProviderMap(predicates.CheckNodeConditionPred)
factory.RemovePredicateKeyFromAlgorithmProviderMap(predicates.CheckNodeMemoryPressurePred)
factory.RemovePredicateKeyFromAlgorithmProviderMap(predicates.CheckNodeDiskPressurePred)
factory.RemovePredicateKeyFromAlgorithmProviderMap(predicates.CheckNodePIDPressurePred)

// Fit is determined based on whether a pod can tolerate all of the node's taints
factory.RegisterMandatoryFitPredicate(predicates.PodToleratesNodeTaintsPred, predicates.PodToleratesNodeTaints)
// Fit is determined based on whether a pod can tolerate unschedulable of node
factory.RegisterMandatoryFitPredicate(predicates.CheckNodeUnschedulablePred, predicates.CheckNodeUnschedulablePredicate)
// Insert Key "PodToleratesNodeTaints" and "CheckNodeUnschedulable" To All Algorithm Provider
// The key will insert to all providers which in algorithmProviderMap[]
// if you just want insert to specific provider, call func InsertPredicateKeyToAlgoProvider()
factory.InsertPredicateKeyToAlgorithmProviderMap(predicates.PodToleratesNodeTaintsPred)
factory.InsertPredicateKeyToAlgorithmProviderMap(predicates.CheckNodeUnschedulablePred)

// Only register EvenPodsSpread predicate & priority if the feature is enabled
if utilfeature.DefaultFeatureGate.Enabled(features.EvenPodsSpread) {
@@ -20,9 +20,6 @@ import (
"fmt"
"testing"

utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/scheduler/factory"
)

@@ -91,9 +88,6 @@ func TestApplyFeatureGates(t *testing.T) {
})
}

// Apply features for algorithm providers.
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.TaintNodesByCondition, true)()

defer ApplyFeatureGates()()

for _, pn := range algorithmProviderNames {
@@ -7,10 +7,7 @@ go_library(
visibility = ["//visibility:public"],
deps = [
"//pkg/apis/core:go_default_library",
"//pkg/features:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/admission:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//staging/src/k8s.io/component-base/featuregate:go_default_library",
],
)

@@ -20,12 +17,10 @@ go_test(
embed = [":go_default_library"],
deps = [
"//pkg/apis/core:go_default_library",
"//pkg/features:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/admission:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library",
"//staging/src/k8s.io/component-base/featuregate:go_default_library",
],
)

0 comments on commit 32e67c2

Please sign in to comment.
You can’t perform that action at this time.