Add kubelet awareness to taint tolerant match caculator.

pkg/api/types.go

@@ -1460,7 +1460,7 @@ type Taint struct {
 	Value string `json:"value,omitempty"`
 	// Required. The effect of the taint on pods
 	// that do not tolerate the taint.
-	// Valid effects are NoSchedule and PreferNoSchedule.
+	// Valid effects are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule.
 	Effect TaintEffect `json:"effect"`
 }
 
@@ -1476,12 +1476,11 @@ const (
 	// new pods onto the node, rather than prohibiting new pods from scheduling
 	// onto the node entirely. Enforced by the scheduler.
 	TaintEffectPreferNoSchedule TaintEffect = "PreferNoSchedule"
-	// NOT YET IMPLEMENTED. TODO: Uncomment field once it is implemented.
 	// Do not allow new pods to schedule onto the node unless they tolerate the taint,
 	// do not allow pods to start on Kubelet unless they tolerate the taint,
 	// but allow all already-running pods to continue running.
 	// Enforced by the scheduler and Kubelet.
-	// TaintEffectNoScheduleNoAdmit TaintEffect = "NoScheduleNoAdmit"
+	TaintEffectNoScheduleNoAdmit TaintEffect = "NoScheduleNoAdmit"
 	// NOT YET IMPLEMENTED. TODO: Uncomment field once it is implemented.
 	// Do not allow new pods to schedule onto the node unless they tolerate the taint,
 	// do not allow pods to start on Kubelet unless they tolerate the taint,
@@ -1504,7 +1503,7 @@ type Toleration struct {
 	// If the operator is Exists, the value should be empty, otherwise just a regular string.
 	Value string `json:"value,omitempty"`
 	// Effect indicates the taint effect to match. Empty means match all taint effects.
-	// When specified, allowed values are NoSchedule and PreferNoSchedule.
+	// When specified, allowed values are NoSchedule,NoScheduleNoAdmit and PreferNoSchedule.
 	Effect TaintEffect `json:"effect,omitempty"`
 	// TODO: For forgiveness (#1574), we'd eventually add at least a grace period
 	// here, and possibly an occurrence threshold and period.

pkg/api/v1/types.go

@@ -1679,7 +1679,7 @@ type Taint struct {
 	Value string `json:"value,omitempty" protobuf:"bytes,2,opt,name=value"`
 	// Required. The effect of the taint on pods
 	// that do not tolerate the taint.
-	// Valid effects are NoSchedule and PreferNoSchedule.
+	// Valid effects are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule.
 	Effect TaintEffect `json:"effect" protobuf:"bytes,3,opt,name=effect,casttype=TaintEffect"`
 }
 
@@ -1695,12 +1695,11 @@ const (
 	// new pods onto the node, rather than prohibiting new pods from scheduling
 	// onto the node entirely. Enforced by the scheduler.
 	TaintEffectPreferNoSchedule TaintEffect = "PreferNoSchedule"
-	// NOT YET IMPLEMENTED. TODO: Uncomment field once it is implemented.
 	// Do not allow new pods to schedule onto the node unless they tolerate the taint,
 	// do not allow pods to start on Kubelet unless they tolerate the taint,
 	// but allow all already-running pods to continue running.
 	// Enforced by the scheduler and Kubelet.
-	// TaintEffectNoScheduleNoAdmit TaintEffect = "NoScheduleNoAdmit"
+	TaintEffectNoScheduleNoAdmit TaintEffect = "NoScheduleNoAdmit"
 	// NOT YET IMPLEMENTED. TODO: Uncomment field once it is implemented.
 	// Do not allow new pods to schedule onto the node unless they tolerate the taint,
 	// do not allow pods to start on Kubelet unless they tolerate the taint,
@@ -1723,7 +1722,7 @@ type Toleration struct {
 	// If the operator is Exists, the value should be empty, otherwise just a regular string.
 	Value string `json:"value,omitempty" protobuf:"bytes,3,opt,name=value"`
 	// Effect indicates the taint effect to match. Empty means match all taint effects.
-	// When specified, allowed values are NoSchedule and PreferNoSchedule.
+	// When specified, allowed values are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule.
 	Effect TaintEffect `json:"effect,omitempty" protobuf:"bytes,4,opt,name=effect,casttype=TaintEffect"`
 	// TODO: For forgiveness (#1574), we'd eventually add at least a grace period
 	// here, and possibly an occurrence threshold and period.

pkg/api/v1/types_swagger_doc_generated.go

@@ -1718,7 +1718,7 @@ var map_Taint = map[string]string{
 	"":       "The node this Taint is attached to has the effect \"effect\" on any pod that that does not tolerate the Taint.",
 	"key":    "Required. The taint key to be applied to a node.",
 	"value":  "Required. The taint value corresponding to the taint key.",
-	"effect": "Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule and PreferNoSchedule.",
+	"effect": "Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule.",
 }
 
 func (Taint) SwaggerDoc() map[string]string {
@@ -1730,7 +1730,7 @@ var map_Toleration = map[string]string{
 	"key":      "Required. Key is the taint key that the toleration applies to.",
 	"operator": "operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.",
 	"value":    "Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string.",
-	"effect":   "Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule and PreferNoSchedule.",
+	"effect":   "Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule.",
 }
 
 func (Toleration) SwaggerDoc() map[string]string {

pkg/api/validation/validation.go

@@ -1776,14 +1776,14 @@ func validateTaintEffect(effect *api.TaintEffect, allowEmpty bool, fldPath *fiel
 	allErrors := field.ErrorList{}
 	switch *effect {
 	// TODO: Replace next line with subsequent commented-out line when implement TaintEffectNoScheduleNoAdmit, TaintEffectNoScheduleNoAdmitNoExecute.
-	case api.TaintEffectNoSchedule, api.TaintEffectPreferNoSchedule:
-		// case api.TaintEffectNoSchedule, api.TaintEffectPreferNoSchedule, api.TaintEffectNoScheduleNoAdmit, api.TaintEffectNoScheduleNoAdmitNoExecute:
+	case api.TaintEffectNoSchedule, api.TaintEffectPreferNoSchedule, api.TaintEffectNoScheduleNoAdmit:
+		// case api.TaintEffectNoSchedule, api.TaintEffectPreferNoSchedule, api.TaintEffectNoScheduleNoAdmitNoExecute:
 	default:
 		validValues := []string{
 			string(api.TaintEffectNoSchedule),
 			string(api.TaintEffectPreferNoSchedule),
+			string(api.TaintEffectNoScheduleNoAdmit),
 			// TODO: Uncomment this block when implement TaintEffectNoScheduleNoAdmit, TaintEffectNoScheduleNoAdmitNoExecute.
-			// string(api.TaintEffectNoScheduleNoAdmit),
 			// string(api.TaintEffectNoScheduleNoAdmitNoExecute),
 		}
 		allErrors = append(allErrors, field.NotSupported(fldPath, effect, validValues))

pkg/generated/openapi/zz_generated.openapi.go

@@ -12451,7 +12451,7 @@ var OpenAPIDefinitions *common.OpenAPIDefinitions = &common.OpenAPIDefinitions{
 					},
 					"effect": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule and PreferNoSchedule.",
+							Description: "Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule.",
 							Type:        []string{"string"},
 							Format:      "",
 						},
@@ -12549,7 +12549,7 @@ var OpenAPIDefinitions *common.OpenAPIDefinitions = &common.OpenAPIDefinitions{
 					},
 					"effect": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule and PreferNoSchedule.",
+							Description: "Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule.",
 							Type:        []string{"string"},
 							Format:      "",
 						},

pkg/kubectl/cmd/taint.go

@@ -181,7 +181,7 @@ func parseTaints(spec []string) ([]api.Taint, []api.Taint, error) {
 				return nil, nil, fmt.Errorf("invalid taint spec: %v, %s", taintSpec, strings.Join(errs, "; "))
 			}
 
-			if parts2[1] != string(api.TaintEffectNoSchedule) && parts2[1] != string(api.TaintEffectPreferNoSchedule) {
+			if parts2[1] != string(api.TaintEffectNoSchedule) && parts2[1] != string(api.TaintEffectPreferNoSchedule) && parts2[1] != string(api.TaintEffectNoScheduleNoAdmit) {
 				return nil, nil, fmt.Errorf("invalid taint spec: %v, unsupported taint effect", taintSpec)
 			}
 

pkg/kubelet/kubelet.go

@@ -2208,6 +2208,7 @@ func (kl *Kubelet) canAdmitPod(pods []*api.Pod, pod *api.Pod) (bool, string, str
 			return false, result.Reason, result.Message
 		}
 	}
+
 	// TODO: When disk space scheduling is implemented (#11976), remove the out-of-disk check here and
 	// add the disk space predicate to predicates.GeneralPredicates.
 	if kl.isOutOfDisk() {

pkg/kubelet/lifecycle/predicate.go

@@ -101,6 +101,53 @@ func (w *predicateAdmitHandler) Admit(attrs *PodAdmitAttributes) PodAdmitResult
 			Message: message,
 		}
 	}
+
+	// Check toleration against taints
+	// NOTE(harryz) consider move PodToleratesNodeTaints to GeneralPredicates to eliminate duplicate code here
+	fit, reasons, err = predicates.PodToleratesNodeTaints(pod, nil, nodeInfo)
+	if err != nil {
+		message := fmt.Sprintf("PodToleratesNodeTaints failed due to %v, which is unexpected.", err)
+		glog.Warningf("Failed to admit pod %v - %s", format.Pod(pod), message)
+		return PodAdmitResult{
+			Admit:   fit,
+			Reason:  "UnexpectedError",
+			Message: message,
+		}
+
+	}
+	if !fit {
+		var reason string
+		var message string
+		if len(reasons) == 0 {
+			message = fmt.Sprint("PodToleratesNodeTaints failed due to unknown reason, which is unexpected.")
+			glog.Warningf("Failed to admit pod %v - %s", format.Pod(pod), message)
+			return PodAdmitResult{
+				Admit:   fit,
+				Reason:  "UnknownReason",
+				Message: message,
+			}
+		}
+		r := reasons[0]
+		switch re := r.(type) {
+		case *predicates.ErrTaintsTolerationsNotMatch:
+			// if kubelet should not care this unfit
+			if !re.SomeUntoleratedTaintIsNoAdmit {
+				return PodAdmitResult{
+					Admit: true,
+				}
+			}
+			reason = "PodToleratesNodeTaints"
+			message = re.Error()
+			glog.Warningf("Failed to admit pod %v - %s", format.Pod(pod), message)
+		}
+
+		return PodAdmitResult{
+			Admit:   fit,
+			Reason:  reason,
+			Message: message,
+		}
+	}
+
 	return PodAdmitResult{
 		Admit: true,
 	}

plugin/pkg/scheduler/algorithm/predicates/error.go

@@ -29,7 +29,6 @@ var (
 	ErrVolumeZoneConflict        = newPredicateFailureError("NoVolumeZoneConflict")
 	ErrNodeSelectorNotMatch      = newPredicateFailureError("MatchNodeSelector")
 	ErrPodAffinityNotMatch       = newPredicateFailureError("MatchInterPodAffinity")
-	ErrTaintsTolerationsNotMatch = newPredicateFailureError("PodToleratesNodeTaints")
 	ErrPodNotMatchHostName       = newPredicateFailureError("HostName")
 	ErrPodNotFitsHostPorts       = newPredicateFailureError("PodFitsHostPorts")
 	ErrNodeLabelPresenceViolated = newPredicateFailureError("CheckNodeLabelPresence")
@@ -42,6 +41,25 @@ var (
 	ErrFakePredicate = newPredicateFailureError("FakePredicateError")
 )
 
+// ErrTaintsTolerationsNotMatch is an error type that indicates with if it should be aware by kubelet.
+type ErrTaintsTolerationsNotMatch struct {
+	SomeUntoleratedTaintIsNoAdmit bool
+}
+
+func newErrTaintsTolerationsNotMatch(someUntoleratedTaintIsNoAdmit bool) *ErrTaintsTolerationsNotMatch {
+	return &ErrTaintsTolerationsNotMatch{
+		SomeUntoleratedTaintIsNoAdmit: someUntoleratedTaintIsNoAdmit,
+	}
+}
+
+func (e *ErrTaintsTolerationsNotMatch) Error() string {
+	return fmt.Sprintf("Taint Toleration unmatched with SomeUntoleratedTaintIsNoAdmit is: %v", e.SomeUntoleratedTaintIsNoAdmit)
+}
+
+func (e *ErrTaintsTolerationsNotMatch) GetReason() string {
+	return fmt.Sprintf("ErrTaintsTolerationsNotMatch, and SomeUntoleratedTaintIsNoAdmit is: %v", e.SomeUntoleratedTaintIsNoAdmit)
+}
+
 // InsufficientResourceError is an error type that indicates what kind of resource limit is
 // hit and caused the unfitting failure.
 type InsufficientResourceError struct {

plugin/pkg/scheduler/algorithm/predicates/predicates.go

@@ -1083,36 +1083,54 @@ func PodToleratesNodeTaints(pod *api.Pod, meta interface{}, nodeInfo *schedulerc
 		return false, nil, err
 	}
 
-	if tolerationsToleratesTaints(tolerations, taints) {
+	if tolerated, someUntoleratedTaintIsNoAdmit := tolerationsToleratesTaints(tolerations, taints); tolerated {
 		return true, nil, nil
+	} else {
+		return false, []algorithm.PredicateFailureReason{newErrTaintsTolerationsNotMatch(someUntoleratedTaintIsNoAdmit)}, nil
 	}
-	return false, []algorithm.PredicateFailureReason{ErrTaintsTolerationsNotMatch}, nil
 }
 
-func tolerationsToleratesTaints(tolerations []api.Toleration, taints []api.Taint) bool {
+// tolerationsToleratesTaints checks if given tolerations can live with given taints.
+// It returns:
+// 1. whether tolerated or not;
+// 2. whether kubelet should be aware if it's unfit.
+func tolerationsToleratesTaints(tolerations []api.Toleration, taints []api.Taint) (bool, bool) {
 	// If the taint list is nil/empty, it is tolerated by all tolerations by default.
 	if len(taints) == 0 {
-		return true
+		return true, false
 	}
 
 	// The taint list isn't nil/empty, a nil/empty toleration list can't tolerate them.
 	if len(tolerations) == 0 {
-		return false
+		// if there's taint has TaintEffectNoScheduleNoAdmit, kubelet should also be aware of this.
+		for _, taint := range taints {
+			if taint.Effect == api.TaintEffectNoScheduleNoAdmit {
+				return false, true
+			}
+		}
+		return false, false
 	}
 
+	someUntoleratedTaintIsNoAdmit := false
+	fits := true
+
 	for i := range taints {
 		taint := &taints[i]
 		// skip taints that have effect PreferNoSchedule, since it is for priorities
 		if taint.Effect == api.TaintEffectPreferNoSchedule {
 			continue
 		}
 
-		if !api.TaintToleratedByTolerations(taint, tolerations) {
-			return false
+		if tolerated := api.TaintToleratedByTolerations(taint, tolerations); !tolerated {
+			fits = false
+			if taint.Effect == api.TaintEffectNoScheduleNoAdmit {
+				someUntoleratedTaintIsNoAdmit = true
+				return fits, someUntoleratedTaintIsNoAdmit
+			}
 		}
 	}
 
-	return true
+	return fits, someUntoleratedTaintIsNoAdmit
 }
 
 // Determine if a pod is scheduled with best-effort QoS