Merge pull request #83998 from draveness/feature/node-affinity-score-…

…plugin feat(scheduler): implement node affinity as score plugin
kubernetes · Oct 17, 2019 · fef8192 · fef8192
2 parents 534051a + 3d74da4
commit fef8192
Show file tree

Hide file tree

Showing 6 changed files with 225 additions and 18 deletions.
diff --git a/pkg/scheduler/api/compatibility/compatibility_test.go b/pkg/scheduler/api/compatibility/compatibility_test.go
@@ -168,7 +168,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 			),
 			wantPrioritizers: sets.NewString(
 				"EqualPriority",
-				"NodeAffinityPriority",
 				"LeastRequestedPriority",
 				"BalancedResourceAllocation",
 				"SelectorSpreadPriority",
@@ -186,6 +185,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 2},
+					{Name: "NodeAffinity", Weight: 2},
 				},
 			},
 		},
@@ -238,7 +238,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"LeastRequestedPriority",
 				"BalancedResourceAllocation",
 				"SelectorSpreadPriority",
-				"NodeAffinityPriority",
 				"InterPodAffinityPriority",
 			),
 			wantPlugins: map[string][]kubeschedulerconfig.Plugin{
@@ -253,6 +252,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 2},
+					{Name: "NodeAffinity", Weight: 2},
 					{Name: "TaintToleration", Weight: 2},
 				},
 			},
@@ -310,7 +310,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"LeastRequestedPriority",
 				"BalancedResourceAllocation",
 				"SelectorSpreadPriority",
-				"NodeAffinityPriority",
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
 			),
@@ -326,6 +325,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 2},
+					{Name: "NodeAffinity", Weight: 2},
 					{Name: "NodePreferAvoidPods", Weight: 2},
 					{Name: "TaintToleration", Weight: 2},
 				},
@@ -393,7 +393,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"LeastRequestedPriority",
 				"BalancedResourceAllocation",
 				"SelectorSpreadPriority",
-				"NodeAffinityPriority",
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
 			),
@@ -409,6 +408,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 2},
+					{Name: "NodeAffinity", Weight: 2},
 					{Name: "NodePreferAvoidPods", Weight: 2},
 					{Name: "TaintToleration", Weight: 2},
 				},
@@ -489,7 +489,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"LeastRequestedPriority",
 				"BalancedResourceAllocation",
 				"SelectorSpreadPriority",
-				"NodeAffinityPriority",
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
 			),
@@ -505,6 +504,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 2},
+					{Name: "NodeAffinity", Weight: 2},
 					{Name: "NodePreferAvoidPods", Weight: 2},
 					{Name: "TaintToleration", Weight: 2},
 				},
@@ -586,7 +586,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"LeastRequestedPriority",
 				"BalancedResourceAllocation",
 				"SelectorSpreadPriority",
-				"NodeAffinityPriority",
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
 			),
@@ -603,6 +602,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 2},
+					{Name: "NodeAffinity", Weight: 2},
 					{Name: "NodePreferAvoidPods", Weight: 2},
 					{Name: "TaintToleration", Weight: 2},
 				},
@@ -689,7 +689,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"LeastRequestedPriority",
 				"BalancedResourceAllocation",
 				"SelectorSpreadPriority",
-				"NodeAffinityPriority",
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
 			),
@@ -706,6 +705,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 2},
+					{Name: "NodeAffinity", Weight: 2},
 					{Name: "NodePreferAvoidPods", Weight: 2},
 					{Name: "TaintToleration", Weight: 2},
 				},
@@ -804,7 +804,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"LeastRequestedPriority",
 				"BalancedResourceAllocation",
 				"SelectorSpreadPriority",
-				"NodeAffinityPriority",
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
 				"RequestedToCapacityRatioPriority",
@@ -822,6 +821,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 2},
+					{Name: "NodeAffinity", Weight: 2},
 					{Name: "NodePreferAvoidPods", Weight: 2},
 					{Name: "TaintToleration", Weight: 2},
 				},
@@ -922,7 +922,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"LeastRequestedPriority",
 				"BalancedResourceAllocation",
 				"SelectorSpreadPriority",
-				"NodeAffinityPriority",
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
 				"RequestedToCapacityRatioPriority",
@@ -940,6 +939,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 2},
+					{Name: "NodeAffinity", Weight: 2},
 					{Name: "NodePreferAvoidPods", Weight: 2},
 					{Name: "TaintToleration", Weight: 2},
 				},
@@ -1040,7 +1040,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"LeastRequestedPriority",
 				"BalancedResourceAllocation",
 				"SelectorSpreadPriority",
-				"NodeAffinityPriority",
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
 				"RequestedToCapacityRatioPriority",
@@ -1058,6 +1057,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 2},
+					{Name: "NodeAffinity", Weight: 2},
 					{Name: "NodePreferAvoidPods", Weight: 2},
 					{Name: "TaintToleration", Weight: 2},
 				},
@@ -1162,7 +1162,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"LeastRequestedPriority",
 				"BalancedResourceAllocation",
 				"SelectorSpreadPriority",
-				"NodeAffinityPriority",
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
 				"RequestedToCapacityRatioPriority",
@@ -1180,6 +1179,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 2},
+					{Name: "NodeAffinity", Weight: 2},
 					{Name: "NodePreferAvoidPods", Weight: 2},
 					{Name: "TaintToleration", Weight: 2},
 				},
@@ -1216,6 +1216,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 	}
 	scoreToPriorityMap := map[string]string{
 		"ImageLocality":       "ImageLocalityPriority",
+		"NodeAffinity":        "NodeAffinityPriority",
 		"NodePreferAvoidPods": "NodePreferAvoidPodsPriority",
 		"TaintToleration":     "TaintTolerationPriority",
 	}

diff --git a/pkg/scheduler/framework/plugins/default_registry.go b/pkg/scheduler/framework/plugins/default_registry.go
@@ -152,6 +152,12 @@ func NewDefaultConfigProducerRegistry() *ConfigProducerRegistry {
 			return
 		})
 
+	registry.RegisterPriority(priorities.NodeAffinityPriority,
+		func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+			plugins.Score = appendToPluginSet(plugins.Score, nodeaffinity.Name, &args.Weight)
+			return
+		})
+
 	registry.RegisterPriority(priorities.ImageLocalityPriority,
 		func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
 			plugins.Score = appendToPluginSet(plugins.Score, imagelocality.Name, &args.Weight)

diff --git a/pkg/scheduler/framework/plugins/nodeaffinity/BUILD b/pkg/scheduler/framework/plugins/nodeaffinity/BUILD
@@ -7,6 +7,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//pkg/scheduler/algorithm/predicates:go_default_library",
+        "//pkg/scheduler/algorithm/priorities:go_default_library",
         "//pkg/scheduler/framework/plugins/migration:go_default_library",
         "//pkg/scheduler/framework/v1alpha1:go_default_library",
         "//pkg/scheduler/nodeinfo:go_default_library",

diff --git a/pkg/scheduler/framework/plugins/nodeaffinity/node_affinity.go b/pkg/scheduler/framework/plugins/nodeaffinity/node_affinity.go
@@ -18,19 +18,24 @@ package nodeaffinity
 
 import (
 	"context"
+	"fmt"
 
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/kubernetes/pkg/scheduler/algorithm/predicates"
+	"k8s.io/kubernetes/pkg/scheduler/algorithm/priorities"
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/migration"
 	framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
 	"k8s.io/kubernetes/pkg/scheduler/nodeinfo"
 )
 
 // NodeAffinity is a plugin that checks if a pod node selector matches the node label.
-type NodeAffinity struct{}
+type NodeAffinity struct {
+	handle framework.FrameworkHandle
+}
 
-var _ = framework.FilterPlugin(&NodeAffinity{})
+var _ framework.FilterPlugin = &NodeAffinity{}
+var _ framework.ScorePlugin = &NodeAffinity{}
 
 // Name is the name of the plugin used in the plugin registry and configurations.
 const Name = "NodeAffinity"
@@ -41,12 +46,36 @@ func (pl *NodeAffinity) Name() string {
 }
 
 // Filter invoked at the filter extension point.
-func (pl *NodeAffinity) Filter(ctx context.Context, _ *framework.CycleState, pod *v1.Pod, nodeInfo *nodeinfo.NodeInfo) *framework.Status {
+func (pl *NodeAffinity) Filter(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeInfo *nodeinfo.NodeInfo) *framework.Status {
 	_, reasons, err := predicates.PodMatchNodeSelector(pod, nil, nodeInfo)
 	return migration.PredicateResultToFrameworkStatus(reasons, err)
 }
 
+// Score invoked at the Score extension point.
+func (pl *NodeAffinity) Score(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeName string) (int64, *framework.Status) {
+	nodeInfo, exist := pl.handle.NodeInfoSnapshot().NodeInfoMap[nodeName]
+	if !exist {
+		return 0, framework.NewStatus(framework.Error, fmt.Sprintf("node %q does not exist in NodeInfoSnapshot", nodeName))
+	}
+
+	meta := migration.PriorityMetadata(state)
+	s, err := priorities.CalculateNodeAffinityPriorityMap(pod, meta, nodeInfo)
+	return s.Score, migration.ErrorToFrameworkStatus(err)
+}
+
+// NormalizeScore invoked after scoring all nodes.
+func (pl *NodeAffinity) NormalizeScore(ctx context.Context, state *framework.CycleState, pod *v1.Pod, scores framework.NodeScoreList) *framework.Status {
+	// Note that CalculateNodeAffinityPriorityReduce doesn't use priority metadata, hence passing nil here.
+	err := priorities.CalculateNodeAffinityPriorityReduce(pod, nil, pl.handle.NodeInfoSnapshot().NodeInfoMap, scores)
+	return migration.ErrorToFrameworkStatus(err)
+}
+
+// ScoreExtensions of the Score plugin.
+func (pl *NodeAffinity) ScoreExtensions() framework.ScoreExtensions {
+	return pl
+}
+
 // New initializes a new plugin and returns it.
-func New(_ *runtime.Unknown, _ framework.FrameworkHandle) (framework.Plugin, error) {
-	return &NodeAffinity{}, nil
+func New(_ *runtime.Unknown, h framework.FrameworkHandle) (framework.Plugin, error) {
+	return &NodeAffinity{handle: h}, nil
 }