koord-descheduler: LowNodeLoad supports configuring resourceWeights (#…

…1240)

Signed-off-by: Joseph <joseph.t.lee@outlook.com>

pkg/descheduler/apis/config/types_loadaware.go

@@ -17,6 +17,7 @@ limitations under the License.
 package config
 
 import (
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -65,6 +66,10 @@ type LowNodeLoadArgs struct {
 	// LowThresholds defines the low usage threshold of resources
 	LowThresholds ResourceThresholds
 
+	// ResourceWeights indicates the weights of resources.
+	// The weights of resources are both 1 by default.
+	ResourceWeights map[corev1.ResourceName]int64
+
 	// AnomalyCondition indicates the node load anomaly thresholds,
 	// the default is 5 consecutive times exceeding HighThresholds,
 	// it is determined that the node is abnormal, and the Pods need to be migrated to reduce the load.

pkg/descheduler/apis/config/v1alpha2/defaults.go

@@ -21,6 +21,7 @@ import (
 	"strconv"
 	"time"
 
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -246,4 +247,24 @@ func SetDefaults_LowNodeLoadArgs(obj *LowNodeLoadArgs) {
 	} else if obj.AnomalyCondition.ConsecutiveAbnormalities == 0 {
 		obj.AnomalyCondition.ConsecutiveAbnormalities = defaultLoadAnomalyCondition.ConsecutiveAbnormalities
 	}
+
+	defaultResourceWeights := map[corev1.ResourceName]int64{
+		corev1.ResourceCPU:    1,
+		corev1.ResourceMemory: 1,
+	}
+	for resourceName := range obj.LowThresholds {
+		defaultResourceWeights[resourceName] = 1
+	}
+	for resourceName := range obj.HighThresholds {
+		defaultResourceWeights[resourceName] = 1
+	}
+	if obj.ResourceWeights == nil {
+		obj.ResourceWeights = defaultResourceWeights
+	} else {
+		for resourceName, weight := range defaultResourceWeights {
+			if v := obj.ResourceWeights[resourceName]; v <= 0 {
+				obj.ResourceWeights[resourceName] = weight
+			}
+		}
+	}
 }

pkg/descheduler/apis/config/v1alpha2/defaults_test.go

@@ -0,0 +1,106 @@
+/*
+Copyright 2022 The Koordinator Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha2
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/pointer"
+)
+
+func TestSetDefaults_LowNodeLoadArgs(t *testing.T) {
+	tests := []struct {
+		name     string
+		args     *LowNodeLoadArgs
+		expected *LowNodeLoadArgs
+	}{
+		{
+			name: "set nodeFit",
+			args: &LowNodeLoadArgs{
+				NodeFit: pointer.Bool(false),
+			},
+			expected: &LowNodeLoadArgs{
+				NodeFit:          pointer.Bool(false),
+				AnomalyCondition: defaultLoadAnomalyCondition,
+				ResourceWeights: map[corev1.ResourceName]int64{
+					corev1.ResourceCPU:    1,
+					corev1.ResourceMemory: 1,
+				},
+			},
+		},
+		{
+			name: "set anomalyCondition",
+			args: &LowNodeLoadArgs{
+				AnomalyCondition: &LoadAnomalyCondition{
+					Timeout:                  &metav1.Duration{Duration: 10 * time.Second},
+					ConsecutiveAbnormalities: 0,
+					ConsecutiveNormalities:   3,
+				},
+			},
+			expected: &LowNodeLoadArgs{
+				NodeFit: pointer.Bool(true),
+				AnomalyCondition: &LoadAnomalyCondition{
+					Timeout:                  &metav1.Duration{Duration: 10 * time.Second},
+					ConsecutiveAbnormalities: defaultLoadAnomalyCondition.ConsecutiveAbnormalities,
+					ConsecutiveNormalities:   3,
+				},
+				ResourceWeights: map[corev1.ResourceName]int64{
+					corev1.ResourceCPU:    1,
+					corev1.ResourceMemory: 1,
+				},
+			},
+		},
+		{
+			name: "set weights",
+			args: &LowNodeLoadArgs{
+				LowThresholds: ResourceThresholds{
+					corev1.ResourceCPU:    30,
+					corev1.ResourceMemory: 30,
+					corev1.ResourcePods:   10,
+				},
+				ResourceWeights: map[corev1.ResourceName]int64{
+					corev1.ResourceCPU:    10,
+					corev1.ResourceMemory: 5,
+				},
+			},
+			expected: &LowNodeLoadArgs{
+				NodeFit:          pointer.Bool(true),
+				AnomalyCondition: defaultLoadAnomalyCondition,
+				LowThresholds: ResourceThresholds{
+					corev1.ResourceCPU:    30,
+					corev1.ResourceMemory: 30,
+					corev1.ResourcePods:   10,
+				},
+				ResourceWeights: map[corev1.ResourceName]int64{
+					corev1.ResourceCPU:    10,
+					corev1.ResourceMemory: 5,
+					corev1.ResourcePods:   1,
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			SetDefaults_LowNodeLoadArgs(tt.args)
+			assert.Equal(t, tt.expected, tt.args)
+		})
+	}
+}

pkg/descheduler/apis/config/v1alpha2/types_loadaware.go

@@ -17,6 +17,7 @@ limitations under the License.
 package v1alpha2
 
 import (
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -64,6 +65,10 @@ type LowNodeLoadArgs struct {
 	// LowThresholds defines the low usage threshold of resources
 	LowThresholds ResourceThresholds `json:"lowThresholds,omitempty"`
 
+	// ResourceWeights indicates the weights of resources.
+	// The weights of CPU and Memory are both 1 by default.
+	ResourceWeights map[corev1.ResourceName]int64 `json:"resourceWeights,omitempty"`
+
 	// AnomalyCondition indicates the node load anomaly thresholds,
 	// the default is 5 consecutive times exceeding HighThresholds,
 	// it is determined that the node is abnormal, and the Pods need to be migrated to reduce the load.

pkg/descheduler/framework/plugins/loadaware/low_node_load.go

@@ -41,7 +41,6 @@ import (
 	nodeutil "github.com/koordinator-sh/koordinator/pkg/descheduler/node"
 	podutil "github.com/koordinator-sh/koordinator/pkg/descheduler/pod"
 	"github.com/koordinator-sh/koordinator/pkg/descheduler/utils/anomaly"
-	"github.com/koordinator-sh/koordinator/pkg/descheduler/utils/sorter"
 )
 
 const (
@@ -199,15 +198,15 @@ func (pl *LowNodeLoad) Balance(ctx context.Context, nodes []*corev1.Node) *frame
 		return true
 	}
 
-	resourceToWeightMap := sorter.GenDefaultResourceToWeightMap(resourceNames)
-	sortNodesByUsage(abnormalNodes, resourceToWeightMap, false)
+	sortNodesByUsage(abnormalNodes, pl.args.ResourceWeights, false)
 
 	evictPodsFromSourceNodes(
 		ctx,
 		abnormalNodes,
 		lowNodes,
 		pl.args.DryRun,
 		pl.args.NodeFit,
+		pl.args.ResourceWeights,
 		pl.handle.Evictor(),
 		pl.podFilter,
 		pl.handle.GetPodsAssignedToNodeFunc(),

pkg/descheduler/framework/plugins/loadaware/utilization_util.go

@@ -225,6 +225,7 @@ func evictPodsFromSourceNodes(
 	sourceNodes, destinationNodes []NodeInfo,
 	dryRun bool,
 	nodeFit bool,
+	resourceWeights map[corev1.ResourceName]int64,
 	podEvictor framework.Evictor,
 	podFilter framework.FilterFunc,
 	nodeIndexer podutil.GetPodsAssignedToNodeFunc,
@@ -284,7 +285,7 @@ func evictPodsFromSourceNodes(
 			removablePods,
 			srcNode.podMetrics,
 			map[string]corev1.ResourceList{srcNode.node.Name: srcNode.node.Status.Allocatable},
-			sorter.GenDefaultResourceToWeightMap(resourceNames),
+			resourceWeights,
 		)
 		evictPods(ctx, dryRun, removablePods, srcNode, totalAvailableUsages, podEvictor, podFilter, continueEviction, evictionReasonGenerator)
 	}
@@ -356,7 +357,7 @@ func evictPods(
 }
 
 // sortNodesByUsage sorts nodes based on usage.
-func sortNodesByUsage(nodes []NodeInfo, resourceToWeightMap sorter.ResourceToWeightMap, ascending bool) {
+func sortNodesByUsage(nodes []NodeInfo, resourceToWeightMap map[corev1.ResourceName]int64, ascending bool) {
 	scorer := sorter.ResourceUsageScorer(resourceToWeightMap)
 	sort.Slice(nodes, func(i, j int) bool {
 		var iNodeUsage, jNodeUsage corev1.ResourceList

pkg/descheduler/framework/plugins/loadaware/utilization_util_test.go

@@ -26,7 +26,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	slov1alpha1 "github.com/koordinator-sh/koordinator/apis/slo/v1alpha1"
-	"github.com/koordinator-sh/koordinator/pkg/descheduler/utils/sorter"
 )
 
 var (
@@ -167,7 +166,11 @@ func TestResourceUsagePercentages(t *testing.T) {
 func TestSortNodesByUsageDescendingOrder(t *testing.T) {
 	nodeList := []NodeInfo{testNode1, testNode2, testNode3}
 	expectedNodeList := []NodeInfo{testNode3, testNode1, testNode2}
-	weightMap := sorter.GenDefaultResourceToWeightMap([]corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory, corev1.ResourcePods})
+	weightMap := map[corev1.ResourceName]int64{
+		corev1.ResourceCPU:    1,
+		corev1.ResourceMemory: 1,
+		corev1.ResourcePods:   1,
+	}
 	sortNodesByUsage(nodeList, weightMap, false)
 
 	assert.Equal(t, expectedNodeList, nodeList)
@@ -176,7 +179,11 @@ func TestSortNodesByUsageDescendingOrder(t *testing.T) {
 func TestSortNodesByUsageAscendingOrder(t *testing.T) {
 	nodeList := []NodeInfo{testNode1, testNode2, testNode3}
 	expectedNodeList := []NodeInfo{testNode2, testNode1, testNode3}
-	weightMap := sorter.GenDefaultResourceToWeightMap([]corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory, corev1.ResourcePods})
+	weightMap := map[corev1.ResourceName]int64{
+		corev1.ResourceCPU:    1,
+		corev1.ResourceMemory: 1,
+		corev1.ResourcePods:   1,
+	}
 	sortNodesByUsage(nodeList, weightMap, true)
 
 	assert.Equal(t, expectedNodeList, nodeList)

pkg/descheduler/utils/sorter/pod.go

@@ -103,7 +103,7 @@ func KoordinatorPriorityClass(p1, p2 *corev1.Pod) int {
 }
 
 // PodUsage compares pods by the actual usage
-func PodUsage(podMetrics map[types.NamespacedName]*slov1alpha1.ResourceMap, nodeAllocatableMap map[string]corev1.ResourceList, resourceToWeightMap ResourceToWeightMap) CompareFn {
+func PodUsage(podMetrics map[types.NamespacedName]*slov1alpha1.ResourceMap, nodeAllocatableMap map[string]corev1.ResourceList, resourceToWeightMap map[corev1.ResourceName]int64) CompareFn {
 	scorer := ResourceUsageScorer(resourceToWeightMap)
 	return func(p1, p2 *corev1.Pod) int {
 		p1Metric, p1Found := podMetrics[types.NamespacedName{Namespace: p1.Namespace, Name: p1.Name}]
@@ -172,6 +172,6 @@ func PodSorter(cmp ...CompareFn) *MultiSorter {
 	return OrderedBy(comparators...)
 }
 
-func SortPodsByUsage(pods []*corev1.Pod, podMetrics map[types.NamespacedName]*slov1alpha1.ResourceMap, nodeAllocatableMap map[string]corev1.ResourceList, resourceToWeightMap ResourceToWeightMap) {
+func SortPodsByUsage(pods []*corev1.Pod, podMetrics map[types.NamespacedName]*slov1alpha1.ResourceMap, nodeAllocatableMap map[string]corev1.ResourceList, resourceToWeightMap map[corev1.ResourceName]int64) {
 	PodSorter(Reverse(PodUsage(podMetrics, nodeAllocatableMap, resourceToWeightMap))).Sort(pods)
 }

pkg/descheduler/utils/sorter/pod_test.go

@@ -124,7 +124,11 @@ func TestSortPods(t *testing.T) {
 			corev1.ResourceMemory: resource.MustParse("512Gi"),
 		},
 	}
-	resourceToWeightMap := GenDefaultResourceToWeightMap([]corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory})
+	resourceToWeightMap := map[corev1.ResourceName]int64{
+		corev1.ResourceCPU:    1,
+		corev1.ResourceMemory: 1,
+		corev1.ResourcePods:   1,
+	}
 	SortPodsByUsage(pods, podMetrics, nodeAllocatableMap, resourceToWeightMap)
 	expectedPodsOrder := []string{"test-18", "test-19", "test-17", "test-16", "test-15", "test-21", "test-20", "test-23", "test-22", "test-9", "test-8", "test-2", "test-3", "test-7", "test-4", "test-6", "test-5", "test-1", "test-11", "test-10", "test-12", "test-13", "test-14"}
 	var podsOrder []string

pkg/descheduler/utils/sorter/scorer.go

@@ -21,18 +21,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 )
 
-// ResourceToWeightMap contains resource name and weight.
-type ResourceToWeightMap map[corev1.ResourceName]int64
-
-func GenDefaultResourceToWeightMap(resourceNames []corev1.ResourceName) ResourceToWeightMap {
-	m := ResourceToWeightMap{}
-	for _, resourceName := range resourceNames {
-		m[resourceName] = 1
-	}
-	return m
-}
-
-func ResourceUsageScorer(resToWeightMap ResourceToWeightMap) func(requested, allocatable corev1.ResourceList) int64 {
+func ResourceUsageScorer(resToWeightMap map[corev1.ResourceName]int64) func(requested, allocatable corev1.ResourceList) int64 {
 	return func(requested, allocatable corev1.ResourceList) int64 {
 		var nodeScore, weightSum int64
 		for resourceName, quantity := range requested {