koord-descheduler: enhance LowNodeLoad scorer (#2092)

Signed-off-by: LY-today <724102053@qq.com>

pkg/descheduler/framework/plugins/loadaware/utilization_util.go

@@ -435,7 +435,8 @@ func balancePods(ctx context.Context,
 			klog.V(4).InfoS("No removable pods on node, try next node", "node", klog.KObj(srcNode.node), "nodePool", nodePoolName)
 			continue
 		}
-		sortPodsOnOneOverloadedNode(srcNode, removablePods, resourceWeights)
+		sortPodsOnOneOverloadedNode(srcNode, removablePods, resourceWeights, prod)
+
 		evictPods(ctx, nodePoolName, dryRun, prod, removablePods, srcNode, totalAvailableUsages, podEvictor, podFilter, continueEviction, evictionReasonGenerator)
 	}
 }
@@ -665,14 +666,28 @@ func calcAverageResourceUsagePercent(nodeUsages map[string]*NodeUsage) (Resource
 	}
 	return average, prodAverage
 }
-func sortPodsOnOneOverloadedNode(srcNode NodeInfo, removablePods []*corev1.Pod, resourceWeights map[corev1.ResourceName]int64) {
+func sortPodsOnOneOverloadedNode(srcNode NodeInfo, removablePods []*corev1.Pod, resourceWeights map[corev1.ResourceName]int64, prod bool) {
 	weights := make(map[corev1.ResourceName]int64)
 	// get the overused resource of this node, and the weights of appropriately using resources will be zero.
-	overusedResources, _ := isNodeOverutilized(srcNode.usage, srcNode.thresholds.highResourceThreshold)
-	for or := range overusedResources {
+	var overusedResources corev1.ResourceList
+	if prod {
+		overusedResources, _ = isNodeOverutilized(srcNode.prodUsage, srcNode.thresholds.prodHighResourceThreshold)
+	} else {
+		overusedResources, _ = isNodeOverutilized(srcNode.usage, srcNode.thresholds.highResourceThreshold)
+	}
+	resourcesThatExceedThresholds := map[corev1.ResourceName]resource.Quantity{}
+	for or, used := range overusedResources {
+		usedCopy := used.DeepCopy()
 		weights[or] = resourceWeights[or]
+		if prod {
+			usedCopy.Sub(*srcNode.thresholds.prodHighResourceThreshold[or])
+		} else {
+			usedCopy.Sub(*srcNode.thresholds.highResourceThreshold[or])
+		}
+		resourcesThatExceedThresholds[or] = usedCopy
 	}
 	sorter.SortPodsByUsage(
+		resourcesThatExceedThresholds,
 		removablePods,
 		srcNode.podMetrics,
 		map[string]corev1.ResourceList{srcNode.node.Name: srcNode.node.Status.Allocatable},

pkg/descheduler/framework/plugins/loadaware/utilization_util_test.go

@@ -226,6 +226,6 @@ func TestSortPodsOnOneOverloadedNode(t *testing.T) {
 		corev1.ResourceCPU:    int64(1),
 		corev1.ResourceMemory: int64(1),
 	}
-	sortPodsOnOneOverloadedNode(nodeInfo, removablePods, resourceWeights)
+	sortPodsOnOneOverloadedNode(nodeInfo, removablePods, resourceWeights, false)
 	assert.Equal(t, expectedResult, removablePods)
 }

pkg/descheduler/utils/sorter/pod.go

@@ -18,6 +18,7 @@ package sorter
 
 import (
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/types"
 	schedulingcorev1helper "k8s.io/component-helpers/scheduling/corev1"
 	apiscorehelper "k8s.io/kubernetes/pkg/apis/core/helper"
@@ -103,16 +104,17 @@ func KoordinatorPriorityClass(p1, p2 *corev1.Pod) int {
 }
 
 // PodUsage compares pods by the actual usage
-func PodUsage(podMetrics map[types.NamespacedName]*slov1alpha1.ResourceMap, nodeAllocatableMap map[string]corev1.ResourceList, resourceToWeightMap map[corev1.ResourceName]int64) CompareFn {
-	scorer := ResourceUsageScorer(resourceToWeightMap)
+func PodUsage(resourcesThatExceedThresholds map[corev1.ResourceName]resource.Quantity, podMetrics map[types.NamespacedName]*slov1alpha1.ResourceMap, resourceToWeightMap map[corev1.ResourceName]int64) CompareFn {
+	scorer := ResourceUsageScorerPod(resourceToWeightMap)
 	return func(p1, p2 *corev1.Pod) int {
 		p1Metric, p1Found := podMetrics[types.NamespacedName{Namespace: p1.Namespace, Name: p1.Name}]
 		p2Metric, p2Found := podMetrics[types.NamespacedName{Namespace: p2.Namespace, Name: p2.Name}]
 		if !p1Found || !p2Found {
 			return cmpBool(!p1Found, !p2Found)
 		}
-		p1Score := scorer(p1Metric.ResourceList, nodeAllocatableMap[p1.Spec.NodeName])
-		p2Score := scorer(p2Metric.ResourceList, nodeAllocatableMap[p2.Spec.NodeName])
+		p1Score := scorer(p1Metric.ResourceList, resourcesThatExceedThresholds)
+		p2Score := scorer(p2Metric.ResourceList, resourcesThatExceedThresholds)
+
 		if p1Score == p2Score {
 			return 0
 		}
@@ -172,6 +174,6 @@ func PodSorter(cmp ...CompareFn) *MultiSorter {
 	return OrderedBy(comparators...)
 }
 
-func SortPodsByUsage(pods []*corev1.Pod, podMetrics map[types.NamespacedName]*slov1alpha1.ResourceMap, nodeAllocatableMap map[string]corev1.ResourceList, resourceToWeightMap map[corev1.ResourceName]int64) {
-	PodSorter(Reverse(PodUsage(podMetrics, nodeAllocatableMap, resourceToWeightMap))).Sort(pods)
+func SortPodsByUsage(resourcesThatExceedThresholds map[corev1.ResourceName]resource.Quantity, pods []*corev1.Pod, podMetrics map[types.NamespacedName]*slov1alpha1.ResourceMap, nodeAllocatableMap map[string]corev1.ResourceList, resourceToWeightMap map[corev1.ResourceName]int64) {
+	PodSorter(Reverse(PodUsage(resourcesThatExceedThresholds, podMetrics, resourceToWeightMap))).Sort(pods)
 }

pkg/descheduler/utils/sorter/pod_test.go

@@ -129,8 +129,69 @@ func TestSortPods(t *testing.T) {
 		corev1.ResourceMemory: 1,
 		corev1.ResourcePods:   1,
 	}
-	SortPodsByUsage(pods, podMetrics, nodeAllocatableMap, resourceToWeightMap)
-	expectedPodsOrder := []string{"test-1", "test-12", "test-18", "test-19", "test-17", "test-16", "test-15", "test-21", "test-20", "test-23", "test-22", "test-9", "test-8", "test-11", "test-10", "test-13", "test-14", "test-2", "test-3", "test-7", "test-4", "test-6", "test-5"}
+	SortPodsByUsage(nil, pods, podMetrics, nodeAllocatableMap, resourceToWeightMap)
+	expectedPodsOrder := []string{"test-1", "test-12", "test-18", "test-16", "test-19", "test-17", "test-15", "test-21", "test-20", "test-23", "test-22", "test-9", "test-8", "test-11", "test-10", "test-13", "test-14", "test-2", "test-3", "test-7", "test-4", "test-6", "test-5"}
+	var podsOrder []string
+	for _, v := range pods {
+		podsOrder = append(podsOrder, v.Name)
+	}
+	assert.Equal(t, expectedPodsOrder, podsOrder)
+}
+
+func TestSortPods2(t *testing.T) {
+	creationTime := time.Now()
+	pods := []*corev1.Pod{
+		makePod("test-1", extension.PriorityBatchValueMin, extension.QoSBE, corev1.PodQOSBestEffort, creationTime),
+		makePod("test-2", extension.PriorityBatchValueMin, extension.QoSBE, corev1.PodQOSBestEffort, creationTime),
+		makePod("test-3", extension.PriorityBatchValueMin, extension.QoSBE, corev1.PodQOSBestEffort, creationTime),
+		makePod("test-4", extension.PriorityBatchValueMin, extension.QoSBE, corev1.PodQOSBestEffort, creationTime),
+	}
+	podMetrics := map[types.NamespacedName]*slov1alpha1.ResourceMap{
+		{Namespace: "default", Name: "test-1"}: {
+			ResourceList: corev1.ResourceList{
+				corev1.ResourceCPU:    resource.MustParse("2"),
+				corev1.ResourceMemory: resource.MustParse("5Gi"),
+			},
+		},
+		{Namespace: "default", Name: "test-2"}: {
+			ResourceList: corev1.ResourceList{
+				corev1.ResourceCPU:    resource.MustParse("3"),
+				corev1.ResourceMemory: resource.MustParse("4Gi"),
+			},
+		},
+		{Namespace: "default", Name: "test-3"}: {
+			ResourceList: corev1.ResourceList{
+				corev1.ResourceCPU:    resource.MustParse("4"),
+				corev1.ResourceMemory: resource.MustParse("3Gi"),
+			},
+		},
+		{Namespace: "default", Name: "test-4"}: {
+			ResourceList: corev1.ResourceList{
+				corev1.ResourceCPU:    resource.MustParse("5"),
+				corev1.ResourceMemory: resource.MustParse("2Gi"),
+			},
+		},
+	}
+
+	nodeAllocatableMap := map[string]corev1.ResourceList{
+		"test-node": {
+			corev1.ResourceCPU:    resource.MustParse("96"),
+			corev1.ResourceMemory: resource.MustParse("512Gi"),
+		},
+	}
+
+	resourcesThatExceedThresholds := map[corev1.ResourceName]resource.Quantity{
+		corev1.ResourceCPU:    resource.MustParse("4"),
+		corev1.ResourceMemory: resource.MustParse("4Gi"),
+	}
+
+	resourceToWeightMap := map[corev1.ResourceName]int64{
+		corev1.ResourceCPU:    1,
+		corev1.ResourceMemory: 2,
+		corev1.ResourcePods:   1,
+	}
+	SortPodsByUsage(resourcesThatExceedThresholds, pods, podMetrics, nodeAllocatableMap, resourceToWeightMap)
+	expectedPodsOrder := []string{"test-2", "test-1", "test-3", "test-4"}
 	var podsOrder []string
 	for _, v := range pods {
 		podsOrder = append(podsOrder, v.Name)

pkg/descheduler/utils/sorter/scorer.go

@@ -37,6 +37,23 @@ func ResourceUsageScorer(resToWeightMap map[corev1.ResourceName]int64) func(requ
 	}
 }
 
+func ResourceUsageScorerPod(resToWeightMap map[corev1.ResourceName]int64) func(requested corev1.ResourceList, allocatable corev1.ResourceList) float64 {
+	return func(requested, allocatable corev1.ResourceList) float64 {
+		var nodeScore float64
+		var weightSum int64
+		for resourceName, quantity := range requested {
+			weight := resToWeightMap[resourceName]
+			resourceScore := mostRequestedScorePod(getResourceValue(resourceName, quantity), getResourceValue(resourceName, allocatable[resourceName]))
+			nodeScore += resourceScore * float64(weight)
+			weightSum += weight
+		}
+		if weightSum == 0 {
+			return 0
+		}
+		return nodeScore / float64(weightSum)
+	}
+}
+
 func mostRequestedScore(requested, capacity int64) int64 {
 	if capacity == 0 {
 		return 0
@@ -50,6 +67,19 @@ func mostRequestedScore(requested, capacity int64) int64 {
 	return (requested * 1000) / capacity
 }
 
+// mostRequestedScorePod The closer the request is to the capacity, the higher the score.The score will be higher when requested >= ratio
+func mostRequestedScorePod(requested, capacity int64) float64 {
+	if capacity == 0 {
+		return 0
+	}
+	ratio := float64(requested) / float64(capacity)
+	if ratio >= 1 {
+		return 1/ratio + 1
+	} else {
+		return ratio
+	}
+}
+
 func getResourceValue(resourceName corev1.ResourceName, quantity resource.Quantity) int64 {
 	if resourceName == corev1.ResourceCPU {
 		return quantity.MilliValue()

pkg/descheduler/utils/sorter/scorer_test.go

@@ -0,0 +1,94 @@
+package sorter
+
+import (
+	"testing"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+)
+
+func TestFunctions(t *testing.T) {
+	resToWeightMap := map[corev1.ResourceName]int64{
+		corev1.ResourceCPU:    2,
+		corev1.ResourceMemory: 3,
+	}
+
+	requested1 := corev1.ResourceList{
+		corev1.ResourceCPU:    *resource.NewMilliQuantity(500, resource.DecimalSI),
+		corev1.ResourceMemory: *resource.NewQuantity(1024, resource.BinarySI),
+	}
+	allocatable1 := corev1.ResourceList{
+		corev1.ResourceCPU:    *resource.NewMilliQuantity(1000, resource.DecimalSI),
+		corev1.ResourceMemory: *resource.NewQuantity(2048, resource.BinarySI),
+	}
+
+	scorer1 := ResourceUsageScorer(resToWeightMap)
+	score1 := scorer1(requested1, allocatable1)
+	if score1 != 500 {
+		t.Fatal("ResourceUsageScorer score1")
+	}
+
+	score2 := mostRequestedScore(800, 1000)
+	if score2 != 800 {
+		t.Fatal("ResourceUsageScorer score2")
+	}
+	score3 := mostRequestedScore(1200, 1000)
+	if score3 != 1000 {
+		t.Fatal("ResourceUsageScorer score3")
+	}
+
+	score4 := mostRequestedScore(1200, 0)
+	if score4 != 0 {
+		t.Fatal("ResourceUsageScorer score4")
+	}
+
+	resToWeightMap = map[corev1.ResourceName]int64{
+		corev1.ResourceCPU:    0,
+		corev1.ResourceMemory: 0,
+	}
+
+	scorer1 = ResourceUsageScorer(resToWeightMap)
+	score1 = scorer1(requested1, allocatable1)
+	if score1 != 0 {
+		t.Fatal("ResourceUsageScorer score5")
+	}
+
+	qCPU := resource.NewMilliQuantity(500, resource.DecimalSI)
+	valueCPU := getResourceValue(corev1.ResourceCPU, *qCPU)
+	if valueCPU != 500 {
+		t.Fatal("ResourceUsageScorer score6")
+	}
+
+	qMem := resource.NewQuantity(1024, resource.BinarySI)
+	valueMem := getResourceValue(corev1.ResourceMemory, *qMem)
+	if valueMem != 1024 {
+		t.Fatal("ResourceUsageScorer score7")
+	}
+}
+
+func TestResourceUsageScorerPod(t *testing.T) {
+	resToWeightMap := map[corev1.ResourceName]int64{
+		corev1.ResourceCPU:    0,
+		corev1.ResourceMemory: 0,
+	}
+
+	scorer1 := ResourceUsageScorerPod(resToWeightMap)
+	requested1 := corev1.ResourceList{
+		corev1.ResourceCPU:    *resource.NewMilliQuantity(500, resource.DecimalSI),
+		corev1.ResourceMemory: *resource.NewQuantity(1024, resource.BinarySI),
+	}
+	allocatable1 := corev1.ResourceList{
+		corev1.ResourceCPU:    *resource.NewMilliQuantity(1000, resource.DecimalSI),
+		corev1.ResourceMemory: *resource.NewQuantity(2048, resource.BinarySI),
+	}
+
+	score1 := scorer1(requested1, allocatable1)
+	if score1 != 0 {
+		t.Fatal("TestResourceUsageScorerPod ResourceUsageScorerPod fail")
+	}
+
+	score := mostRequestedScorePod(100, 0)
+	if score != 0 {
+		t.Fatal("TestResourceUsageScorerPod mostRequestedScorePod fail")
+	}
+}