koord-descheduler: limits frequently migrated workloads (#950)

Signed-off-by: Joseph <joseph.t.lee@outlook.com>
Signed-off-by: Siyu Wang <FillZpp.pub@gmail.com>

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/onsi/ginkgo v1.16.4
 	github.com/onsi/gomega v1.15.0
 	github.com/openkruise/kruise-api v1.3.0
+	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/prashantv/gostub v1.1.0
 	github.com/prometheus/client_golang v1.13.0
 	github.com/spf13/cobra v1.6.1
@@ -223,6 +224,7 @@ require (
 
 replace (
 	github.com/google/cadvisor => github.com/koordinator-sh/cadvisor v0.0.0-20220919031936-833eb74e858e
+	golang.org/x/time => golang.org/x/time v0.3.0
 	k8s.io/api => k8s.io/api v0.22.6
 	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.22.6
 	k8s.io/apimachinery => k8s.io/apimachinery v0.22.6

go.sum

@@ -806,6 +806,7 @@ github.com/openkruise/kruise-api v1.3.0/go.mod h1:9ZX+ycdHKNzcA5ezAf35xOa2Mwfa2B
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/paypal/load-watcher v0.2.1/go.mod h1:MMCDf8aXF5k+K2q6AtMOBZItCvZ3oFAk+zNO4OAtp0w=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
@@ -1347,14 +1348,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs=
-golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
+golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

pkg/descheduler/apis/config/types_pluginargs.go

@@ -91,6 +91,9 @@ type MigrationControllerArgs struct {
 	// Default is false
 	DryRun bool
 
+	// MaxConcurrentReconciles is the maximum number of concurrent Reconciles which can be run. Defaults to 1.
+	MaxConcurrentReconciles int32
+
 	// EvictFailedBarePods allows pods without ownerReferences and in failed phase to be evicted.
 	EvictFailedBarePods bool
 
@@ -125,8 +128,11 @@ type MigrationControllerArgs struct {
 	// Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%).
 	MaxUnavailablePerWorkload *intstr.IntOrString
 
-	// MaxConcurrentReconciles is the maximum number of concurrent Reconciles which can be run. Defaults to 1.
-	MaxConcurrentReconciles int32
+	// ObjectLimiters control the frequency of migration/eviction to make it smoother,
+	// and also protect Pods of the same class from being evicted frequently.
+	// e.g. limiting the frequency of Pods of the same workload being evicted.
+	// The default is to set the MigrationLimitObjectWorkload limiter.
+	ObjectLimiters ObjectLimiterMap
 
 	// DefaultJobMode represents the default operating mode of the PodMigrationJob
 	// Default is PodMigrationJobModeReservationFirst
@@ -145,3 +151,21 @@ type MigrationControllerArgs struct {
 	// DefaultDeleteOptions defines options when deleting migrated pods and preempted pods through the method specified by EvictionPolicy
 	DefaultDeleteOptions *metav1.DeleteOptions
 }
+
+type MigrationLimitObjectType string
+
+const (
+	MigrationLimitObjectWorkload MigrationLimitObjectType = "workload"
+)
+
+type ObjectLimiterMap map[MigrationLimitObjectType]MigrationObjectLimiter
+
+// MigrationObjectLimiter means that if the specified dimension has multiple migrations within the configured time period
+// and exceeds the configured threshold, it will be limited.
+type MigrationObjectLimiter struct {
+	// Duration indicates the time window of the desired limit.
+	Duration metav1.Duration
+	// MaxMigrating indicates the maximum number of migrations/evictions allowed within the window time.
+	// If configured as nil or 0, the maximum number will be calculated according to MaxMigratingPerWorkload.
+	MaxMigrating *intstr.IntOrString
+}

pkg/descheduler/apis/config/v1alpha2/defaults.go

@@ -40,6 +40,14 @@ const (
 	defaultMigrationJobEvictionPolicy = migrationevictor.NativeEvictorName
 )
 
+var (
+	defaultObjectLimiters = map[MigrationLimitObjectType]MigrationObjectLimiter{
+		MigrationLimitObjectWorkload: {
+			Duration: metav1.Duration{Duration: 5 * time.Minute},
+		},
+	}
+)
+
 func addDefaultingFuncs(scheme *runtime.Scheme) error {
 	return RegisterDefaults(scheme)
 }
@@ -215,6 +223,10 @@ func SetDefaults_MigrationControllerArgs(obj *MigrationControllerArgs) {
 	if obj.EvictionPolicy == "" {
 		obj.EvictionPolicy = defaultMigrationJobEvictionPolicy
 	}
+
+	if len(obj.ObjectLimiters) == 0 {
+		obj.ObjectLimiters = defaultObjectLimiters
+	}
 }
 
 func SetDefaults_LowNodeLoadArgs(obj *LowNodeLoadArgs) {

pkg/descheduler/apis/config/v1alpha2/types_pluginargs.go

@@ -128,6 +128,12 @@ type MigrationControllerArgs struct {
 	// Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%).
 	MaxUnavailablePerWorkload *intstr.IntOrString `json:"maxUnavailablePerWorkload,omitempty"`
 
+	// ObjectLimiters control the frequency of migration/eviction to make it smoother,
+	// and also protect Pods of the same class from being evicted frequently.
+	// e.g. limiting the frequency of Pods of the same workload being evicted.
+	// The default is to set the MigrationLimitObjectWorkload limiter.
+	ObjectLimiters ObjectLimiterMap `json:"objectLimiters,omitempty"`
+
 	// DefaultJobMode represents the default operating mode of the PodMigrationJob
 	// Default is PodMigrationJobModeReservationFirst
 	DefaultJobMode string `json:"defaultJobMode,omitempty"`
@@ -145,3 +151,21 @@ type MigrationControllerArgs struct {
 	// DefaultDeleteOptions defines options when deleting migrated pods and preempted pods through the method specified by EvictionPolicy
 	DefaultDeleteOptions *metav1.DeleteOptions `json:"defaultDeleteOptions,omitempty"`
 }
+
+type MigrationLimitObjectType string
+
+const (
+	MigrationLimitObjectWorkload MigrationLimitObjectType = "workload"
+)
+
+type ObjectLimiterMap map[MigrationLimitObjectType]MigrationObjectLimiter
+
+// MigrationObjectLimiter means that if the specified dimension has multiple migrations within the configured time period
+// and exceeds the configured threshold, it will be limited.
+type MigrationObjectLimiter struct {
+	// Duration indicates the time window of the desired limit.
+	Duration metav1.Duration `json:"duration,omitempty"`
+	// MaxMigrating indicates the maximum number of migrations/evictions allowed within the window time.
+	// If configured as 0, the maximum number will be calculated according to MaxMigratingPerWorkload.
+	MaxMigrating *intstr.IntOrString `json:"maxMigrating,omitempty"`
+}