Merge pull request #590 from ffromani/resync-exclusive-resources

nodetopologymatch: allow to track only pods with exclusive resources

apis/config/types.go

@@ -142,6 +142,42 @@ type ScoringStrategy struct {
 	Resources []schedconfig.ResourceSpec
 }
 
+// ForeignPodsDetectMode is a "string" type.
+type ForeignPodsDetectMode string
+
+const (
+	ForeignPodsDetectNone                   ForeignPodsDetectMode = "None"
+	ForeignPodsDetectAll                    ForeignPodsDetectMode = "All"
+	ForeignPodsDetectOnlyExclusiveResources ForeignPodsDetectMode = "OnlyExclusiveResources"
+)
+
+// CacheResyncMethod is a "string" type.
+type CacheResyncMethod string
+
+const (
+	CacheResyncAutodetect             CacheResyncMethod = "Autodetect"
+	CacheResyncAll                    CacheResyncMethod = "All"
+	CacheResyncOnlyExclusiveResources CacheResyncMethod = "OnlyExclusiveResources"
+)
+
+// NodeResourceTopologyCache define configuration details for the NodeResourceTopology cache.
+type NodeResourceTopologyCache struct {
+	// ForeignPodsDetect sets how foreign pods should be handled.
+	// Foreign pods are pods detected running on nodes managed by a NodeResourceTopologyMatch-enabled
+	// scheduler, but not scheduled by this scheduler instance, likely because this is running as
+	// secondary scheduler. To make sure the cache is consistent, foreign pods need to be handled.
+	// Has no effect if caching is disabled (CacheResyncPeriod is zero) or
+	// if DiscardReservedNodes is enabled. If unspecified, default is "All".
+	ForeignPodsDetect *ForeignPodsDetectMode
+	// ResyncMethod sets how the resync behaves to compute the expected node state.
+	// "All" consider all pods to compute the node state. "OnlyExclusiveResources" consider
+	// only pods regardless of their QoS which have exclusive resources assigned to their
+	// containers (CPUs, devices...).
+	// Has no effect if caching is disabled (CacheResyncPeriod is zero) or if DiscardReservedNodes
+	// is enabled. "Autodetect" is the default, reads hint from NRT objects. Fallback is "All".
+	ResyncMethod *CacheResyncMethod
+}
+
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
 // NodeResourceTopologyMatchArgs holds arguments used to configure the NodeResourceTopologyMatch plugin
@@ -156,6 +192,8 @@ type NodeResourceTopologyMatchArgs struct {
 	// this option takes precedence over CacheResyncPeriodSeconds
 	// if DiscardReservedNodes is enabled, CacheResyncPeriodSeconds option is noop
 	DiscardReservedNodes bool
+	// Cache enables to fine tune the caching behavior
+	Cache *NodeResourceTopologyCache
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

apis/config/v1/defaults.go

@@ -71,6 +71,10 @@ var (
 		{Name: string(v1.ResourceMemory), Weight: 1},
 	}
 
+	defaultForeignPodsDetect = ForeignPodsDetectAll
+
+	defaultResyncMethod = CacheResyncAutodetect
+
 	// Defaults for NetworkOverhead
 	// DefaultWeightsName contains the default costs to be used by networkAware plugins
 	DefaultWeightsName = "UserDefined"
@@ -151,6 +155,17 @@ func SetDefaults_NodeResourceTopologyMatchArgs(obj *NodeResourceTopologyMatchArg
 			obj.ScoringStrategy.Resources[i].Weight = 1
 		}
 	}
+
+	if obj.Cache == nil {
+		obj.Cache = &NodeResourceTopologyCache{}
+	}
+	if obj.Cache.ForeignPodsDetect == nil {
+		obj.Cache.ForeignPodsDetect = &defaultForeignPodsDetect
+
+	}
+	if obj.Cache.ResyncMethod == nil {
+		obj.Cache.ResyncMethod = &defaultResyncMethod
+	}
 }
 
 // SetDefaults_PreemptionTolerationArgs reuses SetDefaults_DefaultPreemptionArgs

apis/config/v1/defaults_test.go

@@ -142,6 +142,10 @@ func TestSchedulingDefaults(t *testing.T) {
 					Type:      LeastAllocated,
 					Resources: defaultResourceSpec,
 				},
+				Cache: &NodeResourceTopologyCache{
+					ForeignPodsDetect: &defaultForeignPodsDetect,
+					ResyncMethod:      &defaultResyncMethod,
+				},
 			},
 		},
 		{

apis/config/v1/types.go

@@ -139,6 +139,42 @@ type ScoringStrategy struct {
 	Resources []schedulerconfigv1.ResourceSpec `json:"resources,omitempty"`
 }
 
+// ForeignPodsDetectMode is a "string" type.
+type ForeignPodsDetectMode string
+
+const (
+	ForeignPodsDetectNone                   ForeignPodsDetectMode = "None"
+	ForeignPodsDetectAll                    ForeignPodsDetectMode = "All"
+	ForeignPodsDetectOnlyExclusiveResources ForeignPodsDetectMode = "OnlyExclusiveResources"
+)
+
+// CacheResyncMethod is a "string" type.
+type CacheResyncMethod string
+
+const (
+	CacheResyncAutodetect             CacheResyncMethod = "Autodetect"
+	CacheResyncAll                    CacheResyncMethod = "All"
+	CacheResyncOnlyExclusiveResources CacheResyncMethod = "OnlyExclusiveResources"
+)
+
+// NodeResourceTopologyCache define configuration details for the NodeResourceTopology cache.
+type NodeResourceTopologyCache struct {
+	// ForeignPodsDetect sets how foreign pods should be handled.
+	// Foreign pods are pods detected running on nodes managed by a NodeResourceTopologyMatch-enabled
+	// scheduler, but not scheduled by this scheduler instance, likely because this is running as
+	// secondary scheduler. To make sure the cache is consistent, foreign pods need to be handled.
+	// Has no effect if caching is disabled (CacheResyncPeriod is zero) or if DiscardReservedNodes
+	// is enabled. If unspecified, default is "All". Use "None" to disable.
+	ForeignPodsDetect *ForeignPodsDetectMode `json:"foreignPodsDetect,omitempty"`
+	// ResyncMethod sets how the resync behaves to compute the expected node state.
+	// "All" consider all pods to compute the node state. "OnlyExclusiveResources" consider
+	// only pods regardless of their QoS which have exclusive resources assigned to their
+	// containers (CPUs, devices...).
+	// Has no effect if caching is disabled (CacheResyncPeriod is zero) or if DiscardReservedNodes
+	// is enabled. "Autodetect" is the default, reads hint from NRT objects. Fallback is "All".
+	ResyncMethod *CacheResyncMethod `json:"resyncMethod,omitempty"`
+}
+
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
 // NodeResourceTopologyMatchArgs holds arguments used to configure the NodeResourceTopologyMatch plugin
@@ -153,6 +189,8 @@ type NodeResourceTopologyMatchArgs struct {
 	// this option takes precedence over CacheResyncPeriodSeconds
 	// if DiscardReservedNodes is enabled, CacheResyncPeriodSeconds option is noop
 	DiscardReservedNodes bool `json:"discardReservedNodes,omitempty"`
+	// Cache enables to fine tune the caching behavior
+	Cache *NodeResourceTopologyCache `json:"cache,omitempty"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

apis/config/v1beta2/defaults.go

@@ -71,6 +71,10 @@ var (
 		{Name: string(v1.ResourceCPU), Weight: 1},
 		{Name: string(v1.ResourceMemory), Weight: 1},
 	}
+
+	defaultForeignPodsDetect = ForeignPodsDetectAll
+
+	defaultResyncMethod = CacheResyncAutodetect
 )
 
 // SetDefaults_CoschedulingArgs sets the default parameters for Coscheduling plugin.
@@ -149,6 +153,16 @@ func SetDefaults_NodeResourceTopologyMatchArgs(obj *NodeResourceTopologyMatchArg
 			obj.ScoringStrategy.Resources[i].Weight = 1
 		}
 	}
+
+	if obj.Cache == nil {
+		obj.Cache = &NodeResourceTopologyCache{}
+	}
+	if obj.Cache.ForeignPodsDetect == nil {
+		obj.Cache.ForeignPodsDetect = &defaultForeignPodsDetect
+	}
+	if obj.Cache.ResyncMethod == nil {
+		obj.Cache.ResyncMethod = &defaultResyncMethod
+	}
 }
 
 // SetDefaults_PreemptionTolerationArgs reuses SetDefaults_DefaultPreemptionArgs

apis/config/v1beta2/defaults_test.go

@@ -144,6 +144,10 @@ func TestSchedulingDefaults(t *testing.T) {
 					Type:      LeastAllocated,
 					Resources: defaultResourceSpec,
 				},
+				Cache: &NodeResourceTopologyCache{
+					ForeignPodsDetect: &defaultForeignPodsDetect,
+					ResyncMethod:      &defaultResyncMethod,
+				},
 			},
 		},
 		{

apis/config/v1beta2/types.go

@@ -135,6 +135,42 @@ type ScoringStrategy struct {
 	Resources []schedulerconfigv1beta2.ResourceSpec `json:"resources,omitempty"`
 }
 
+// ForeignPodsDetectMode is a "string" type.
+type ForeignPodsDetectMode string
+
+const (
+	ForeignPodsDetectNone                   ForeignPodsDetectMode = "None"
+	ForeignPodsDetectAll                    ForeignPodsDetectMode = "All"
+	ForeignPodsDetectOnlyExclusiveResources ForeignPodsDetectMode = "OnlyExclusiveResources"
+)
+
+// CacheResyncMethod is a "string" type.
+type CacheResyncMethod string
+
+const (
+	CacheResyncAutodetect             CacheResyncMethod = "Autodetect"
+	CacheResyncAll                    CacheResyncMethod = "All"
+	CacheResyncOnlyExclusiveResources CacheResyncMethod = "OnlyExclusiveResources"
+)
+
+// NodeResourceTopologyCache define configuration details for the NodeResourceTopology cache.
+type NodeResourceTopologyCache struct {
+	// ForeignPodsDetect sets how foreign pods should be handled.
+	// Foreign pods are pods detected running on nodes managed by a NodeResourceTopologyMatch-enabled
+	// scheduler, but not scheduled by this scheduler instance, likely because this is running as
+	// secondary scheduler. To make sure the cache is consistent, foreign pods need to be handled.
+	// Has no effect if caching is disabled (CacheResyncPeriod is zero) or if DiscardReservedNodes
+	// is enabled. If unspecified, default is "All". Use "None" to disable.
+	ForeignPodsDetect *ForeignPodsDetectMode `json:"foreignPodsDetect,omitempty"`
+	// ResyncMethod sets how the resync behaves to compute the expected node state.
+	// "All" consider all pods to compute the node state. "OnlyExclusiveResources" consider
+	// only pods regardless of their QoS which have exclusive resources assigned to their
+	// containers (CPUs, devices...).
+	// Has no effect if caching is disabled (CacheResyncPeriod is zero) or if DiscardReservedNodes
+	// is enabled. "Autodetect" is the default, reads hint from NRT objects. Fallback is "All".
+	ResyncMethod *CacheResyncMethod `json:"resyncMethod,omitempty"`
+}
+
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
 // NodeResourceTopologyMatchArgs holds arguments used to configure the NodeResourceTopologyMatch plugin
@@ -152,6 +188,8 @@ type NodeResourceTopologyMatchArgs struct {
 	// this option takes precedence over CacheResyncPeriodSeconds
 	// if DiscardReservedNodes is enabled, CacheResyncPeriodSeconds option is noop
 	DiscardReservedNodes bool `json:"discardReservedNodes,omitempty"`
+	// Cache enables to fine tune the caching behavior
+	Cache *NodeResourceTopologyCache `json:"cache,omitempty"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object