diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index 4309e92b0d0a..8fff2721b748 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -15869,6 +15869,46 @@ } ] }, + "com.github.karmada-io.karmada.pkg.apis.work.v1alpha2.GracefulEvictionTask": { + "description": "GracefulEvictionTask represents an graceful eviction task.", + "type": "object", + "required": [ + "name", + "reason", + "producer" + ], + "properties": { + "creationTimestamp": { + "description": "CreationTimestamp is a timestamp representing the server time when this object was created. Clients should not set this value to avoid the time inconsistency issue. It is represented in RFC3339 form(like '2021-04-25T10:02:10Z') and is in UTC.\n\nPopulated by the system. Read-only.", + "default": {}, + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Time" + }, + "message": { + "description": "Message is a human-readable message indicating details about the eviction. This may be an empty string.", + "type": "string" + }, + "name": { + "description": "Name of the cluster where the eviction perform from.", + "type": "string", + "default": "" + }, + "producer": { + "description": "Producer indicates the controller who triggered the eviction.", + "type": "string", + "default": "" + }, + "reason": { + "description": "Reason contains a programmatic identifier indicating the reason for the eviction. Producers may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty.", + "type": "string", + "default": "" + }, + "replicas": { + "description": "Replicas indicates the number of replicas should be evicted. Should be ignored for resource type that doesn't have replica.", + "type": "integer", + "format": "int32" + } + } + }, "com.github.karmada-io.karmada.pkg.apis.work.v1alpha2.NodeClaim": { "description": "NodeClaim represents the node claim HardNodeAffinity, NodeSelector and Tolerations required by each replica.", "type": "object", @@ -16040,6 +16080,14 @@ "$ref": "#/definitions/com.github.karmada-io.karmada.pkg.apis.work.v1alpha2.TargetCluster" } }, + "gracefulEvictionTasks": { + "description": "GracefulEvictionTasks holds the eviction tasks that are expected to perform the eviction in a graceful way. The intended workflow is: 1. Once the controller(such as 'taint-manager') decided to evict the resource that\n is referenced by current ResourceBinding or ClusterResourceBinding from a target\n cluster, it removes(or scale down the replicas) the target from Clusters(.spec.Clusters)\n and builds a graceful eviction task.\n2. The scheduler may perform a re-scheduler and probably select a substitute cluster\n to take over the evicting workload(resource).\n3. The graceful eviction controller takes care of the graceful eviction tasks and\n performs the final removal after the workload(resource) is available on the substitute\n cluster or exceed the grace termination period(defaults to 10 minutes).", + "type": "array", + "items": { + "default": {}, + "$ref": "#/definitions/com.github.karmada-io.karmada.pkg.apis.work.v1alpha2.GracefulEvictionTask" + } + }, "propagateDeps": { "description": "PropagateDeps tells if relevant resources should be propagated automatically. It is inherited from PropagationPolicy or ClusterPropagationPolicy. default false.", "type": "boolean" @@ -16087,6 +16135,11 @@ "default": {}, "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Condition" } + }, + "schedulerObservedGeneration": { + "description": "SchedulerObservedGeneration is the generation(.metadata.generation) observed by the scheduler. If SchedulerObservedGeneration is less than the generation in metadata means the scheduler hasn't confirmed the scheduling result or hasn't done the schedule yet.", + "type": "integer", + "format": "int64" } } }, diff --git a/charts/karmada/_crds/bases/work.karmada.io_clusterresourcebindings.yaml b/charts/karmada/_crds/bases/work.karmada.io_clusterresourcebindings.yaml index 36e7552bbea1..7652d7ad4728 100644 --- a/charts/karmada/_crds/bases/work.karmada.io_clusterresourcebindings.yaml +++ b/charts/karmada/_crds/bases/work.karmada.io_clusterresourcebindings.yaml @@ -262,6 +262,66 @@ spec: - name type: object type: array + gracefulEvictionTasks: + description: 'GracefulEvictionTasks holds the eviction tasks that + are expected to perform the eviction in a graceful way. The intended + workflow is: 1. Once the controller(such as ''taint-manager'') decided + to evict the resource that is referenced by current ResourceBinding + or ClusterResourceBinding from a target cluster, it removes(or scale + down the replicas) the target from Clusters(.spec.Clusters) and + builds a graceful eviction task. 2. The scheduler may perform a + re-scheduler and probably select a substitute cluster to take over + the evicting workload(resource). 3. The graceful eviction controller + takes care of the graceful eviction tasks and performs the final + removal after the workload(resource) is available on the substitute + cluster or exceed the grace termination period(defaults to 10 minutes).' + items: + description: GracefulEvictionTask represents an graceful eviction + task. + properties: + creationTimestamp: + description: "CreationTimestamp is a timestamp representing + the server time when this object was created. Clients should + not set this value to avoid the time inconsistency issue. + It is represented in RFC3339 form(like '2021-04-25T10:02:10Z') + and is in UTC. \n Populated by the system. Read-only." + format: date-time + type: string + message: + description: Message is a human-readable message indicating + details about the eviction. This may be an empty string. + maxLength: 1024 + type: string + name: + description: Name of the cluster where the eviction perform + from. + type: string + producer: + description: Producer indicates the controller who triggered + the eviction. + type: string + reason: + description: Reason contains a programmatic identifier indicating + the reason for the eviction. Producers may define expected + values and meanings for this field, and whether the values + are considered a guaranteed API. The value should be a CamelCase + string. This field may not be empty. + maxLength: 32 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + replicas: + description: Replicas indicates the number of replicas should + be evicted. Should be ignored for resource type that doesn't + have replica. + format: int32 + type: integer + required: + - name + - producer + - reason + type: object + type: array propagateDeps: description: PropagateDeps tells if relevant resources should be propagated automatically. It is inherited from PropagationPolicy or ClusterPropagationPolicy. @@ -607,6 +667,13 @@ spec: - type type: object type: array + schedulerObservedGeneration: + description: SchedulerObservedGeneration is the generation(.metadata.generation) + observed by the scheduler. If SchedulerObservedGeneration is less + than the generation in metadata means the scheduler hasn't confirmed + the scheduling result or hasn't done the schedule yet. + format: int64 + type: integer type: object required: - spec diff --git a/charts/karmada/_crds/bases/work.karmada.io_resourcebindings.yaml b/charts/karmada/_crds/bases/work.karmada.io_resourcebindings.yaml index 57a60a731d41..692e96e6d58d 100644 --- a/charts/karmada/_crds/bases/work.karmada.io_resourcebindings.yaml +++ b/charts/karmada/_crds/bases/work.karmada.io_resourcebindings.yaml @@ -262,6 +262,66 @@ spec: - name type: object type: array + gracefulEvictionTasks: + description: 'GracefulEvictionTasks holds the eviction tasks that + are expected to perform the eviction in a graceful way. The intended + workflow is: 1. Once the controller(such as ''taint-manager'') decided + to evict the resource that is referenced by current ResourceBinding + or ClusterResourceBinding from a target cluster, it removes(or scale + down the replicas) the target from Clusters(.spec.Clusters) and + builds a graceful eviction task. 2. The scheduler may perform a + re-scheduler and probably select a substitute cluster to take over + the evicting workload(resource). 3. The graceful eviction controller + takes care of the graceful eviction tasks and performs the final + removal after the workload(resource) is available on the substitute + cluster or exceed the grace termination period(defaults to 10 minutes).' + items: + description: GracefulEvictionTask represents an graceful eviction + task. + properties: + creationTimestamp: + description: "CreationTimestamp is a timestamp representing + the server time when this object was created. Clients should + not set this value to avoid the time inconsistency issue. + It is represented in RFC3339 form(like '2021-04-25T10:02:10Z') + and is in UTC. \n Populated by the system. Read-only." + format: date-time + type: string + message: + description: Message is a human-readable message indicating + details about the eviction. This may be an empty string. + maxLength: 1024 + type: string + name: + description: Name of the cluster where the eviction perform + from. + type: string + producer: + description: Producer indicates the controller who triggered + the eviction. + type: string + reason: + description: Reason contains a programmatic identifier indicating + the reason for the eviction. Producers may define expected + values and meanings for this field, and whether the values + are considered a guaranteed API. The value should be a CamelCase + string. This field may not be empty. + maxLength: 32 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + replicas: + description: Replicas indicates the number of replicas should + be evicted. Should be ignored for resource type that doesn't + have replica. + format: int32 + type: integer + required: + - name + - producer + - reason + type: object + type: array propagateDeps: description: PropagateDeps tells if relevant resources should be propagated automatically. It is inherited from PropagationPolicy or ClusterPropagationPolicy. @@ -607,6 +667,13 @@ spec: - type type: object type: array + schedulerObservedGeneration: + description: SchedulerObservedGeneration is the generation(.metadata.generation) + observed by the scheduler. If SchedulerObservedGeneration is less + than the generation in metadata means the scheduler hasn't confirmed + the scheduling result or hasn't done the schedule yet. + format: int64 + type: integer type: object required: - spec diff --git a/pkg/apis/work/v1alpha2/binding_types.go b/pkg/apis/work/v1alpha2/binding_types.go index c87429997eed..d5f78e67d5a0 100644 --- a/pkg/apis/work/v1alpha2/binding_types.go +++ b/pkg/apis/work/v1alpha2/binding_types.go @@ -72,6 +72,22 @@ type ResourceBindingSpec struct { // +optional Clusters []TargetCluster `json:"clusters,omitempty"` + // GracefulEvictionTasks holds the eviction tasks that are expected to perform + // the eviction in a graceful way. + // The intended workflow is: + // 1. Once the controller(such as 'taint-manager') decided to evict the resource that + // is referenced by current ResourceBinding or ClusterResourceBinding from a target + // cluster, it removes(or scale down the replicas) the target from Clusters(.spec.Clusters) + // and builds a graceful eviction task. + // 2. The scheduler may perform a re-scheduler and probably select a substitute cluster + // to take over the evicting workload(resource). + // 3. The graceful eviction controller takes care of the graceful eviction tasks and + // performs the final removal after the workload(resource) is available on the substitute + // cluster or exceed the grace termination period(defaults to 10 minutes). + // + // +optional + GracefulEvictionTasks []GracefulEvictionTask `json:"gracefulEvictionTasks,omitempty"` + // RequiredBy represents the list of Bindings that depend on the referencing resource. // +optional RequiredBy []BindingSnapshot `json:"requiredBy,omitempty"` @@ -142,6 +158,47 @@ type TargetCluster struct { Replicas int32 `json:"replicas,omitempty"` } +// GracefulEvictionTask represents an graceful eviction task. +type GracefulEvictionTask struct { + // Name of the cluster where the eviction perform from. + Name string `json:"name"` + + // Reason contains a programmatic identifier indicating the reason for the eviction. + // Producers may define expected values and meanings for this field, + // and whether the values are considered a guaranteed API. + // The value should be a CamelCase string. + // This field may not be empty. + // +required + // +kubebuilder:validation:MaxLength=32 + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:Pattern=`^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$` + Reason string `json:"reason"` + + // Message is a human-readable message indicating details about the eviction. + // This may be an empty string. + // +optional + // +kubebuilder:validation:MaxLength=1024 + Message string `json:"message,omitempty"` + + // Producer indicates the controller who triggered the eviction. + // +required + Producer string `json:"producer"` + + // CreationTimestamp is a timestamp representing the server time when this object was + // created. + // Clients should not set this value to avoid the time inconsistency issue. + // It is represented in RFC3339 form(like '2021-04-25T10:02:10Z') and is in UTC. + // + // Populated by the system. Read-only. + // +optional + CreationTimestamp metav1.Time `json:"creationTimestamp,omitempty"` + + // Replicas indicates the number of replicas should be evicted. + // Should be ignored for resource type that doesn't have replica. + // +optional + Replicas int32 `json:"replicas,omitempty"` +} + // BindingSnapshot is a snapshot of a ResourceBinding or ClusterResourceBinding. type BindingSnapshot struct { // Namespace represents the namespace of the Binding. @@ -161,6 +218,11 @@ type BindingSnapshot struct { // ResourceBindingStatus represents the overall status of the strategy as well as the referenced resources. type ResourceBindingStatus struct { + // SchedulerObservedGeneration is the generation(.metadata.generation) observed by the scheduler. + // If SchedulerObservedGeneration is less than the generation in metadata means the scheduler hasn't confirmed + // the scheduling result or hasn't done the schedule yet. + // +optional + SchedulerObservedGeneration int64 `json:"schedulerObservedGeneration,omitempty"` // Conditions contain the different condition statuses. // +optional Conditions []metav1.Condition `json:"conditions,omitempty"` diff --git a/pkg/apis/work/v1alpha2/well_known_constants.go b/pkg/apis/work/v1alpha2/well_known_constants.go index 902856fe3a7c..a642733322f4 100644 --- a/pkg/apis/work/v1alpha2/well_known_constants.go +++ b/pkg/apis/work/v1alpha2/well_known_constants.go @@ -59,3 +59,16 @@ const ( // https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids ResourceTemplateUIDAnnotation = "resourcetemplate.karmada.io/uid" ) + +// Define eviction reasons. +const ( + // EvictionReasonExceedTaintToleration describes the eviction is triggered + // because can not tolerate taint or exceed toleration period of time. + EvictionReasonExceedTaintToleration = "ExceedTaintToleration" +) + +// Define eviction producers. +const ( + // EvictionProducerTaintManager represents the name of taint manager. + EvictionProducerTaintManager = "TaintManager" +) diff --git a/pkg/apis/work/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/work/v1alpha2/zz_generated.deepcopy.go index d9476bd5bee3..c0b25c45c8f1 100644 --- a/pkg/apis/work/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/work/v1alpha2/zz_generated.deepcopy.go @@ -114,6 +114,23 @@ func (in *ClusterResourceBindingList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GracefulEvictionTask) DeepCopyInto(out *GracefulEvictionTask) { + *out = *in + in.CreationTimestamp.DeepCopyInto(&out.CreationTimestamp) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GracefulEvictionTask. +func (in *GracefulEvictionTask) DeepCopy() *GracefulEvictionTask { + if in == nil { + return nil + } + out := new(GracefulEvictionTask) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NodeClaim) DeepCopyInto(out *NodeClaim) { *out = *in @@ -268,6 +285,13 @@ func (in *ResourceBindingSpec) DeepCopyInto(out *ResourceBindingSpec) { *out = make([]TargetCluster, len(*in)) copy(*out, *in) } + if in.GracefulEvictionTasks != nil { + in, out := &in.GracefulEvictionTasks, &out.GracefulEvictionTasks + *out = make([]GracefulEvictionTask, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.RequiredBy != nil { in, out := &in.RequiredBy, &out.RequiredBy *out = make([]BindingSnapshot, len(*in)) diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index 16eaa7217bd8..42f023c1c3c0 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -98,6 +98,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.BindingSnapshot": schema_pkg_apis_work_v1alpha2_BindingSnapshot(ref), "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.ClusterResourceBinding": schema_pkg_apis_work_v1alpha2_ClusterResourceBinding(ref), "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.ClusterResourceBindingList": schema_pkg_apis_work_v1alpha2_ClusterResourceBindingList(ref), + "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.GracefulEvictionTask": schema_pkg_apis_work_v1alpha2_GracefulEvictionTask(ref), "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.NodeClaim": schema_pkg_apis_work_v1alpha2_NodeClaim(ref), "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.ObjectReference": schema_pkg_apis_work_v1alpha2_ObjectReference(ref), "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.ReplicaRequirements": schema_pkg_apis_work_v1alpha2_ReplicaRequirements(ref), @@ -4210,6 +4211,67 @@ func schema_pkg_apis_work_v1alpha2_ClusterResourceBindingList(ref common.Referen } } +func schema_pkg_apis_work_v1alpha2_GracefulEvictionTask(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "GracefulEvictionTask represents an graceful eviction task.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "name": { + SchemaProps: spec.SchemaProps{ + Description: "Name of the cluster where the eviction perform from.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "reason": { + SchemaProps: spec.SchemaProps{ + Description: "Reason contains a programmatic identifier indicating the reason for the eviction. Producers may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "message": { + SchemaProps: spec.SchemaProps{ + Description: "Message is a human-readable message indicating details about the eviction. This may be an empty string.", + Type: []string{"string"}, + Format: "", + }, + }, + "producer": { + SchemaProps: spec.SchemaProps{ + Description: "Producer indicates the controller who triggered the eviction.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "creationTimestamp": { + SchemaProps: spec.SchemaProps{ + Description: "CreationTimestamp is a timestamp representing the server time when this object was created. Clients should not set this value to avoid the time inconsistency issue. It is represented in RFC3339 form(like '2021-04-25T10:02:10Z') and is in UTC.\n\nPopulated by the system. Read-only.", + Default: map[string]interface{}{}, + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Time"), + }, + }, + "replicas": { + SchemaProps: spec.SchemaProps{ + Description: "Replicas indicates the number of replicas should be evicted. Should be ignored for resource type that doesn't have replica.", + Type: []string{"integer"}, + Format: "int32", + }, + }, + }, + Required: []string{"name", "reason", "producer"}, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/apis/meta/v1.Time"}, + } +} + func schema_pkg_apis_work_v1alpha2_NodeClaim(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -4504,6 +4566,20 @@ func schema_pkg_apis_work_v1alpha2_ResourceBindingSpec(ref common.ReferenceCallb }, }, }, + "gracefulEvictionTasks": { + SchemaProps: spec.SchemaProps{ + Description: "GracefulEvictionTasks holds the eviction tasks that are expected to perform the eviction in a graceful way. The intended workflow is: 1. Once the controller(such as 'taint-manager') decided to evict the resource that\n is referenced by current ResourceBinding or ClusterResourceBinding from a target\n cluster, it removes(or scale down the replicas) the target from Clusters(.spec.Clusters)\n and builds a graceful eviction task.\n2. The scheduler may perform a re-scheduler and probably select a substitute cluster\n to take over the evicting workload(resource).\n3. The graceful eviction controller takes care of the graceful eviction tasks and\n performs the final removal after the workload(resource) is available on the substitute\n cluster or exceed the grace termination period(defaults to 10 minutes).", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.GracefulEvictionTask"), + }, + }, + }, + }, + }, "requiredBy": { SchemaProps: spec.SchemaProps{ Description: "RequiredBy represents the list of Bindings that depend on the referencing resource.", @@ -4523,7 +4599,7 @@ func schema_pkg_apis_work_v1alpha2_ResourceBindingSpec(ref common.ReferenceCallb }, }, Dependencies: []string{ - "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.BindingSnapshot", "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.ObjectReference", "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.ReplicaRequirements", "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.TargetCluster"}, + "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.BindingSnapshot", "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.GracefulEvictionTask", "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.ObjectReference", "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.ReplicaRequirements", "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.TargetCluster"}, } } @@ -4534,6 +4610,13 @@ func schema_pkg_apis_work_v1alpha2_ResourceBindingStatus(ref common.ReferenceCal Description: "ResourceBindingStatus represents the overall status of the strategy as well as the referenced resources.", Type: []string{"object"}, Properties: map[string]spec.Schema{ + "schedulerObservedGeneration": { + SchemaProps: spec.SchemaProps{ + Description: "SchedulerObservedGeneration is the generation(.metadata.generation) observed by the scheduler. If SchedulerObservedGeneration is less than the generation in metadata means the scheduler hasn't confirmed the scheduling result or hasn't done the schedule yet.", + Type: []string{"integer"}, + Format: "int64", + }, + }, "conditions": { SchemaProps: spec.SchemaProps{ Description: "Conditions contain the different condition statuses.",