From 89f9c96644975c0822ca782200ec6689b181569c Mon Sep 17 00:00:00 2001 From: RainbowMango Date: Thu, 28 Jul 2022 12:03:51 +0800 Subject: [PATCH] Extend Binding API adding graceful eviction Signed-off-by: RainbowMango --- api/openapi-spec/swagger.json | 53 ++++++++++++ ...rk.karmada.io_clusterresourcebindings.yaml | 67 +++++++++++++++ .../work.karmada.io_resourcebindings.yaml | 67 +++++++++++++++ pkg/apis/work/v1alpha2/binding_types.go | 63 ++++++++++++++ .../work/v1alpha2/well_known_constants.go | 13 +++ .../work/v1alpha2/zz_generated.deepcopy.go | 29 +++++++ pkg/generated/openapi/zz_generated.openapi.go | 85 ++++++++++++++++++- 7 files changed, 376 insertions(+), 1 deletion(-) diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index b63e41cc77ec..e1955b55e877 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -15873,6 +15873,46 @@ } ] }, + "com.github.karmada-io.karmada.pkg.apis.work.v1alpha2.GracefulEvictionTask": { + "description": "GracefulEvictionTask represents a graceful eviction task.", + "type": "object", + "required": [ + "fromCluster", + "reason", + "producer" + ], + "properties": { + "creationTimestamp": { + "description": "CreationTimestamp is a timestamp representing the server time when this object was created. Clients should not set this value to avoid the time inconsistency issue. It is represented in RFC3339 form(like '2021-04-25T10:02:10Z') and is in UTC.\n\nPopulated by the system. Read-only.", + "default": {}, + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Time" + }, + "fromCluster": { + "description": "FromCluster which cluster the eviction perform from.", + "type": "string", + "default": "" + }, + "message": { + "description": "Message is a human-readable message indicating details about the eviction. This may be an empty string.", + "type": "string" + }, + "producer": { + "description": "Producer indicates the controller who triggered the eviction.", + "type": "string", + "default": "" + }, + "reason": { + "description": "Reason contains a programmatic identifier indicating the reason for the eviction. Producers may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty.", + "type": "string", + "default": "" + }, + "replicas": { + "description": "Replicas indicates the number of replicas should be evicted. Should be ignored for resource type that doesn't have replica.", + "type": "integer", + "format": "int32" + } + } + }, "com.github.karmada-io.karmada.pkg.apis.work.v1alpha2.NodeClaim": { "description": "NodeClaim represents the node claim HardNodeAffinity, NodeSelector and Tolerations required by each replica.", "type": "object", @@ -16044,6 +16084,14 @@ "$ref": "#/definitions/com.github.karmada-io.karmada.pkg.apis.work.v1alpha2.TargetCluster" } }, + "gracefulEvictionTasks": { + "description": "GracefulEvictionTasks holds the eviction tasks that are expected to perform the eviction in a graceful way. The intended workflow is: 1. Once the controller(such as 'taint-manager') decided to evict the resource that\n is referenced by current ResourceBinding or ClusterResourceBinding from a target\n cluster, it removes(or scale down the replicas) the target from Clusters(.spec.Clusters)\n and builds a graceful eviction task.\n2. The scheduler may perform a re-scheduler and probably select a substitute cluster\n to take over the evicting workload(resource).\n3. The graceful eviction controller takes care of the graceful eviction tasks and\n performs the final removal after the workload(resource) is available on the substitute\n cluster or exceed the grace termination period(defaults to 10 minutes).", + "type": "array", + "items": { + "default": {}, + "$ref": "#/definitions/com.github.karmada-io.karmada.pkg.apis.work.v1alpha2.GracefulEvictionTask" + } + }, "propagateDeps": { "description": "PropagateDeps tells if relevant resources should be propagated automatically. It is inherited from PropagationPolicy or ClusterPropagationPolicy. default false.", "type": "boolean" @@ -16091,6 +16139,11 @@ "default": {}, "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Condition" } + }, + "schedulerObservedGeneration": { + "description": "SchedulerObservedGeneration is the generation(.metadata.generation) observed by the scheduler. If SchedulerObservedGeneration is less than the generation in metadata means the scheduler hasn't confirmed the scheduling result or hasn't done the schedule yet.", + "type": "integer", + "format": "int64" } } }, diff --git a/charts/karmada/_crds/bases/work.karmada.io_clusterresourcebindings.yaml b/charts/karmada/_crds/bases/work.karmada.io_clusterresourcebindings.yaml index 628f1e92fa6b..58fd98879863 100644 --- a/charts/karmada/_crds/bases/work.karmada.io_clusterresourcebindings.yaml +++ b/charts/karmada/_crds/bases/work.karmada.io_clusterresourcebindings.yaml @@ -264,6 +264,66 @@ spec: - name type: object type: array + gracefulEvictionTasks: + description: 'GracefulEvictionTasks holds the eviction tasks that + are expected to perform the eviction in a graceful way. The intended + workflow is: 1. Once the controller(such as ''taint-manager'') decided + to evict the resource that is referenced by current ResourceBinding + or ClusterResourceBinding from a target cluster, it removes(or scale + down the replicas) the target from Clusters(.spec.Clusters) and + builds a graceful eviction task. 2. The scheduler may perform a + re-scheduler and probably select a substitute cluster to take over + the evicting workload(resource). 3. The graceful eviction controller + takes care of the graceful eviction tasks and performs the final + removal after the workload(resource) is available on the substitute + cluster or exceed the grace termination period(defaults to 10 minutes).' + items: + description: GracefulEvictionTask represents a graceful eviction + task. + properties: + creationTimestamp: + description: "CreationTimestamp is a timestamp representing + the server time when this object was created. Clients should + not set this value to avoid the time inconsistency issue. + It is represented in RFC3339 form(like '2021-04-25T10:02:10Z') + and is in UTC. \n Populated by the system. Read-only." + format: date-time + type: string + fromCluster: + description: FromCluster which cluster the eviction perform + from. + type: string + message: + description: Message is a human-readable message indicating + details about the eviction. This may be an empty string. + maxLength: 1024 + type: string + producer: + description: Producer indicates the controller who triggered + the eviction. + type: string + reason: + description: Reason contains a programmatic identifier indicating + the reason for the eviction. Producers may define expected + values and meanings for this field, and whether the values + are considered a guaranteed API. The value should be a CamelCase + string. This field may not be empty. + maxLength: 32 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + replicas: + description: Replicas indicates the number of replicas should + be evicted. Should be ignored for resource type that doesn't + have replica. + format: int32 + type: integer + required: + - fromCluster + - producer + - reason + type: object + type: array propagateDeps: description: PropagateDeps tells if relevant resources should be propagated automatically. It is inherited from PropagationPolicy or ClusterPropagationPolicy. @@ -609,6 +669,13 @@ spec: - type type: object type: array + schedulerObservedGeneration: + description: SchedulerObservedGeneration is the generation(.metadata.generation) + observed by the scheduler. If SchedulerObservedGeneration is less + than the generation in metadata means the scheduler hasn't confirmed + the scheduling result or hasn't done the schedule yet. + format: int64 + type: integer type: object required: - spec diff --git a/charts/karmada/_crds/bases/work.karmada.io_resourcebindings.yaml b/charts/karmada/_crds/bases/work.karmada.io_resourcebindings.yaml index ef5e1781a9c4..a6a1447bc4f1 100644 --- a/charts/karmada/_crds/bases/work.karmada.io_resourcebindings.yaml +++ b/charts/karmada/_crds/bases/work.karmada.io_resourcebindings.yaml @@ -264,6 +264,66 @@ spec: - name type: object type: array + gracefulEvictionTasks: + description: 'GracefulEvictionTasks holds the eviction tasks that + are expected to perform the eviction in a graceful way. The intended + workflow is: 1. Once the controller(such as ''taint-manager'') decided + to evict the resource that is referenced by current ResourceBinding + or ClusterResourceBinding from a target cluster, it removes(or scale + down the replicas) the target from Clusters(.spec.Clusters) and + builds a graceful eviction task. 2. The scheduler may perform a + re-scheduler and probably select a substitute cluster to take over + the evicting workload(resource). 3. The graceful eviction controller + takes care of the graceful eviction tasks and performs the final + removal after the workload(resource) is available on the substitute + cluster or exceed the grace termination period(defaults to 10 minutes).' + items: + description: GracefulEvictionTask represents a graceful eviction + task. + properties: + creationTimestamp: + description: "CreationTimestamp is a timestamp representing + the server time when this object was created. Clients should + not set this value to avoid the time inconsistency issue. + It is represented in RFC3339 form(like '2021-04-25T10:02:10Z') + and is in UTC. \n Populated by the system. Read-only." + format: date-time + type: string + fromCluster: + description: FromCluster which cluster the eviction perform + from. + type: string + message: + description: Message is a human-readable message indicating + details about the eviction. This may be an empty string. + maxLength: 1024 + type: string + producer: + description: Producer indicates the controller who triggered + the eviction. + type: string + reason: + description: Reason contains a programmatic identifier indicating + the reason for the eviction. Producers may define expected + values and meanings for this field, and whether the values + are considered a guaranteed API. The value should be a CamelCase + string. This field may not be empty. + maxLength: 32 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + replicas: + description: Replicas indicates the number of replicas should + be evicted. Should be ignored for resource type that doesn't + have replica. + format: int32 + type: integer + required: + - fromCluster + - producer + - reason + type: object + type: array propagateDeps: description: PropagateDeps tells if relevant resources should be propagated automatically. It is inherited from PropagationPolicy or ClusterPropagationPolicy. @@ -609,6 +669,13 @@ spec: - type type: object type: array + schedulerObservedGeneration: + description: SchedulerObservedGeneration is the generation(.metadata.generation) + observed by the scheduler. If SchedulerObservedGeneration is less + than the generation in metadata means the scheduler hasn't confirmed + the scheduling result or hasn't done the schedule yet. + format: int64 + type: integer type: object required: - spec diff --git a/pkg/apis/work/v1alpha2/binding_types.go b/pkg/apis/work/v1alpha2/binding_types.go index c2ba69e96b05..e4747b5b82e4 100644 --- a/pkg/apis/work/v1alpha2/binding_types.go +++ b/pkg/apis/work/v1alpha2/binding_types.go @@ -72,6 +72,22 @@ type ResourceBindingSpec struct { // +optional Clusters []TargetCluster `json:"clusters,omitempty"` + // GracefulEvictionTasks holds the eviction tasks that are expected to perform + // the eviction in a graceful way. + // The intended workflow is: + // 1. Once the controller(such as 'taint-manager') decided to evict the resource that + // is referenced by current ResourceBinding or ClusterResourceBinding from a target + // cluster, it removes(or scale down the replicas) the target from Clusters(.spec.Clusters) + // and builds a graceful eviction task. + // 2. The scheduler may perform a re-scheduler and probably select a substitute cluster + // to take over the evicting workload(resource). + // 3. The graceful eviction controller takes care of the graceful eviction tasks and + // performs the final removal after the workload(resource) is available on the substitute + // cluster or exceed the grace termination period(defaults to 10 minutes). + // + // +optional + GracefulEvictionTasks []GracefulEvictionTask `json:"gracefulEvictionTasks,omitempty"` + // RequiredBy represents the list of Bindings that depend on the referencing resource. // +optional RequiredBy []BindingSnapshot `json:"requiredBy,omitempty"` @@ -142,6 +158,48 @@ type TargetCluster struct { Replicas int32 `json:"replicas,omitempty"` } +// GracefulEvictionTask represents a graceful eviction task. +type GracefulEvictionTask struct { + // FromCluster which cluster the eviction perform from. + // +required + FromCluster string `json:"fromCluster"` + + // Replicas indicates the number of replicas should be evicted. + // Should be ignored for resource type that doesn't have replica. + // +optional + Replicas *int32 `json:"replicas,omitempty"` + + // Reason contains a programmatic identifier indicating the reason for the eviction. + // Producers may define expected values and meanings for this field, + // and whether the values are considered a guaranteed API. + // The value should be a CamelCase string. + // This field may not be empty. + // +required + // +kubebuilder:validation:MaxLength=32 + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:Pattern=`^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$` + Reason string `json:"reason"` + + // Message is a human-readable message indicating details about the eviction. + // This may be an empty string. + // +optional + // +kubebuilder:validation:MaxLength=1024 + Message string `json:"message,omitempty"` + + // Producer indicates the controller who triggered the eviction. + // +required + Producer string `json:"producer"` + + // CreationTimestamp is a timestamp representing the server time when this object was + // created. + // Clients should not set this value to avoid the time inconsistency issue. + // It is represented in RFC3339 form(like '2021-04-25T10:02:10Z') and is in UTC. + // + // Populated by the system. Read-only. + // +optional + CreationTimestamp metav1.Time `json:"creationTimestamp,omitempty"` +} + // BindingSnapshot is a snapshot of a ResourceBinding or ClusterResourceBinding. type BindingSnapshot struct { // Namespace represents the namespace of the Binding. @@ -161,6 +219,11 @@ type BindingSnapshot struct { // ResourceBindingStatus represents the overall status of the strategy as well as the referenced resources. type ResourceBindingStatus struct { + // SchedulerObservedGeneration is the generation(.metadata.generation) observed by the scheduler. + // If SchedulerObservedGeneration is less than the generation in metadata means the scheduler hasn't confirmed + // the scheduling result or hasn't done the schedule yet. + // +optional + SchedulerObservedGeneration int64 `json:"schedulerObservedGeneration,omitempty"` // Conditions contain the different condition statuses. // +optional Conditions []metav1.Condition `json:"conditions,omitempty"` diff --git a/pkg/apis/work/v1alpha2/well_known_constants.go b/pkg/apis/work/v1alpha2/well_known_constants.go index 902856fe3a7c..abd7b8fbc32b 100644 --- a/pkg/apis/work/v1alpha2/well_known_constants.go +++ b/pkg/apis/work/v1alpha2/well_known_constants.go @@ -59,3 +59,16 @@ const ( // https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids ResourceTemplateUIDAnnotation = "resourcetemplate.karmada.io/uid" ) + +// Define eviction reasons. +const ( + // EvictionReasonTaintUntolerated describes the eviction is triggered + // because can not tolerate taint or exceed toleration period of time. + EvictionReasonTaintUntolerated = "TaintUntolerated" +) + +// Define eviction producers. +const ( + // EvictionProducerTaintManager represents the name of taint manager. + EvictionProducerTaintManager = "TaintManager" +) diff --git a/pkg/apis/work/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/work/v1alpha2/zz_generated.deepcopy.go index d9476bd5bee3..141396ae7556 100644 --- a/pkg/apis/work/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/work/v1alpha2/zz_generated.deepcopy.go @@ -114,6 +114,28 @@ func (in *ClusterResourceBindingList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GracefulEvictionTask) DeepCopyInto(out *GracefulEvictionTask) { + *out = *in + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } + in.CreationTimestamp.DeepCopyInto(&out.CreationTimestamp) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GracefulEvictionTask. +func (in *GracefulEvictionTask) DeepCopy() *GracefulEvictionTask { + if in == nil { + return nil + } + out := new(GracefulEvictionTask) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NodeClaim) DeepCopyInto(out *NodeClaim) { *out = *in @@ -268,6 +290,13 @@ func (in *ResourceBindingSpec) DeepCopyInto(out *ResourceBindingSpec) { *out = make([]TargetCluster, len(*in)) copy(*out, *in) } + if in.GracefulEvictionTasks != nil { + in, out := &in.GracefulEvictionTasks, &out.GracefulEvictionTasks + *out = make([]GracefulEvictionTask, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.RequiredBy != nil { in, out := &in.RequiredBy, &out.RequiredBy *out = make([]BindingSnapshot, len(*in)) diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index 35fa44135c62..67496c412a3a 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -98,6 +98,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.BindingSnapshot": schema_pkg_apis_work_v1alpha2_BindingSnapshot(ref), "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.ClusterResourceBinding": schema_pkg_apis_work_v1alpha2_ClusterResourceBinding(ref), "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.ClusterResourceBindingList": schema_pkg_apis_work_v1alpha2_ClusterResourceBindingList(ref), + "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.GracefulEvictionTask": schema_pkg_apis_work_v1alpha2_GracefulEvictionTask(ref), "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.NodeClaim": schema_pkg_apis_work_v1alpha2_NodeClaim(ref), "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.ObjectReference": schema_pkg_apis_work_v1alpha2_ObjectReference(ref), "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.ReplicaRequirements": schema_pkg_apis_work_v1alpha2_ReplicaRequirements(ref), @@ -4217,6 +4218,67 @@ func schema_pkg_apis_work_v1alpha2_ClusterResourceBindingList(ref common.Referen } } +func schema_pkg_apis_work_v1alpha2_GracefulEvictionTask(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "GracefulEvictionTask represents a graceful eviction task.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "fromCluster": { + SchemaProps: spec.SchemaProps{ + Description: "FromCluster which cluster the eviction perform from.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "replicas": { + SchemaProps: spec.SchemaProps{ + Description: "Replicas indicates the number of replicas should be evicted. Should be ignored for resource type that doesn't have replica.", + Type: []string{"integer"}, + Format: "int32", + }, + }, + "reason": { + SchemaProps: spec.SchemaProps{ + Description: "Reason contains a programmatic identifier indicating the reason for the eviction. Producers may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "message": { + SchemaProps: spec.SchemaProps{ + Description: "Message is a human-readable message indicating details about the eviction. This may be an empty string.", + Type: []string{"string"}, + Format: "", + }, + }, + "producer": { + SchemaProps: spec.SchemaProps{ + Description: "Producer indicates the controller who triggered the eviction.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "creationTimestamp": { + SchemaProps: spec.SchemaProps{ + Description: "CreationTimestamp is a timestamp representing the server time when this object was created. Clients should not set this value to avoid the time inconsistency issue. It is represented in RFC3339 form(like '2021-04-25T10:02:10Z') and is in UTC.\n\nPopulated by the system. Read-only.", + Default: map[string]interface{}{}, + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Time"), + }, + }, + }, + Required: []string{"fromCluster", "reason", "producer"}, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/apis/meta/v1.Time"}, + } +} + func schema_pkg_apis_work_v1alpha2_NodeClaim(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -4511,6 +4573,20 @@ func schema_pkg_apis_work_v1alpha2_ResourceBindingSpec(ref common.ReferenceCallb }, }, }, + "gracefulEvictionTasks": { + SchemaProps: spec.SchemaProps{ + Description: "GracefulEvictionTasks holds the eviction tasks that are expected to perform the eviction in a graceful way. The intended workflow is: 1. Once the controller(such as 'taint-manager') decided to evict the resource that\n is referenced by current ResourceBinding or ClusterResourceBinding from a target\n cluster, it removes(or scale down the replicas) the target from Clusters(.spec.Clusters)\n and builds a graceful eviction task.\n2. The scheduler may perform a re-scheduler and probably select a substitute cluster\n to take over the evicting workload(resource).\n3. The graceful eviction controller takes care of the graceful eviction tasks and\n performs the final removal after the workload(resource) is available on the substitute\n cluster or exceed the grace termination period(defaults to 10 minutes).", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.GracefulEvictionTask"), + }, + }, + }, + }, + }, "requiredBy": { SchemaProps: spec.SchemaProps{ Description: "RequiredBy represents the list of Bindings that depend on the referencing resource.", @@ -4530,7 +4606,7 @@ func schema_pkg_apis_work_v1alpha2_ResourceBindingSpec(ref common.ReferenceCallb }, }, Dependencies: []string{ - "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.BindingSnapshot", "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.ObjectReference", "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.ReplicaRequirements", "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.TargetCluster"}, + "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.BindingSnapshot", "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.GracefulEvictionTask", "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.ObjectReference", "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.ReplicaRequirements", "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2.TargetCluster"}, } } @@ -4541,6 +4617,13 @@ func schema_pkg_apis_work_v1alpha2_ResourceBindingStatus(ref common.ReferenceCal Description: "ResourceBindingStatus represents the overall status of the strategy as well as the referenced resources.", Type: []string{"object"}, Properties: map[string]spec.Schema{ + "schedulerObservedGeneration": { + SchemaProps: spec.SchemaProps{ + Description: "SchedulerObservedGeneration is the generation(.metadata.generation) observed by the scheduler. If SchedulerObservedGeneration is less than the generation in metadata means the scheduler hasn't confirmed the scheduling result or hasn't done the schedule yet.", + Type: []string{"integer"}, + Format: "int64", + }, + }, "conditions": { SchemaProps: spec.SchemaProps{ Description: "Conditions contain the different condition statuses.",