From 1c25d7cd14020e2717071274fcdf60a771880d5f Mon Sep 17 00:00:00 2001 From: Ryan Faircloth Date: Tue, 14 Feb 2023 07:50:03 -0500 Subject: [PATCH] feat: Add TopologySpreadConstraints support to the humiocluster CRD This PR fixes #342 with the addition of TopologySpreadConstraints to the CR. Node instances of operator deployed by helm will not automatically update the CR for those instances the CR must be applied manually --- api/v1alpha1/humiocluster_types.go | 3 + api/v1alpha1/zz_generated.deepcopy.go | 7 + .../crds/core.humio.com_humioclusters.yaml | 211 ++++++++++++++++++ .../bases/core.humio.com_humioclusters.yaml | 211 ++++++++++++++++++ controllers/humiocluster_defaults.go | 6 + controllers/humiocluster_pods.go | 3 + 6 files changed, 441 insertions(+) diff --git a/api/v1alpha1/humiocluster_types.go b/api/v1alpha1/humiocluster_types.go index 815d5a32..0bb498e7 100644 --- a/api/v1alpha1/humiocluster_types.go +++ b/api/v1alpha1/humiocluster_types.go @@ -176,6 +176,9 @@ type HumioNodeSpec struct { // Tolerations defines the tolerations that will be attached to the humio pods Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + // TopologySpreadConstraints defines the topologySpreadConstraints that will be attached to the humio pods + TopologySpreadConstraints []corev1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"` + // SidecarContainers can be used in advanced use-cases where you want one or more sidecar container added to the // Humio pod to help out in debugging purposes. SidecarContainers []corev1.Container `json:"sidecarContainer,omitempty"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 6a4bb571..d27416a1 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -991,6 +991,13 @@ func (in *HumioNodeSpec) DeepCopyInto(out *HumioNodeSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.TopologySpreadConstraints != nil { + in, out := &in.TopologySpreadConstraints, &out.TopologySpreadConstraints + *out = make([]v1.TopologySpreadConstraint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.SidecarContainers != nil { in, out := &in.SidecarContainers, &out.SidecarContainers *out = make([]v1.Container, len(*in)) diff --git a/charts/humio-operator/crds/core.humio.com_humioclusters.yaml b/charts/humio-operator/crds/core.humio.com_humioclusters.yaml index fb50d2f7..02d3862a 100644 --- a/charts/humio-operator/crds/core.humio.com_humioclusters.yaml +++ b/charts/humio-operator/crds/core.humio.com_humioclusters.yaml @@ -11938,6 +11938,116 @@ spec: type: string type: object type: array + topologySpreadConstraints: + description: TopologySpreadConstraints defines the topologySpreadConstraints + that will be attached to the humio pods + items: + description: TopologySpreadConstraint specifies how to + spread matching pods among the given topology. + properties: + labelSelector: + description: LabelSelector is used to find matching + pods. Pods that match this label selector are counted + to determine the number of pods in their corresponding + topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement is + a selector that contains values, a key, and + an operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + the values array must be empty. This array + is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + maxSkew: + description: 'MaxSkew describes the degree to which + pods may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, + it is the maximum permitted difference between the + number of matching pods in the target topology and + the global minimum. For example, in a 3-zone cluster, + MaxSkew is set to 1, and pods with the same labelSelector + spread as 1/1/0: | zone1 | zone2 | zone3 | | P | P | | + - if MaxSkew is 1, incoming pod can only be scheduled + to zone3 to become 1/1/1; scheduling it onto zone1(zone2) + would make the ActualSkew(2-0) on zone1(zone2) violate + MaxSkew(1). - if MaxSkew is 2, incoming pod can + be scheduled onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, + it is used to give higher precedence to topologies + that satisfy it. It''s a required field. Default + value is 1 and 0 is not allowed.' + format: int32 + type: integer + topologyKey: + description: TopologyKey is the key of node labels. + Nodes that have a label with this key and identical + values are considered to be in the same topology. + We consider each as a "bucket", and + try to put balanced number of pods into each bucket. + It's a required field. + type: string + whenUnsatisfiable: + description: 'WhenUnsatisfiable indicates how to deal + with a pod if it doesn''t satisfy the spread constraint. + - DoNotSchedule (default) tells the scheduler not + to schedule it. - ScheduleAnyway tells the scheduler + to schedule the pod in any location, but giving + higher precedence to topologies that would help + reduce the skew. A constraint is considered "Unsatisfiable" + for an incoming pod if and only if every possible + node assignment for that pod would violate "MaxSkew" + on some topology. For example, in a 3-zone cluster, + MaxSkew is set to 1, and pods with the same labelSelector + spread as 3/1/1: | zone1 | zone2 | zone3 | | P P + P | P | P | If WhenUnsatisfiable is set + to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) + on zone2(zone3) satisfies MaxSkew(1). In other words, + the cluster can still be imbalanced, but scheduler + won''t make it *more* imbalanced. It''s a required + field.' + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array updateStrategy: description: UpdateStrategy controls how Humio pods are updated when changes are made to the HumioCluster resource @@ -13490,6 +13600,107 @@ spec: type: string type: object type: array + topologySpreadConstraints: + description: TopologySpreadConstraints defines the topologySpreadConstraints + that will be attached to the humio pods + items: + description: TopologySpreadConstraint specifies how to spread matching + pods among the given topology. + properties: + labelSelector: + description: LabelSelector is used to find matching pods. Pods + that match this label selector are counted to determine the + number of pods in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists or + DoesNotExist, the values array must be empty. This + array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is + "key", the operator is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + maxSkew: + description: 'MaxSkew describes the degree to which pods may + be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, + it is the maximum permitted difference between the number + of matching pods in the target topology and the global minimum. + For example, in a 3-zone cluster, MaxSkew is set to 1, and + pods with the same labelSelector spread as 1/1/0: | zone1 + | zone2 | zone3 | | P | P | | - if MaxSkew is + 1, incoming pod can only be scheduled to zone3 to become 1/1/1; + scheduling it onto zone1(zone2) would make the ActualSkew(2-0) + on zone1(zone2) violate MaxSkew(1). - if MaxSkew is 2, incoming + pod can be scheduled onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, + it is used to give higher precedence to topologies that satisfy + it. It''s a required field. Default value is 1 and 0 is not + allowed.' + format: int32 + type: integer + topologyKey: + description: TopologyKey is the key of node labels. Nodes that + have a label with this key and identical values are considered + to be in the same topology. We consider each + as a "bucket", and try to put balanced number of pods into + each bucket. It's a required field. + type: string + whenUnsatisfiable: + description: 'WhenUnsatisfiable indicates how to deal with a + pod if it doesn''t satisfy the spread constraint. - DoNotSchedule + (default) tells the scheduler not to schedule it. - ScheduleAnyway + tells the scheduler to schedule the pod in any location, but + giving higher precedence to topologies that would help reduce + the skew. A constraint is considered "Unsatisfiable" for + an incoming pod if and only if every possible node assignment + for that pod would violate "MaxSkew" on some topology. For + example, in a 3-zone cluster, MaxSkew is set to 1, and pods + with the same labelSelector spread as 3/1/1: | zone1 | zone2 + | zone3 | | P P P | P | P | If WhenUnsatisfiable is + set to DoNotSchedule, incoming pod can only be scheduled to + zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on + zone2(zone3) satisfies MaxSkew(1). In other words, the cluster + can still be imbalanced, but scheduler won''t make it *more* + imbalanced. It''s a required field.' + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array updateStrategy: description: UpdateStrategy controls how Humio pods are updated when changes are made to the HumioCluster resource that results in a diff --git a/config/crd/bases/core.humio.com_humioclusters.yaml b/config/crd/bases/core.humio.com_humioclusters.yaml index fb50d2f7..02d3862a 100644 --- a/config/crd/bases/core.humio.com_humioclusters.yaml +++ b/config/crd/bases/core.humio.com_humioclusters.yaml @@ -11938,6 +11938,116 @@ spec: type: string type: object type: array + topologySpreadConstraints: + description: TopologySpreadConstraints defines the topologySpreadConstraints + that will be attached to the humio pods + items: + description: TopologySpreadConstraint specifies how to + spread matching pods among the given topology. + properties: + labelSelector: + description: LabelSelector is used to find matching + pods. Pods that match this label selector are counted + to determine the number of pods in their corresponding + topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement is + a selector that contains values, a key, and + an operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + the values array must be empty. This array + is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + maxSkew: + description: 'MaxSkew describes the degree to which + pods may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, + it is the maximum permitted difference between the + number of matching pods in the target topology and + the global minimum. For example, in a 3-zone cluster, + MaxSkew is set to 1, and pods with the same labelSelector + spread as 1/1/0: | zone1 | zone2 | zone3 | | P | P | | + - if MaxSkew is 1, incoming pod can only be scheduled + to zone3 to become 1/1/1; scheduling it onto zone1(zone2) + would make the ActualSkew(2-0) on zone1(zone2) violate + MaxSkew(1). - if MaxSkew is 2, incoming pod can + be scheduled onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, + it is used to give higher precedence to topologies + that satisfy it. It''s a required field. Default + value is 1 and 0 is not allowed.' + format: int32 + type: integer + topologyKey: + description: TopologyKey is the key of node labels. + Nodes that have a label with this key and identical + values are considered to be in the same topology. + We consider each as a "bucket", and + try to put balanced number of pods into each bucket. + It's a required field. + type: string + whenUnsatisfiable: + description: 'WhenUnsatisfiable indicates how to deal + with a pod if it doesn''t satisfy the spread constraint. + - DoNotSchedule (default) tells the scheduler not + to schedule it. - ScheduleAnyway tells the scheduler + to schedule the pod in any location, but giving + higher precedence to topologies that would help + reduce the skew. A constraint is considered "Unsatisfiable" + for an incoming pod if and only if every possible + node assignment for that pod would violate "MaxSkew" + on some topology. For example, in a 3-zone cluster, + MaxSkew is set to 1, and pods with the same labelSelector + spread as 3/1/1: | zone1 | zone2 | zone3 | | P P + P | P | P | If WhenUnsatisfiable is set + to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) + on zone2(zone3) satisfies MaxSkew(1). In other words, + the cluster can still be imbalanced, but scheduler + won''t make it *more* imbalanced. It''s a required + field.' + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array updateStrategy: description: UpdateStrategy controls how Humio pods are updated when changes are made to the HumioCluster resource @@ -13490,6 +13600,107 @@ spec: type: string type: object type: array + topologySpreadConstraints: + description: TopologySpreadConstraints defines the topologySpreadConstraints + that will be attached to the humio pods + items: + description: TopologySpreadConstraint specifies how to spread matching + pods among the given topology. + properties: + labelSelector: + description: LabelSelector is used to find matching pods. Pods + that match this label selector are counted to determine the + number of pods in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists or + DoesNotExist, the values array must be empty. This + array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is + "key", the operator is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + maxSkew: + description: 'MaxSkew describes the degree to which pods may + be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, + it is the maximum permitted difference between the number + of matching pods in the target topology and the global minimum. + For example, in a 3-zone cluster, MaxSkew is set to 1, and + pods with the same labelSelector spread as 1/1/0: | zone1 + | zone2 | zone3 | | P | P | | - if MaxSkew is + 1, incoming pod can only be scheduled to zone3 to become 1/1/1; + scheduling it onto zone1(zone2) would make the ActualSkew(2-0) + on zone1(zone2) violate MaxSkew(1). - if MaxSkew is 2, incoming + pod can be scheduled onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, + it is used to give higher precedence to topologies that satisfy + it. It''s a required field. Default value is 1 and 0 is not + allowed.' + format: int32 + type: integer + topologyKey: + description: TopologyKey is the key of node labels. Nodes that + have a label with this key and identical values are considered + to be in the same topology. We consider each + as a "bucket", and try to put balanced number of pods into + each bucket. It's a required field. + type: string + whenUnsatisfiable: + description: 'WhenUnsatisfiable indicates how to deal with a + pod if it doesn''t satisfy the spread constraint. - DoNotSchedule + (default) tells the scheduler not to schedule it. - ScheduleAnyway + tells the scheduler to schedule the pod in any location, but + giving higher precedence to topologies that would help reduce + the skew. A constraint is considered "Unsatisfiable" for + an incoming pod if and only if every possible node assignment + for that pod would violate "MaxSkew" on some topology. For + example, in a 3-zone cluster, MaxSkew is set to 1, and pods + with the same labelSelector spread as 3/1/1: | zone1 | zone2 + | zone3 | | P P P | P | P | If WhenUnsatisfiable is + set to DoNotSchedule, incoming pod can only be scheduled to + zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on + zone2(zone3) satisfies MaxSkew(1). In other words, the cluster + can still be imbalanced, but scheduler won''t make it *more* + imbalanced. It''s a required field.' + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array updateStrategy: description: UpdateStrategy controls how Humio pods are updated when changes are made to the HumioCluster resource that results in a diff --git a/controllers/humiocluster_defaults.go b/controllers/humiocluster_defaults.go index fc1291d4..3fd6032f 100644 --- a/controllers/humiocluster_defaults.go +++ b/controllers/humiocluster_defaults.go @@ -116,6 +116,7 @@ func NewHumioNodeManagerFromHumioCluster(hc *humiov1alpha1.HumioCluster) *HumioN PodSecurityContext: hc.Spec.PodSecurityContext, Resources: hc.Spec.Resources, Tolerations: hc.Spec.Tolerations, + TopologySpreadConstraints: hc.Spec.TopologySpreadConstraints, TerminationGracePeriodSeconds: hc.Spec.TerminationGracePeriodSeconds, Affinity: hc.Spec.Affinity, SidecarContainers: hc.Spec.SidecarContainers, @@ -177,6 +178,7 @@ func NewHumioNodeManagerFromHumioNodePool(hc *humiov1alpha1.HumioCluster, hnp *h PodSecurityContext: hnp.PodSecurityContext, Resources: hnp.Resources, Tolerations: hnp.Tolerations, + TopologySpreadConstraints: hnp.TopologySpreadConstraints, TerminationGracePeriodSeconds: hnp.TerminationGracePeriodSeconds, Affinity: hnp.Affinity, SidecarContainers: hnp.SidecarContainers, @@ -726,6 +728,10 @@ func (hnp HumioNodePool) GetTolerations() []corev1.Toleration { return hnp.humioNodeSpec.Tolerations } +func (hnp HumioNodePool) GetTopologySpreadConstraints() []corev1.TopologySpreadConstraint { + return hnp.humioNodeSpec.TopologySpreadConstraints +} + func (hnp HumioNodePool) GetResources() corev1.ResourceRequirements { return hnp.humioNodeSpec.Resources } diff --git a/controllers/humiocluster_pods.go b/controllers/humiocluster_pods.go index efe2f2d8..40c47444 100644 --- a/controllers/humiocluster_pods.go +++ b/controllers/humiocluster_pods.go @@ -303,6 +303,7 @@ func ConstructPod(hnp *HumioNodePool, humioNodeName string, attachments *podAtta }, Affinity: hnp.GetAffinity(), Tolerations: hnp.GetTolerations(), + TopologySpreadConstraints: hnp.GetTopologySpreadConstraints(), SecurityContext: hnp.GetPodSecurityContext(), TerminationGracePeriodSeconds: hnp.GetTerminationGracePeriodSeconds(), }, @@ -768,6 +769,8 @@ func sanitizePod(hnp *HumioNodePool, pod *corev1.Pod) *corev1.Pod { pod.Spec.PreemptionPolicy = nil pod.Spec.DeprecatedServiceAccount = "" pod.Spec.Tolerations = hnp.GetTolerations() + pod.Spec.TopologySpreadConstraints = hnp.GetTopologySpreadConstraints() + for i := range pod.Spec.InitContainers { pod.Spec.InitContainers[i].ImagePullPolicy = hnp.GetImagePullPolicy() pod.Spec.InitContainers[i].TerminationMessagePath = ""