-
Notifications
You must be signed in to change notification settings - Fork 57
/
node_group.go
271 lines (225 loc) · 11 KB
/
node_group.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
package controller
import (
"fmt"
"io"
"time"
"github.com/atlassian/escalator/pkg/k8s"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/yaml"
v1lister "k8s.io/client-go/listers/core/v1"
)
// DefaultNodeGroup is used for any pods that don't have a node selector defined
const DefaultNodeGroup = "default"
// NodeGroupOptions represents a nodegroup running on our cluster
// We differentiate nodegroups by their node label
type NodeGroupOptions struct {
Name string `json:"name,omitempty" yaml:"name,omitempty"`
LabelKey string `json:"label_key,omitempty" yaml:"label_key,omitempty"`
LabelValue string `json:"label_value,omitempty" yaml:"label_value,omitempty"`
CloudProviderGroupName string `json:"cloud_provider_group_name,omitempty" yaml:"cloud_provider_group_name,omitempty"`
MinNodes int `json:"min_nodes,omitempty" yaml:"min_nodes,omitempty"`
MaxNodes int `json:"max_nodes,omitempty" yaml:"max_nodes,omitempty"`
DryMode bool `json:"dry_mode,omitempty" yaml:"dry_mode,omitempty"`
TaintUpperCapacityThresholdPercent int `json:"taint_upper_capacity_threshold_percent,omitempty" yaml:"taint_upper_capacity_threshold_percent,omitempty"`
TaintLowerCapacityThresholdPercent int `json:"taint_lower_capacity_threshold_percent,omitempty" yaml:"taint_lower_capacity_threshold_percent,omitempty"`
ScaleUpThresholdPercent int `json:"scale_up_threshold_percent,omitempty" yaml:"scale_up_threshold_percent,omitempty"`
SlowNodeRemovalRate int `json:"slow_node_removal_rate,omitempty" yaml:"slow_node_removal_rate,omitempty"`
FastNodeRemovalRate int `json:"fast_node_removal_rate,omitempty" yaml:"fast_node_removal_rate,omitempty"`
SoftDeleteGracePeriod string `json:"soft_delete_grace_period,omitempty" yaml:"soft_delete_grace_period,omitempty"`
HardDeleteGracePeriod string `json:"hard_delete_grace_period,omitempty" yaml:"soft_delete_grace_period,omitempty"`
ScaleUpCoolDownPeriod string `json:"scale_up_cool_down_period,omitempty" yaml:"scale_up_cool_down_period,omitempty"`
TaintEffect v1.TaintEffect `json:"taint_effect,omitempty" yaml:"taint_effect,omitempty"`
// Private variables for storing the parsed duration from the string
softDeleteGracePeriodDuration time.Duration
hardDeleteGracePeriodDuration time.Duration
scaleUpCoolDownPeriodDuration time.Duration
}
// UnmarshalNodeGroupOptions decodes the yaml or json reader into a struct
func UnmarshalNodeGroupOptions(reader io.Reader) ([]NodeGroupOptions, error) {
var wrapper struct {
NodeGroups []NodeGroupOptions `json:"node_groups" yaml:"node_groups"`
}
if err := yaml.NewYAMLOrJSONDecoder(reader, 4096).Decode(&wrapper); err != nil {
return []NodeGroupOptions{}, err
}
return wrapper.NodeGroups, nil
}
// ValidateNodeGroup is a safety check to validate that a nodegroup has valid options
func ValidateNodeGroup(nodegroup NodeGroupOptions) []error {
var problems []error
checkThat := func(cond bool, format string, output ...interface{}) {
if !cond {
problems = append(problems, fmt.Errorf(format, output...))
}
}
checkThat(len(nodegroup.Name) > 0, "name cannot be empty")
checkThat(len(nodegroup.LabelKey) > 0, "label_key cannot be empty")
checkThat(len(nodegroup.LabelValue) > 0, "label_value cannot be empty")
checkThat(len(nodegroup.CloudProviderGroupName) > 0, "cloud_provider_group_name cannot be empty")
checkThat(nodegroup.TaintUpperCapacityThresholdPercent > 0, "taint_upper_capacity_threshold_percent must be larger than 0")
checkThat(nodegroup.TaintLowerCapacityThresholdPercent > 0, "taint_lower_capacity_threshold_percent must be larger than 0")
checkThat(nodegroup.ScaleUpThresholdPercent > 0, "scale_up_threshold_percent must be larger than 0")
checkThat(nodegroup.TaintLowerCapacityThresholdPercent < nodegroup.TaintUpperCapacityThresholdPercent,
"taint_lower_capacity_threshold_percent must be less than taint_upper_capacity_threshold_percent")
checkThat(nodegroup.TaintUpperCapacityThresholdPercent < nodegroup.ScaleUpThresholdPercent,
"taint_upper_capacity_threshold_percent must be less than scale_up_threshold_percent")
// Allow exclusion of the MinNodes and MaxNodes options so that we can "auto discover" them from the cloud provider
if !nodegroup.autoDiscoverMinMaxNodeOptions() {
checkThat(nodegroup.MinNodes < nodegroup.MaxNodes, "min_nodes must be less than max_nodes")
checkThat(nodegroup.MaxNodes > 0, "max_nodes must be larger than 0")
checkThat(nodegroup.MinNodes > 0, "min_nodes must be larger than 0")
}
checkThat(nodegroup.SlowNodeRemovalRate <= nodegroup.FastNodeRemovalRate, "slow_node_removal_rate must be less than fast_node_removal_rate")
checkThat(len(nodegroup.SoftDeleteGracePeriod) > 0, "soft_delete_grace_period must not be empty")
checkThat(len(nodegroup.HardDeleteGracePeriod) > 0, "hard_delete_grace_period must not be empty")
checkThat(nodegroup.SoftDeleteGracePeriodDuration() > 0, "soft_delete_grace_period failed to parse into a time.Duration. check your formatting.")
checkThat(nodegroup.HardDeleteGracePeriodDuration() > 0, "hard_delete_grace_period failed to parse into a time.Duration. check your formatting.")
checkThat(nodegroup.SoftDeleteGracePeriodDuration() < nodegroup.HardDeleteGracePeriodDuration(), "soft_delete_grace_period must be less than hard_delete_grace_period")
checkThat(len(nodegroup.ScaleUpCoolDownPeriod) > 0, "scale_up_cool_down_period must not be empty")
checkThat(nodegroup.ScaleUpCoolDownPeriodDuration() > 0, "soft_delete_grace_period failed to parse into a time.Duration. check your formatting.")
checkThat(validTaintEffect(nodegroup.TaintEffect), "taint_effect must be valid kubernetes taint")
return problems
}
// Empty String is valid value for TaintEffect as AddToBeRemovedTaint method will default to NoSchedule
func validTaintEffect(taintEffect v1.TaintEffect) bool {
return len(taintEffect) == 0 || k8s.TaintEffectTypes[taintEffect]
}
// SoftDeleteGracePeriodDuration lazily returns/parses the softDeleteGracePeriod string into a duration
func (n *NodeGroupOptions) SoftDeleteGracePeriodDuration() time.Duration {
if n.softDeleteGracePeriodDuration == 0 {
duration, err := time.ParseDuration(n.SoftDeleteGracePeriod)
if err != nil {
return 0
}
n.softDeleteGracePeriodDuration = duration
}
return n.softDeleteGracePeriodDuration
}
// HardDeleteGracePeriodDuration lazily returns/parses the hardDeleteGracePeriodDuration string into a duration
func (n *NodeGroupOptions) HardDeleteGracePeriodDuration() time.Duration {
if n.hardDeleteGracePeriodDuration == 0 {
duration, err := time.ParseDuration(n.HardDeleteGracePeriod)
if err != nil {
return 0
}
n.hardDeleteGracePeriodDuration = duration
}
return n.hardDeleteGracePeriodDuration
}
// ScaleUpCoolDownPeriodDuration lazily returns/parses the scaleUpCoolDownPeriod string into a duration
func (n *NodeGroupOptions) ScaleUpCoolDownPeriodDuration() time.Duration {
if n.scaleUpCoolDownPeriodDuration == 0 {
duration, err := time.ParseDuration(n.ScaleUpCoolDownPeriod)
if err != nil {
return 0
}
n.scaleUpCoolDownPeriodDuration = duration
}
return n.scaleUpCoolDownPeriodDuration
}
// autoDiscoverMinMaxNodeOptions returns whether the min_nodes and max_nodes options should be "auto-discovered" from the cloud provider
func (n *NodeGroupOptions) autoDiscoverMinMaxNodeOptions() bool {
return n.MinNodes == 0 && n.MaxNodes == 0
}
// NodeGroupLister is just a light wrapper around a pod lister and node lister
// Used for grouping a nodegroup and their listers
type NodeGroupLister struct {
// Pod lister
Pods k8s.PodLister
// Node lister
Nodes k8s.NodeLister
}
// NewPodAffinityFilterFunc creates a new PodFilterFunc based on filtering by label selectors
func NewPodAffinityFilterFunc(labelKey, labelValue string) k8s.PodFilterFunc {
return func(pod *v1.Pod) bool {
// filter out daemonsets
if k8s.PodIsDaemonSet(pod) {
return false
}
// check the node selector
if value, ok := pod.Spec.NodeSelector[labelKey]; ok {
if value == labelValue {
return true
}
}
// finally, if the pod has an affinity for our selector then we will include it
if pod.Spec.Affinity != nil && pod.Spec.Affinity.NodeAffinity != nil && pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil {
if pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms != nil {
for _, term := range pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms {
for _, expression := range term.MatchExpressions {
if expression.Key == labelKey {
for _, value := range expression.Values {
if value == labelValue {
return true
}
}
}
}
}
}
}
return false
}
}
// NewPodDefaultFilterFunc creates a new PodFilterFunc that includes pods that do not have a selector
func NewPodDefaultFilterFunc() k8s.PodFilterFunc {
return func(pod *v1.Pod) bool {
// filter out daemonsets
if k8s.PodIsDaemonSet(pod) {
return false
}
// filter out static pods
if k8s.PodIsStatic(pod) {
return false
}
// Only include pods that pass the following:
// - Don't have a nodeSelector
// - Don't have an affinity
return len(pod.Spec.NodeSelector) == 0 && pod.Spec.Affinity == nil
}
}
// NewNodeLabelFilterFunc creates a new NodeFilterFunc based on filtering by node labels
func NewNodeLabelFilterFunc(labelKey, labelValue string) k8s.NodeFilterFunc {
return func(node *v1.Node) bool {
if value, ok := node.ObjectMeta.Labels[labelKey]; ok {
if value == labelValue {
return true
}
}
return false
}
}
// NewNodeGroupLister creates a new group from the backing lister and nodegroup filter
func NewNodeGroupLister(allPodsLister v1lister.PodLister, allNodesLister v1lister.NodeLister, nodeGroup NodeGroupOptions) *NodeGroupLister {
return &NodeGroupLister{
k8s.NewFilteredPodsLister(allPodsLister, NewPodAffinityFilterFunc(nodeGroup.LabelKey, nodeGroup.LabelValue)),
k8s.NewFilteredNodesLister(allNodesLister, NewNodeLabelFilterFunc(nodeGroup.LabelKey, nodeGroup.LabelValue)),
}
}
// NewDefaultNodeGroupLister creates a new group from the backing lister and nodegroup filter with the default filter
func NewDefaultNodeGroupLister(allPodsLister v1lister.PodLister, allNodesLister v1lister.NodeLister, nodeGroup NodeGroupOptions) *NodeGroupLister {
return &NodeGroupLister{
k8s.NewFilteredPodsLister(allPodsLister, NewPodDefaultFilterFunc()),
k8s.NewFilteredNodesLister(allNodesLister, NewNodeLabelFilterFunc(nodeGroup.LabelKey, nodeGroup.LabelValue)),
}
}
type nodeGroupsStateOpts struct {
nodeGroups []NodeGroupOptions
client Client
}
// BuildNodeGroupsState builds a node group state
func BuildNodeGroupsState(opts nodeGroupsStateOpts) map[string]*NodeGroupState {
nodeGroupsState := make(map[string]*NodeGroupState)
for _, ng := range opts.nodeGroups {
nodeGroupsState[ng.Name] = &NodeGroupState{
Opts: ng,
NodeGroupLister: opts.client.Listers[ng.Name],
// Setup the scaleLock timeouts for this nodegroup
scaleUpLock: scaleLock{
minimumLockDuration: ng.ScaleUpCoolDownPeriodDuration(),
nodegroup: ng.Name,
},
}
}
return nodeGroupsState
}