forked from kubernetes-retired/contrib
-
Notifications
You must be signed in to change notification settings - Fork 0
/
scale_down.go
189 lines (159 loc) · 6.04 KB
/
scale_down.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"fmt"
"reflect"
"time"
"k8s.io/contrib/cluster-autoscaler/cloudprovider"
"k8s.io/contrib/cluster-autoscaler/simulator"
kube_api "k8s.io/kubernetes/pkg/api"
kube_client "k8s.io/kubernetes/pkg/client/unversioned"
"k8s.io/kubernetes/plugin/pkg/scheduler/schedulercache"
"github.com/golang/glog"
)
// ScaleDownResult represents the state of scale down.
type ScaleDownResult int
const (
// ScaleDownError - scale down finished with error.
ScaleDownError ScaleDownResult = iota
// ScaleDownNoUnneeded - no unneeded nodes and no errors.
ScaleDownNoUnneeded ScaleDownResult = iota
// ScaleDownNoNodeDeleted - unneeded nodes present but not available for deletion.
ScaleDownNoNodeDeleted ScaleDownResult = iota
// ScaleDownNodeDeleted - a node was deleted.
ScaleDownNodeDeleted ScaleDownResult = iota
)
// FindUnneededNodes calculates which nodes are not needed, i.e. all pods can be scheduled somewhere else,
// and updates unneededNodes map accordingly. It also returns information where pods can be rescheduld.
func FindUnneededNodes(nodes []*kube_api.Node,
unneededNodes map[string]time.Time,
utilizationThreshold float64,
pods []*kube_api.Pod,
predicateChecker *simulator.PredicateChecker,
oldHints map[string]string,
tracker *simulator.UsageTracker,
timestamp time.Time) (unnededTimeMap map[string]time.Time, podReschedulingHints map[string]string) {
currentlyUnneededNodes := make([]*kube_api.Node, 0)
nodeNameToNodeInfo := schedulercache.CreateNodeNameToInfoMap(pods)
// Phase1 - look at the nodes utilization.
for _, node := range nodes {
nodeInfo, found := nodeNameToNodeInfo[node.Name]
if !found {
glog.Errorf("Node info for %s not found", node.Name)
continue
}
utilization, err := simulator.CalculateUtilization(node, nodeInfo)
if err != nil {
glog.Warningf("Failed to calculate utilization for %s: %v", node.Name, err)
}
glog.V(4).Infof("Node %s - utilization %f", node.Name, utilization)
if utilization >= utilizationThreshold {
glog.V(4).Infof("Node %s is not suitable for removal - utilization to big (%f)", node.Name, utilization)
continue
}
currentlyUnneededNodes = append(currentlyUnneededNodes, node)
}
// Phase2 - check which nodes can be probably removed using fast drain.
nodesToRemove, newHints, err := simulator.FindNodesToRemove(currentlyUnneededNodes, nodes, pods,
nil, predicateChecker,
len(currentlyUnneededNodes), true, oldHints, tracker, timestamp)
if err != nil {
glog.Errorf("Error while simulating node drains: %v", err)
return map[string]time.Time{}, oldHints
}
// Update the timestamp map.
now := time.Now()
result := make(map[string]time.Time)
for _, node := range nodesToRemove {
name := node.Name
if val, found := unneededNodes[name]; !found {
result[name] = now
} else {
result[name] = val
}
}
return result, newHints
}
// ScaleDown tries to scale down the cluster. It returns ScaleDownResult indicating if any node was
// removed and error if such occured.
func ScaleDown(
nodes []*kube_api.Node,
unneededNodes map[string]time.Time,
unneededTime time.Duration,
pods []*kube_api.Pod,
cloudProvider cloudprovider.CloudProvider,
client *kube_client.Client,
predicateChecker *simulator.PredicateChecker,
oldHints map[string]string,
usageTracker *simulator.UsageTracker) (ScaleDownResult, error) {
now := time.Now()
candidates := make([]*kube_api.Node, 0)
for _, node := range nodes {
if val, found := unneededNodes[node.Name]; found {
glog.V(2).Infof("%s was unneeded for %s", node.Name, now.Sub(val).String())
// Check how long the node was underutilized.
if !val.Add(unneededTime).Before(now) {
continue
}
nodeGroup, err := cloudProvider.NodeGroupForNode(node)
if err != nil {
glog.Errorf("Error while checking node group for %s: %v", node.Name, err)
continue
}
if nodeGroup == nil || reflect.ValueOf(nodeGroup).IsNil() {
glog.V(4).Infof("Skipping %s - no node group config", node.Name)
continue
}
size, err := nodeGroup.TargetSize()
if err != nil {
glog.Errorf("Error while checking node group size %s: %v", nodeGroup.Id(), err)
continue
}
if size <= nodeGroup.MinSize() {
glog.V(1).Infof("Skipping %s - node group min size reached", node.Name)
continue
}
candidates = append(candidates, node)
}
}
if len(candidates) == 0 {
glog.Infof("No candidates for scale down")
return ScaleDownNoUnneeded, nil
}
// We look for only 1 node so new hints may be incomplete.
nodesToRemove, _, err := simulator.FindNodesToRemove(candidates, nodes, pods, client, predicateChecker, 1, false,
oldHints, usageTracker, time.Now())
if err != nil {
return ScaleDownError, fmt.Errorf("Find node to remove failed: %v", err)
}
if len(nodesToRemove) == 0 {
glog.V(1).Infof("No node to remove")
return ScaleDownNoNodeDeleted, nil
}
nodeToRemove := nodesToRemove[0]
glog.Infof("Removing %s", nodeToRemove.Name)
nodeGroup, err := cloudProvider.NodeGroupForNode(nodeToRemove)
if err != nil {
return ScaleDownError, fmt.Errorf("failed to node group for %s: %v", nodeToRemove.Name, err)
}
if nodeGroup == nil || reflect.ValueOf(nodeGroup).IsNil() {
return ScaleDownError, fmt.Errorf("picked node that doesn't belong to a node group: %s", nodeToRemove.Name)
}
err = nodeGroup.DeleteNodes([]*kube_api.Node{nodeToRemove})
simulator.RemoveNodeFromTracker(usageTracker, nodeToRemove.Name, unneededNodes)
if err != nil {
return ScaleDownError, fmt.Errorf("Failed to delete %s: %v", nodeToRemove.Name, err)
}
return ScaleDownNodeDeleted, nil
}