-
Notifications
You must be signed in to change notification settings - Fork 79
/
node.go
186 lines (169 loc) · 6.05 KB
/
node.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
package resource
import (
"context"
"time"
"github.com/aws/aws-k8s-tester/e2e/framework/utils"
log "github.com/cihub/seelog"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
)
const (
// poll is how often to Poll pods, nodes and claims.
poll = 2 * time.Second
// singleCallTimeout is how long to try single API calls (like 'get' or 'list'). Used to prevent
// transient failures from failing tests.
// TODO: client should not apply this timeout to Watch calls. Increased from 30s until that is fixed.
singleCallTimeout = 5 * time.Minute
// TaintNodeNotReady will be added when node is not ready
// and feature-gate for TaintBasedEvictions flag is enabled,
// and removed when node becomes ready.
TaintNodeNotReady = "node.kubernetes.io/not-ready"
// TaintNodeUnreachable will be added when node becomes unreachable
// (corresponding to NodeReady status ConditionUnknown)
// and feature-gate for TaintBasedEvictions flag is enabled,
// and removed when node becomes reachable (NodeReady status ConditionTrue).
TaintNodeUnreachable = "node.kubernetes.io/unreachable"
)
var (
// UnreachableTaintTemplate is the taint for when a node becomes unreachable.
UnreachableTaintTemplate = &corev1.Taint{
Key: TaintNodeUnreachable,
Effect: v1.TaintEffectNoExecute,
}
// NotReadyTaintTemplate is the taint for when a node is not ready for
// executing pods
NotReadyTaintTemplate = &corev1.Taint{
Key: TaintNodeNotReady,
Effect: v1.TaintEffectNoExecute,
}
)
type NodeManager struct {
cs kubernetes.Interface
}
func NewNodeManager(cs kubernetes.Interface) *NodeManager {
return &NodeManager{
cs: cs,
}
}
func (m *NodeManager) WaitNodeExists(ctx context.Context, n *corev1.Node) (*corev1.Node, error) {
var (
observedN *corev1.Node
err error
)
return observedN, wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) {
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
observedN, err = m.cs.CoreV1().Nodes().Get(ctx, n.Name, metav1.GetOptions{})
cancel()
if err != nil {
if serr, ok := err.(*errors.StatusError); ok {
switch serr.ErrStatus.Reason {
case "NotFound":
return false, nil
default:
return false, err
}
}
return false, err
}
return true, nil
}, ctx.Done())
}
func (m *NodeManager) WaitNodeReady(ctx context.Context, n *corev1.Node) (*corev1.Node, error) {
var (
observedN *corev1.Node
err error
)
return observedN, wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) {
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
observedN, err = m.cs.CoreV1().Nodes().Get(ctx, n.Name, metav1.GetOptions{})
cancel()
if err != nil {
return false, err
}
return isNodeReady(observedN), nil
}, ctx.Done())
}
func isNodeReady(node *v1.Node) bool {
for _, condition := range node.Status.Conditions {
if condition.Type == v1.NodeReady {
// Check if the node has taints UnreachableTaintTemplate or NotReadyTaintTemplate
hasTaints := false
taints := node.Spec.Taints
for _, taint := range taints {
if taint.MatchTaint(UnreachableTaintTemplate) || taint.MatchTaint(NotReadyTaintTemplate) {
log.Debug("node (%s) has taint %s", node.Name, taint.String())
hasTaints = true
break
}
}
if !hasTaints && condition.Status == v1.ConditionTrue {
log.Debug("node (%s) has ready status %s", node.Name, condition.Status)
return true
}
log.Debug("node (%s) has ready status %s", node.Name, condition.Status)
}
}
return false
}
// // TODO: better to change to a easy read name
// func isNodeConditionSetAsExpected(node *v1.Node, conditionType v1.NodeConditionType, wantTrue, silent bool) bool {
// // Check the node readiness condition (logging all).
// for _, cond := range node.Status.Conditions {
// // Ensure that the condition type and the status matches as desired.
// if cond.Type == conditionType {
// // For NodeReady condition we need to check Taints as well
// if cond.Type == v1.NodeReady {
// hasNodeControllerTaints := false
// // For NodeReady we need to check if Taints are gone as well
// taints := node.Spec.Taints
// for _, taint := range taints {
// if taint.MatchTaint(UnreachableTaintTemplate) || taint.MatchTaint(NotReadyTaintTemplate) {
// hasNodeControllerTaints = true
// break
// }
// }
// if wantTrue {
// if (cond.Status == v1.ConditionTrue) && !hasNodeControllerTaints {
// return true
// }
// msg := ""
// if !hasNodeControllerTaints {
// msg = fmt.Sprintf("Condition %s of node %s is %v instead of %t. Reason: %v, message: %v",
// conditionType, node.Name, cond.Status == v1.ConditionTrue, wantTrue, cond.Reason, cond.Message)
// }
// msg = fmt.Sprintf("Condition %s of node %s is %v, but Node is tainted by NodeController with %v. Failure",
// conditionType, node.Name, cond.Status == v1.ConditionTrue, taints)
// if !silent {
// log.Debugf(msg)
// }
// return false
// }
// // TODO: check if the Node is tainted once we enable NC notReady/unreachable taints by default
// if cond.Status != v1.ConditionTrue {
// return true
// }
// if !silent {
// log.Debugf("Condition %s of node %s is %v instead of %t. Reason: %v, message: %v",
// conditionType, node.Name, cond.Status == v1.ConditionTrue, wantTrue, cond.Reason, cond.Message)
// }
// return false
// }
// if (wantTrue && (cond.Status == v1.ConditionTrue)) || (!wantTrue && (cond.Status != v1.ConditionTrue)) {
// return true
// }
// if !silent {
// log.Debugf("Condition %s of node %s is %v instead of %t. Reason: %v, message: %v",
// conditionType, node.Name, cond.Status == v1.ConditionTrue, wantTrue, cond.Reason, cond.Message)
// }
// return false
// }
// }
// if !silent {
// log.Debugf("Couldn't find condition %v on node %v", conditionType, node.Name)
// }
// return false
// }