-
Notifications
You must be signed in to change notification settings - Fork 14
/
repair_drain_start.go
125 lines (105 loc) · 2.88 KB
/
repair_drain_start.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
package op
import (
"context"
"fmt"
"time"
"github.com/cybozu-go/cke"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
)
type repairDrainStartOp struct {
finished bool
entry *cke.RepairQueueEntry
config *cke.Repair
apiserver *cke.Node
}
func RepairDrainStartOp(apiserver *cke.Node, entry *cke.RepairQueueEntry, config *cke.Repair) cke.Operator {
return &repairDrainStartOp{
entry: entry,
config: config,
apiserver: apiserver,
}
}
func (o *repairDrainStartOp) Name() string {
return "repair-drain-start"
}
func (o *repairDrainStartOp) NextCommand() cke.Commander {
if o.finished {
return nil
}
o.finished = true
attempts := 1
if o.config.EvictRetries != nil {
attempts = *o.config.EvictRetries + 1
}
interval := 0 * time.Second
if o.config.EvictInterval != nil {
interval = time.Second * time.Duration(*o.config.EvictInterval)
}
return repairDrainStartCommand{
entry: o.entry,
protectedNamespaces: o.config.ProtectedNamespaces,
apiserver: o.apiserver,
evictAttempts: attempts,
evictInterval: interval,
}
}
func (o *repairDrainStartOp) Targets() []string {
return []string{o.entry.Address}
}
type repairDrainStartCommand struct {
entry *cke.RepairQueueEntry
protectedNamespaces *metav1.LabelSelector
apiserver *cke.Node
evictAttempts int
evictInterval time.Duration
}
func (c repairDrainStartCommand) Run(ctx context.Context, inf cke.Infrastructure, _ string) error {
cs, err := inf.K8sClient(ctx, c.apiserver)
if err != nil {
return err
}
nodesAPI := cs.CoreV1().Nodes()
protected, err := listProtectedNamespaces(ctx, cs, c.protectedNamespaces)
if err != nil {
return err
}
err = func() error {
c.entry.Status = cke.RepairStatusProcessing
c.entry.StepStatus = cke.RepairStepStatusDraining
c.entry.LastTransitionTime = time.Now().Truncate(time.Second).UTC()
err := inf.Storage().UpdateRepairsEntry(ctx, c.entry)
if err != nil {
return err
}
err = checkJobPodNotExist(ctx, cs, c.entry.Nodename)
if err != nil {
return err
}
// Note: The annotation name is shared with reboot operations.
_, err = nodesAPI.Patch(ctx, c.entry.Nodename, types.StrategicMergePatchType, []byte(`
{
"metadata":{"annotations":{"`+CKEAnnotationReboot+`": "true"}},
"spec":{"unschedulable": true}
}
`), metav1.PatchOptions{})
if err != nil {
return fmt.Errorf("failed to cordon node %s: %v", c.entry.Address, err)
}
return nil
}()
if err != nil {
return repairDrainBackOff(ctx, inf, c.entry, err)
}
err = evictOrDeleteNodePod(ctx, cs, c.entry.Nodename, protected, c.evictAttempts, c.evictInterval)
if err != nil {
return repairDrainBackOff(ctx, inf, c.entry, err)
}
return nil
}
func (c repairDrainStartCommand) Command() cke.Command {
return cke.Command{
Name: "repairDrainStartCommand",
Target: c.entry.Address,
}
}