This repository has been archived by the owner on Mar 28, 2020. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 741
/
chaos.go
100 lines (85 loc) · 2.56 KB
/
chaos.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
// Copyright 2016 The etcd-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package chaos
import (
"context"
"math/rand"
"github.com/sirupsen/logrus"
"golang.org/x/time/rate"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/kubernetes"
)
// Monkeys knows how to crush pods and nodes.
type Monkeys struct {
kubecli kubernetes.Interface
}
func NewMonkeys(kubecli kubernetes.Interface) *Monkeys {
return &Monkeys{kubecli: kubecli}
}
type CrashConfig struct {
Namespace string
Selector labels.Selector
KillRate rate.Limit
KillProbability float64
KillMax int
}
// TODO: respect context in k8s operations.
func (m *Monkeys) CrushPods(ctx context.Context, c *CrashConfig) {
burst := int(c.KillRate)
if burst <= 0 {
burst = 1
}
limiter := rate.NewLimiter(c.KillRate, burst)
ls := c.Selector.String()
ns := c.Namespace
for {
err := limiter.Wait(ctx)
if err != nil { // user cancellation
logrus.Infof("crushPods is canceled for selector %v by the user: %v", ls, err)
return
}
if p := rand.Float64(); p > c.KillProbability {
logrus.Infof("skip killing pod: probability: %v, got p: %v", c.KillProbability, p)
continue
}
pods, err := m.kubecli.CoreV1().Pods(ns).List(metav1.ListOptions{LabelSelector: ls})
if err != nil {
logrus.Errorf("failed to list pods for selector %v: %v", ls, err)
continue
}
if len(pods.Items) == 0 {
logrus.Infof("no pods to kill for selector %v", ls)
continue
}
max := len(pods.Items)
kmax := rand.Intn(c.KillMax) + 1
if kmax < max {
max = kmax
}
logrus.Infof("start to kill %d pods for selector %v", max, ls)
tokills := make(map[string]struct{})
for len(tokills) < max {
tokills[pods.Items[rand.Intn(len(pods.Items))].Name] = struct{}{}
}
for tokill := range tokills {
err = m.kubecli.CoreV1().Pods(ns).Delete(tokill, metav1.NewDeleteOptions(0))
if err != nil {
logrus.Errorf("failed to kill pod %v: %v", tokill, err)
continue
}
logrus.Infof("killed pod %v for selector %v", tokill, ls)
}
}
}