-
Notifications
You must be signed in to change notification settings - Fork 71
/
monkey.go
106 lines (94 loc) · 2.79 KB
/
monkey.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
//
// DISCLAIMER
//
// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
package chaos
import (
"context"
"github.com/rs/zerolog"
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/arangodb/kube-arangodb/pkg/logging"
"github.com/arangodb/kube-arangodb/pkg/util"
"github.com/arangodb/kube-arangodb/pkg/util/errors"
"github.com/arangodb/kube-arangodb/pkg/util/timer"
)
var (
chaosMonkeyLogger = logging.Global().RegisterAndGetLogger("chaos-monkey", logging.Info)
)
// Monkey is the service that introduces chaos in the deployment
// if allowed and enabled.
type Monkey struct {
namespace, name string
log logging.Logger
context Context
}
func (m Monkey) WrapLogger(in *zerolog.Event) *zerolog.Event {
return in.Str("namespace", m.namespace).Str("name", m.name)
}
// NewMonkey creates a new chaos monkey with given context.
func NewMonkey(namespace, name string, context Context) *Monkey {
m := &Monkey{
context: context,
namespace: namespace,
name: name,
}
m.log = chaosMonkeyLogger.WrapObj(m)
return m
}
// Run the monkey until the given channel is closed.
func (m Monkey) Run(stopCh <-chan struct{}) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
for {
spec := m.context.GetSpec()
if spec.Chaos.IsEnabled() {
// Gamble to set if we must introduce chaos
chance := float64(spec.Chaos.GetKillPodProbability()) / 100.0
if util.Rand().Float64() < chance {
// Let's introduce pod chaos
if err := m.killRandomPod(ctx); err != nil {
m.log.Err(err).Info("Failed to kill random pod")
}
}
}
select {
case <-timer.After(spec.Chaos.GetInterval()):
// Continue
case <-stopCh:
// We're done
return
}
}
}
// killRandomPod fetches all owned pods and tries to kill one.
func (m Monkey) killRandomPod(ctx context.Context) error {
pods, err := m.context.GetOwnedPods(ctx)
if err != nil {
return errors.WithStack(err)
}
if len(pods) <= 1 {
// Not enough pods
return nil
}
p := pods[util.Rand().Intn(len(pods))]
m.log.Str("pod-name", p.GetName()).Info("Killing pod")
if err := m.context.DeletePod(ctx, p.GetName(), meta.DeleteOptions{}); err != nil {
return errors.WithStack(err)
}
return nil
}