-
Notifications
You must be signed in to change notification settings - Fork 42
/
perturb.go
144 lines (124 loc) · 4.89 KB
/
perturb.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
package app
import (
"context"
"time"
"github.com/omni-network/omni/e2e/types"
"github.com/omni-network/omni/lib/errors"
"github.com/omni-network/omni/lib/log"
rpctypes "github.com/cometbft/cometbft/rpc/core/types"
e2e "github.com/cometbft/cometbft/test/e2e/pkg"
"github.com/cometbft/cometbft/test/e2e/pkg/infra/docker"
)
// perturb the running testnet.
func perturb(ctx context.Context, testnet types.Testnet) error {
for _, node := range testnet.Nodes {
for _, perturbation := range node.Perturbations {
_, err := perturbNode(ctx, node, perturbation)
if err != nil {
return err
}
time.Sleep(3 * time.Second) // Give network some time to recover between each
}
}
for service, purturbs := range testnet.Perturb {
for _, p := range purturbs {
if err := perturbService(ctx, service, testnet.Dir, p); err != nil {
return errors.Wrap(err, "purturb service", "service", service)
}
time.Sleep(3 * time.Second) // Give network some time to recover between each
}
}
return nil
}
// perturbService perturbs a docker service with a given perturbation.
func perturbService(ctx context.Context, service string, testnetDir string, perturb types.Perturb) error {
ctx = log.WithCtx(ctx, "service", service)
log.Info(ctx, "Perturbing service", "perturb", perturb)
switch perturb {
case types.PerturbRestart:
if err := docker.ExecCompose(ctx, testnetDir, "restart", service); err != nil {
return errors.Wrap(err, "restart service")
}
case types.PerturbStopStart:
if err := docker.ExecCompose(ctx, testnetDir, "stop", service); err != nil {
return errors.Wrap(err, "stop service")
}
time.Sleep(5 * time.Second)
if err := docker.ExecCompose(ctx, testnetDir, "start", service); err != nil {
return errors.Wrap(err, "start service")
}
case types.PerturbRollback:
if err := docker.ExecCompose(ctx, testnetDir, "stop", service); err != nil {
return errors.Wrap(err, "stop service")
}
if err := docker.ExecCompose(ctx, testnetDir, "run", service, "rollback"); err != nil {
return errors.Wrap(err, "rollback service")
}
if err := docker.ExecCompose(ctx, testnetDir, "start", service); err != nil {
return errors.Wrap(err, "start service")
}
case types.PerturbFuzzyHeadAttRoot, types.PerturbFuzzyHeadDropBlocks, types.PerturbFuzzyHeadDropMsgs, types.PerturbFuzzyHeadMoreMsgs:
if err := docker.ExecCompose(ctx, testnetDir, "exec", service, "wget", "-O-", "localhost:8545/fuzzy_enable?perturb="+string(perturb)); err != nil {
return errors.Wrap(err, "enable fuzzy head")
}
time.Sleep(6 * time.Second)
if err := docker.ExecCompose(ctx, testnetDir, "exec", service, "wget", "-O-", "localhost:8545/fuzzy_disable"); err != nil {
return errors.Wrap(err, "disable fuzzy head")
}
default:
return errors.New("unknown service perturbation")
}
log.Info(ctx, "Perturbed service", "perturb", perturb)
return nil
}
// perturbNode perturbs a node with a given perturbation, returning its status
// after recovering.
func perturbNode(ctx context.Context, node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.ResultStatus, error) {
testnet := node.Testnet
name := node.Name
ctx = log.WithCtx(ctx, "name", name)
switch perturbation {
case e2e.PerturbationDisconnect:
networkName := testnet.Name + "_" + testnet.Name
log.Info(ctx, "Perturb node: disconnect")
if err := docker.Exec(ctx, "network", "disconnect", networkName, name); err != nil {
return nil, errors.Wrap(err, "disconnect node from network")
}
time.Sleep(10 * time.Second)
if err := docker.Exec(ctx, "network", "connect", networkName, name); err != nil {
return nil, errors.Wrap(err, "connect node tp network")
}
case e2e.PerturbationKill:
log.Info(ctx, "Perturb node: kill")
if err := docker.ExecCompose(ctx, testnet.Dir, "kill", "-s", "SIGKILL", name); err != nil {
return nil, errors.Wrap(err, "kill node")
}
if err := docker.ExecCompose(ctx, testnet.Dir, "start", name); err != nil {
return nil, errors.Wrap(err, "start node")
}
case e2e.PerturbationPause:
log.Info(ctx, "Perturb node: pause")
if err := docker.ExecCompose(ctx, testnet.Dir, "pause", name); err != nil {
return nil, errors.Wrap(err, "pause node")
}
time.Sleep(10 * time.Second)
if err := docker.ExecCompose(ctx, testnet.Dir, "unpause", name); err != nil {
return nil, errors.Wrap(err, "unpause node")
}
case e2e.PerturbationRestart:
log.Info(ctx, "Perturb node: restart")
if err := docker.ExecCompose(ctx, testnet.Dir, "restart", name); err != nil {
return nil, errors.Wrap(err, "restart node")
}
case e2e.PerturbationUpgrade:
return nil, errors.New("upgrade perturbation not supported")
default:
return nil, errors.New("unexpected perturbation type", "type", perturbation)
}
status, err := waitForNode(ctx, node, 0, 20*time.Second)
if err != nil {
return nil, err
}
log.Info(ctx, "Node recovered from perturbation", "height", status.SyncInfo.LatestBlockHeight)
return status, nil
}