forked from strangelove-ventures/interchaintest
/
setup.go
240 lines (203 loc) · 7.13 KB
/
setup.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
package dockerutil
import (
"bytes"
"context"
"fmt"
"os"
"strings"
"time"
"github.com/avast/retry-go/v4"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/client"
"github.com/docker/docker/errdefs"
)
// DockerSetupTestingT is a subset of testing.T required for DockerSetup.
type DockerSetupTestingT interface {
Helper()
Name() string
Failed() bool
Cleanup(func())
Logf(format string, args ...any)
}
// CleanupLabel is a docker label key targeted by DockerSetup when it cleans up docker resources.
//
// "interchaintest" is perhaps a better name. However, for backwards compatability we preserve the original name of "ibc-test"
// with the hyphen. Otherwise, we run the risk of causing "container already exists" errors because DockerSetup
// is unable to clean old resources from docker engine.
const CleanupLabel = "ibc-test"
// CleanupLabel is the "old" format.
// Note that any new labels should follow the reverse DNS format suggested at
// https://docs.docker.com/config/labels-custom-metadata/#key-format-recommendations.
const (
// LabelPrefix is the reverse DNS format "namespace" for interchaintest Docker labels.
LabelPrefix = "ventures.strangelove.interchaintest."
// NodeOwnerLabel indicates the logical node owning a particular object (probably a volume).
NodeOwnerLabel = LabelPrefix + "node-owner"
)
// KeepVolumesOnFailure determines whether volumes associated with a test
// using DockerSetup are retained or deleted following a test failure.
//
// The value is false by default, but can be initialized to true by setting the
// environment variable IBCTEST_SKIP_FAILURE_CLEANUP to a non-empty value.
// Alternatively, importers of the dockerutil package may set the variable to true.
// Because dockerutil is an internal package, the public API for setting this value
// is interchaintest.KeepDockerVolumesOnFailure(bool).
var KeepVolumesOnFailure = os.Getenv("IBCTEST_SKIP_FAILURE_CLEANUP") != ""
// DockerSetup returns a new Docker Client and the ID of a configured network, associated with t.
//
// If any part of the setup fails, DockerSetup panics because the test cannot continue.
func DockerSetup(t DockerSetupTestingT) (*client.Client, string) {
t.Helper()
cli, err := client.NewClientWithOpts(client.FromEnv)
if err != nil {
panic(fmt.Errorf("failed to create docker client: %v", err))
}
// Clean up docker resources at end of test.
t.Cleanup(dockerCleanup(t, cli))
// Also eagerly clean up any leftover resources from a previous test run,
// e.g. if the test was interrupted.
dockerCleanup(t, cli)()
name := fmt.Sprintf("interchaintest-%s", RandLowerCaseLetterString(8))
network, err := cli.NetworkCreate(context.TODO(), name, types.NetworkCreate{
CheckDuplicate: true,
Labels: map[string]string{CleanupLabel: t.Name()},
})
if err != nil {
panic(fmt.Errorf("failed to create docker network: %v", err))
}
return cli, network.ID
}
// dockerCleanup will clean up Docker containers, networks, and the other various config files generated in testing
func dockerCleanup(t DockerSetupTestingT, cli *client.Client) func() {
return func() {
showContainerLogs := os.Getenv("SHOW_CONTAINER_LOGS") != ""
containerLogTail := os.Getenv("CONTAINER_LOG_TAIL")
ctx := context.TODO()
cs, err := cli.ContainerList(ctx, types.ContainerListOptions{
All: true,
Filters: filters.NewArgs(
filters.Arg("label", CleanupLabel+"="+t.Name()),
),
})
if err != nil {
t.Logf("Failed to list containers during docker cleanup: %v", err)
return
}
for _, c := range cs {
stopTimeout := 10 * time.Second
deadline := time.Now().Add(stopTimeout)
if err := cli.ContainerStop(ctx, c.ID, &stopTimeout); isLoggableStopError(err) {
t.Logf("Failed to stop container %s during docker cleanup: %v", c.ID, err)
}
waitCtx, cancel := context.WithDeadline(ctx, deadline.Add(500*time.Millisecond))
waitCh, errCh := cli.ContainerWait(waitCtx, c.ID, container.WaitConditionNotRunning)
select {
case <-waitCtx.Done():
t.Logf("Timed out waiting for container %s", c.ID)
case err := <-errCh:
t.Logf("Failed to wait for container %s during docker cleanup: %v", c.ID, err)
case res := <-waitCh:
if res.Error != nil {
t.Logf("Error while waiting for container %s during docker cleanup: %s", c.ID, res.Error.Message)
}
// Ignoring statuscode for now.
}
cancel()
if t.Failed() || showContainerLogs {
logTail := "50"
if containerLogTail != "" {
logTail = containerLogTail
}
rc, err := cli.ContainerLogs(ctx, c.ID, types.ContainerLogsOptions{
ShowStdout: true,
ShowStderr: true,
Tail: logTail,
})
if err == nil {
b := new(bytes.Buffer)
_, err := b.ReadFrom(rc)
if err == nil {
t.Logf("Container logs - {%s}\n%s", strings.Join(c.Names, " "), b.String())
}
}
}
if err := cli.ContainerRemove(ctx, c.ID, types.ContainerRemoveOptions{
// Not removing volumes with the container, because we separately handle them conditionally.
Force: true,
}); err != nil {
t.Logf("Failed to remove container %s during docker cleanup: %v", c.ID, err)
}
}
pruneVolumesWithRetry(ctx, t, cli)
pruneNetworksWithRetry(ctx, t, cli)
}
}
func pruneVolumesWithRetry(ctx context.Context, t DockerSetupTestingT, cli *client.Client) {
if KeepVolumesOnFailure && t.Failed() {
return
}
var msg string
err := retry.Do(
func() error {
res, err := cli.VolumesPrune(ctx, filters.NewArgs(filters.Arg("label", CleanupLabel+"="+t.Name())))
if err != nil {
if errdefs.IsConflict(err) {
// Prune is already in progress; try again.
return err
}
// Give up on any other error.
return retry.Unrecoverable(err)
}
if len(res.VolumesDeleted) > 0 {
msg = fmt.Sprintf("Pruned %d volumes, reclaiming approximately %.1f MB", len(res.VolumesDeleted), float64(res.SpaceReclaimed)/(1024*1024))
}
return nil
},
retry.Context(ctx),
retry.DelayType(retry.FixedDelay),
)
if err != nil {
t.Logf("Failed to prune volumes during docker cleanup: %v", err)
return
}
if msg != "" {
// Odd to Logf %s, but this is a defensive way to keep the DockerSetupTestingT interface
// with only Logf and not need to add Log.
t.Logf("%s", msg)
}
}
func pruneNetworksWithRetry(ctx context.Context, t DockerSetupTestingT, cli *client.Client) {
var deleted []string
err := retry.Do(
func() error {
res, err := cli.NetworksPrune(ctx, filters.NewArgs(filters.Arg("label", CleanupLabel+"="+t.Name())))
if err != nil {
if errdefs.IsConflict(err) {
// Prune is already in progress; try again.
return err
}
// Give up on any other error.
return retry.Unrecoverable(err)
}
deleted = res.NetworksDeleted
return nil
},
retry.Context(ctx),
retry.DelayType(retry.FixedDelay),
)
if err != nil {
t.Logf("Failed to prune networks during docker cleanup: %v", err)
return
}
if len(deleted) > 0 {
t.Logf("Pruned unused networks: %v", deleted)
}
}
func isLoggableStopError(err error) bool {
if err == nil {
return false
}
return !(errdefs.IsNotModified(err) || errdefs.IsNotFound(err))
}