/
instance.go
470 lines (407 loc) · 13.5 KB
/
instance.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
// Mgmt
// Copyright (C) 2013-2024+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
//
// Additional permission under GNU GPL version 3 section 7
//
// If you modify this program, or any covered work, by linking or combining it
// with embedded mcl code and modules (and that the embedded mcl code and
// modules which link with this program, contain a copy of their source code in
// the authoritative form) containing parts covered by the terms of any other
// license, the licensors of this program grant you additional permission to
// convey the resulting work. Furthermore, the licensors of this program grant
// the original author, James Shubin, additional permission to update this
// additional permission if he deems it necessary to achieve the goals of this
// additional permission.
package integration
import (
"context"
"fmt"
"os"
"os/exec"
"path"
"strings"
"sync"
"syscall"
"github.com/purpleidea/mgmt/util/errwrap"
"github.com/purpleidea/mgmt/util/recwatch"
)
const (
// RootDirectory is the directory that is exposed in the per instance
// directory which can be used by that instance safely.
RootDirectory = "root"
// PrefixDirectory is the directory that is exposed in the per instance
// directory which is used for the mgmt prefix.
PrefixDirectory = "prefix"
// ConvergedStatusFile is the name of the file which is used for the
// converged status tracking.
ConvergedStatusFile = "csf.txt"
// StdoutStderrFile is the name of the file which is used for the
// command output.
StdoutStderrFile = "stdoutstderr.txt"
// longTimeout is a high bound of time we're willing to wait for events.
// If we exceed this timeout, then it's likely we are blocked somewhere.
longTimeout = 60 // seconds
// convergedTimeout is the number of seconds we wait for our instance to
// remain unchanged to be considered as converged.
convergedTimeout = 15 // seconds
// dirMode is the the mode used when making directories.
dirMode = 0755
// fileMode is the the mode used when making files.
fileMode = 0644
)
// Instance represents a single running mgmt instance. It is a building block
// that can be used to run standalone tests, or combined to run clustered tests.
// It can also be used to run a standalone etcd server instance.
type Instance struct {
// Etcd specifies that this is a pure etcd instance instead of an mgmt
// one.
Etcd bool
// EtcdServer specifies we're connecting to an etcd instance instead of
// a normal mgmt peer.
EtcdServer bool
// Hostname is a unique identifier for this instance.
Hostname string
// Preserve prevents the runtime output from being explicitly deleted.
// This is helpful for running analysis or tests on the output.
Preserve bool
// Logf is a logger which should be used.
Logf func(format string, v ...interface{})
// Debug enables more verbosity.
Debug bool
// dir is the directory where all files will be written under.
dir string
tmpPrefixDirectory string
testRootDirectory string
convergedStatusFile string
convergedStatusIndex int
cmd *exec.Cmd
clientURL string // set when launched with run
serverURL string
}
// Init runs some initialization for this instance. It errors if the struct was
// populated in an invalid way, or if it can't initialize correctly.
func (obj *Instance) Init() error {
if obj.Etcd && obj.EtcdServer {
return fmt.Errorf("if we're etcd, we're not connecting to one")
}
if obj.Hostname == "" {
return fmt.Errorf("must specify a hostname")
}
// create temporary directory to use during testing
if obj.dir == "" {
var err error
obj.dir, err = os.MkdirTemp("", fmt.Sprintf("mgmt-integration-%s-", obj.Hostname))
if err != nil {
return errwrap.Wrapf(err, "can't create temporary directory")
}
}
tmpPrefix := path.Join(obj.dir, PrefixDirectory)
if err := os.MkdirAll(tmpPrefix, dirMode); err != nil {
return errwrap.Wrapf(err, "can't create prefix directory")
}
obj.tmpPrefixDirectory = tmpPrefix
testRootDirectory := path.Join(obj.dir, RootDirectory)
if err := os.MkdirAll(testRootDirectory, dirMode); err != nil {
return errwrap.Wrapf(err, "can't create instance root directory")
}
obj.testRootDirectory = testRootDirectory
obj.convergedStatusFile = path.Join(obj.dir, ConvergedStatusFile)
return nil
}
// Close cleans up after we're done with this instance.
func (obj *Instance) Close() error {
if !obj.Preserve {
if obj.dir == "" || obj.dir == "/" {
panic("obj.dir is set to a dangerous path")
}
if err := os.RemoveAll(obj.dir); err != nil { // dangerous ;)
return errwrap.Wrapf(err, "can't remove instance dir")
}
}
obj.Kill() // safety
return nil
}
// Run launches the instance. It returns an error if it was unable to launch.
func (obj *Instance) Run(seeds []*Instance) error {
if obj.cmd != nil {
return fmt.Errorf("an instance is already running")
}
if len(seeds) == 0 {
// set so that Deploy can know where to connect
// also set so that we can allow others to find us and connect
obj.clientURL = "http://127.0.0.1:2379"
obj.serverURL = "http://127.0.0.1:2380"
} else {
// pick next available pair of ports
var maxClientPort, maxServerPort int
for _, instance := range seeds {
clientPort, err := ParsePort(instance.clientURL)
if err != nil {
return errwrap.Wrapf(err, "could not parse client URL from `%s`", instance.Hostname)
}
if clientPort > maxClientPort {
maxClientPort = clientPort
}
serverPort, err := ParsePort(instance.serverURL)
if err != nil {
return errwrap.Wrapf(err, "could not parse server URL from `%s`", instance.Hostname)
}
if serverPort > maxServerPort {
maxServerPort = serverPort
}
}
if maxClientPort+2 == maxServerPort || maxClientPort == maxServerPort+2 {
return fmt.Errorf("port conflict found")
}
obj.clientURL = fmt.Sprintf("http://127.0.0.1:%d", maxClientPort+2) // odd
obj.serverURL = fmt.Sprintf("http://127.0.0.1:%d", maxServerPort+2) // even
}
cmdName, err := BinaryPath()
if err != nil {
return err
}
// run `mgmt etcd`
if obj.Etcd {
cmdArgs := []string{
"etcd", // mode
//fmt.Sprintf("--name=%s", obj.Hostname),
fmt.Sprintf("--listen-client-urls=%s", obj.clientURL),
fmt.Sprintf("--listen-peer-urls=%s", obj.serverURL),
fmt.Sprintf("--advertise-client-urls=%s", obj.clientURL),
//fmt.Sprintf("--advertise-peer-urls=%s", obj.serverURL),
fmt.Sprintf("--data-dir=%s", obj.tmpPrefixDirectory),
}
obj.Logf("run: %s %s", cmdName, strings.Join(cmdArgs, " "))
obj.cmd = exec.Command(cmdName, cmdArgs...)
//obj.cmd.Env = []string{
// fmt.Sprintf("MGMT_TEST_ROOT=%s", obj.testRootDirectory),
//}
obj.cmd.Dir = obj.tmpPrefixDirectory // run program in pwd if ""
// output file for storing logs
file, err := os.Create(path.Join(obj.dir, StdoutStderrFile))
if err != nil {
return errwrap.Wrapf(err, "error creating log file")
}
defer file.Close()
obj.cmd.Stdout = file
obj.cmd.Stderr = file
if err := obj.cmd.Start(); err != nil {
return errwrap.Wrapf(err, "error starting etcd")
}
return nil
}
cmdArgs := []string{
"run", // mode
fmt.Sprintf("--hostname=%s", obj.Hostname),
fmt.Sprintf("--client-urls=%s", obj.clientURL),
fmt.Sprintf("--server-urls=%s", obj.serverURL),
fmt.Sprintf("--prefix=%s", obj.tmpPrefixDirectory),
fmt.Sprintf("--converged-timeout=%d", convergedTimeout),
"--converged-timeout-no-exit",
fmt.Sprintf("--converged-status-file=%s", obj.convergedStatusFile),
}
if len(seeds) > 0 {
urls := []string{}
for _, instance := range seeds {
if instance.cmd == nil {
return fmt.Errorf("instance `%s` has not started yet", instance.Hostname)
}
urls = append(urls, instance.clientURL)
}
s := fmt.Sprintf("--seeds=%s", urls[0])
// TODO: we could just add all the seeds instead...
//s := fmt.Sprintf("--seeds=%s", strings.Join(urls, ","))
cmdArgs = append(cmdArgs, s)
}
if obj.EtcdServer {
cmdArgs = append(cmdArgs, "--no-server")
cmdArgs = append(cmdArgs, "--ideal-cluster-size=1")
}
gapi := "empty" // empty GAPI (for now)
cmdArgs = append(cmdArgs, gapi)
obj.Logf("run: %s %s", cmdName, strings.Join(cmdArgs, " "))
obj.cmd = exec.Command(cmdName, cmdArgs...)
obj.cmd.Env = []string{
fmt.Sprintf("MGMT_TEST_ROOT=%s", obj.testRootDirectory),
}
// output file for storing logs
file, err := os.Create(path.Join(obj.dir, StdoutStderrFile))
if err != nil {
return errwrap.Wrapf(err, "error creating log file")
}
defer file.Close()
obj.cmd.Stdout = file
obj.cmd.Stderr = file
if err := obj.cmd.Start(); err != nil {
return errwrap.Wrapf(err, "error starting mgmt")
}
return nil
}
// Kill the process immediately. This is a `kill -9` for if things get stuck.
func (obj *Instance) Kill() error {
if obj.cmd == nil {
return nil // already dead
}
if obj.cmd.Process == nil {
return nil // nothing running
}
// cause a stack dump first if we can
_ = obj.cmd.Process.Signal(syscall.SIGQUIT)
return obj.cmd.Process.Kill()
}
// Quit sends a friendly shutdown request to the process. You can specify a
// context if you'd like to exit earlier. If you trigger an early exit with the
// context, then this will end up running a `kill -9` so it can return.
func (obj *Instance) Quit(ctx context.Context) error {
if obj.cmd == nil {
return fmt.Errorf("no process is running")
}
if err := obj.cmd.Process.Signal(os.Interrupt); err != nil {
return errwrap.Wrapf(err, "could not send interrupt signal")
}
var err error
wg := &sync.WaitGroup{}
done := make(chan error)
wg.Add(1)
go func() {
defer wg.Done()
done <- obj.cmd.Wait()
close(done)
}()
wg.Add(1)
go func() {
defer wg.Done()
select {
case err = <-done:
case <-ctx.Done():
obj.Kill() // should cause the Wait() to exit
err = ctx.Err()
}
}()
wg.Wait()
obj.cmd = nil
return err
}
// Wait until the first converged state we hit. It is not necessary to use the
// `--converged-timeout` option with mgmt for this to work. It tracks this via
// the `--converged-status-file` option which can be used to track the varying
// convergence status.
func (obj *Instance) Wait(ctx context.Context) error {
//if obj.cmd == nil { // TODO: should we include this?
// return fmt.Errorf("no process is running")
//}
if obj.Etcd {
return fmt.Errorf("etcd Wait not implemented")
}
recurse := false
recWatcher, err := recwatch.NewRecWatcher(obj.convergedStatusFile, recurse)
if err != nil {
return errwrap.Wrapf(err, "could not watch file")
}
defer recWatcher.Close()
startup := make(chan struct{})
close(startup)
for {
select {
// FIXME: instead of sending one event here, the recwatch
// library should send one initial event at startup...
case <-startup:
startup = nil
// send an initial event
case event, ok := <-recWatcher.Events():
if !ok {
return fmt.Errorf("file watcher shut down")
}
if err := event.Error; err != nil {
return errwrap.Wrapf(err, "error event received")
}
startup = nil
// send event...
case <-ctx.Done():
startup = nil
return ctx.Err()
}
contents, err := os.ReadFile(obj.convergedStatusFile)
if err != nil {
continue // file might not exist yet, wait for an event
}
raw := strings.Split(string(contents), "\n")
lines := []string{}
for _, x := range raw {
if x == "" { // drop blank lines!
continue
}
lines = append(lines, x)
}
if c := len(lines); c < obj.convergedStatusIndex {
return fmt.Errorf("file is missing lines or was truncated, got: %d", c)
}
var converged bool
for i := obj.convergedStatusIndex; i < len(lines); i++ {
obj.convergedStatusIndex = i + 1 // new max
line := lines[i]
if line == "true" { // converged!
converged = true
}
}
if converged {
return nil
}
}
}
// DeployLang deploys some code to the cluster.
func (obj *Instance) DeployLang(code string) error {
//if obj.cmd == nil { // TODO: should we include this?
// return fmt.Errorf("no process is running")
//}
filename := path.Join(obj.dir, "deploy.mcl")
data := []byte(code)
if err := os.WriteFile(filename, data, fileMode); err != nil {
return err
}
cmdName, err := BinaryPath()
if err != nil {
return err
}
cmdArgs := []string{
"deploy", // mode
"--no-git",
fmt.Sprintf("--seeds=%s", obj.clientURL),
"lang", filename,
}
obj.Logf("run: %s %s", cmdName, strings.Join(cmdArgs, " "))
cmd := exec.Command(cmdName, cmdArgs...)
stdoutStderr, err := cmd.CombinedOutput() // does cmd.Run() for us!
obj.Logf("stdout/stderr:\n%s", stdoutStderr)
if err != nil {
return errwrap.Wrapf(err, "can't run deploy")
}
return nil
}
// Dir returns the dir where the instance can write to. You should only use this
// after Init has been called, or it won't have been created and determined yet.
func (obj *Instance) Dir() string {
return obj.dir
}
// CombinedOutput returns the logged output from the instance.
func (obj *Instance) CombinedOutput() (string, error) {
contents, err := os.ReadFile(path.Join(obj.dir, StdoutStderrFile))
if err != nil {
return "", err
}
return string(contents), nil
}