forked from purpleidea/mgmt
-
Notifications
You must be signed in to change notification settings - Fork 0
/
cluster.go
224 lines (198 loc) · 6.39 KB
/
cluster.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package integration
import (
"context"
"fmt"
"io/ioutil"
"os"
"path"
"time"
multierr "github.com/hashicorp/go-multierror"
errwrap "github.com/pkg/errors"
)
// Cluster represents an mgmt cluster. It uses the instance building blocks to
// run clustered tests.
type Cluster struct {
// Hostnames is the list of unique identifiers for this cluster.
Hostnames []string
// Preserve prevents the runtime output from being explicitly deleted.
// This is helpful for running analysis or tests on the output.
Preserve bool
// Debug enables more verbosity.
Debug bool
// dir is the directory where all files will be written under.
dir string
instances map[string]*Instance
}
// Init runs some initialization for this Cluster. It errors if the struct was
// populated in an invalid way, or if it can't initialize correctly.
func (obj *Cluster) Init() error {
obj.instances = make(map[string]*Instance)
// create temporary directory to use during testing
var err error
if obj.dir == "" {
obj.dir, err = ioutil.TempDir("", "mgmt-integration-cluster-")
if err != nil {
return errwrap.Wrapf(err, "can't create temporary directory")
}
}
for _, h := range obj.Hostnames {
instancePrefix := path.Join(obj.dir, h)
if err := os.MkdirAll(instancePrefix, dirMode); err != nil {
return errwrap.Wrapf(err, "can't create instance directory")
}
obj.instances[h] = &Instance{
Hostname: h,
Preserve: obj.Preserve,
Debug: obj.Debug,
dir: instancePrefix,
}
if e := obj.instances[h].Init(); e != nil {
err = multierr.Append(err, e)
}
}
return err
}
// Close cleans up after we're done with this Cluster.
func (obj *Cluster) Close() error {
var err error
// do this in reverse for fun
for i := len(obj.Hostnames) - 1; i >= 0; i-- {
h := obj.Hostnames[i]
instance, exists := obj.instances[h]
if !exists {
continue
}
if e := instance.Close(); e != nil {
err = multierr.Append(err, e)
}
}
if !obj.Preserve {
if obj.dir == "" || obj.dir == "/" {
panic("obj.dir is set to a dangerous path")
}
if err := os.RemoveAll(obj.dir); err != nil { // dangerous ;)
return errwrap.Wrapf(err, "can't remove instance dir")
}
}
return err
}
// RunLinear starts up each instance linearly, one at a time.
func (obj *Cluster) RunLinear() error {
for i, h := range obj.Hostnames {
// build a list of earlier instances that have already run
seeds := []*Instance{}
for j := 0; j < i; j++ {
x := obj.instances[obj.Hostnames[j]]
seeds = append(seeds, x)
}
instance, exists := obj.instances[h]
if !exists {
return fmt.Errorf("instance `%s` not found", h)
}
if err := instance.Run(seeds); err != nil {
return errwrap.Wrapf(err, "trouble running instance `%s`", h)
}
// FIXME: consider removing this wait entirely
// wait for startup before continuing with the next one
ctx, cancel := context.WithTimeout(context.Background(), longTimeout*time.Second)
defer cancel()
if err := instance.Wait(ctx); err != nil { // wait to get a converged signal
return errwrap.Wrapf(err, "mgmt wait on instance `%s` failed", h) // timeout expired
}
}
return nil
}
// Kill the cluster immediately. This is a `kill -9` for if things get stuck.
func (obj *Cluster) Kill() error {
var err error
// do this in reverse for fun
for i := len(obj.Hostnames) - 1; i >= 0; i-- {
h := obj.Hostnames[i]
instance, exists := obj.instances[h]
if !exists {
continue
}
if e := instance.Kill(); e != nil {
err = multierr.Append(err, e)
}
}
return err
}
// Quit sends a friendly shutdown request to the cluster. You can specify a
// context if you'd like to exit earlier. If you trigger an early exit with the
// context, then this will end up running a `kill -9` so it can return. Remember
// to leave a longer timeout when using a context since this will have to call
// quit on each member individually.
func (obj *Cluster) Quit(ctx context.Context) error {
var err error
// do this in reverse for fun
for i := len(obj.Hostnames) - 1; i >= 0; i-- {
h := obj.Hostnames[i]
instance, exists := obj.instances[h]
if !exists {
continue
}
if e := instance.Quit(ctx); e != nil {
err = multierr.Append(err, e)
}
}
return err
}
// Wait until the first converged state is hit for each member in the cluster.
// Remember to leave a longer timeout when using a context since this will have
// to call wait on each member individually.
func (obj *Cluster) Wait(ctx context.Context) error {
var err error
for _, h := range obj.Hostnames {
instance, exists := obj.instances[h]
if !exists {
continue
}
// TODO: do we want individual waits?
//ctx, cancel := context.WithTimeout(context.Background(), longTimeout*time.Second)
//defer cancel()
if e := instance.Wait(ctx); e != nil {
err = multierr.Append(err, e)
}
}
return err
}
// DeployLang deploys some code to the cluster. It arbitrarily picks the first
// host to run the deploy on.
func (obj *Cluster) DeployLang(code string) error {
if len(obj.Hostnames) == 0 {
return fmt.Errorf("must have at least one host to deploy")
}
h := obj.Hostnames[0]
instance, exists := obj.instances[h]
if !exists {
return fmt.Errorf("instance `%s` not found", h)
}
return instance.DeployLang(code)
}
// Instances returns the map of instances attached to this cluster. It is most
// useful after a cluster has started. Before Init, it won't have any entries.
func (obj *Cluster) Instances() map[string]*Instance {
return obj.instances
}
// Dir returns the dir where the instance can write to. You should only use this
// after Init has been called, or it won't have been created and determined yet.
func (obj *Cluster) Dir() string {
return obj.dir
}