forked from hashicorp/nomad
-
Notifications
You must be signed in to change notification settings - Fork 0
/
run.go
272 lines (231 loc) · 6.76 KB
/
run.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
package command
import (
"fmt"
"os"
"os/exec"
"strings"
capi "github.com/hashicorp/consul/api"
hclog "github.com/hashicorp/go-hclog"
vapi "github.com/hashicorp/vault/api"
"github.com/mitchellh/cli"
)
func RunCommandFactory(meta Meta) cli.CommandFactory {
return func() (cli.Command, error) {
return &Run{Meta: meta}, nil
}
}
type Run struct {
Meta
}
func (c *Run) Help() string {
helpText := `
Usage: nomad-e2e run (<provider>/<name>)...
Two modes exist when using the run command.
When no arguments are given to the run command, it will launch
the e2e test suite against the Nomad cluster specified by the
NOMAD_ADDR environment variable. If this is not set, it defaults
to 'http://localhost:4646'
Multiple arguments may be given to specify one or more environments to
provision and run the e2e tests against. These are given in the form of
<provider>/<name>. Globs are support, for example 'aws/*' would run tests
against all of the environments under the aws provider. When using this mode,
all of the provision flags are supported.
General Options:
` + generalOptionsUsage() + `
Run Options:
-run regex
Sets a regular expression for what tests to run. Uses '/' as a separator
to allow hierarchy between Suite/Case/Test.
Example '-run MyTestSuite' would only run tests under the MyTestSuite suite.
-slow
If set, will only run test suites marked as slow.
`
return strings.TrimSpace(helpText)
}
func (c *Run) Synopsis() string {
return "Runs the e2e test suite"
}
func (c *Run) Run(args []string) int {
var envPath string
var nomadBinary string
var tfPath string
var slow bool
var run string
cmdFlags := c.FlagSet("run")
cmdFlags.Usage = func() { c.Ui.Output(c.Help()) }
cmdFlags.StringVar(&envPath, "env-path", DefaultEnvironmentsPath, "Path to e2e environment terraform configs")
cmdFlags.StringVar(&nomadBinary, "nomad-binary", "", "")
cmdFlags.StringVar(&tfPath, "tf-path", "", "")
cmdFlags.StringVar(&run, "run", "", "Regex to target specific test suites/cases")
cmdFlags.BoolVar(&slow, "slow", false, "Toggle slow running suites")
if err := cmdFlags.Parse(args); err != nil {
c.logger.Error("failed to parse flags", "error", err)
return 1
}
if c.verbose {
c.logger.SetLevel(hclog.Debug)
}
args = cmdFlags.Args()
if len(args) == 0 {
c.logger.Info("no environments specified, running test suite locally")
report, err := c.runTest(&runOpts{
run: run,
slow: slow,
verbose: c.verbose,
})
if err != nil {
c.logger.Error("failed to run test suite", "error", err)
return 1
}
if report.TotalFailedTests > 0 {
c.Ui.Error("***FAILED***")
c.Ui.Error(report.Summary())
return 1
}
c.Ui.Output("PASSED!")
if c.verbose {
c.Ui.Output(report.Summary())
}
return 0
}
environments := []*environment{}
for _, e := range args {
if len(strings.Split(e, "/")) != 2 {
c.logger.Error("argument should be formated as <provider>/<environment>", "args", e)
return 1
}
envs, err := envsFromGlob(envPath, e, tfPath, c.logger)
if err != nil {
c.logger.Error("failed to build environment", "environment", e, "error", err)
return 1
}
environments = append(environments, envs...)
}
// Use go-getter to fetch the nomad binary
nomadPath, err := fetchBinary(nomadBinary)
defer os.RemoveAll(nomadPath)
if err != nil {
c.logger.Error("failed to fetch nomad binary", "error", err)
return 1
}
envCount := len(environments)
c.logger.Debug("starting tests", "totalEnvironments", envCount)
failedEnvs := map[string]*TestReport{}
for i, env := range environments {
logger := c.logger.With("name", env.name, "provider", env.provider)
logger.Debug("provisioning environment")
results, err := env.provision(nomadPath)
if err != nil {
logger.Error("failed to provision environment", "error", err)
return 1
}
opts := &runOpts{
provider: env.provider,
env: env.name,
slow: slow,
run: run,
verbose: c.verbose,
nomadAddr: results.nomadAddr,
consulAddr: results.consulAddr,
vaultAddr: results.vaultAddr,
}
var report *TestReport
if report, err = c.runTest(opts); err != nil {
logger.Error("failed to run tests against environment", "error", err)
return 1
}
if report.TotalFailedTests > 0 {
c.Ui.Error(fmt.Sprintf("[%d/%d] %s: ***FAILED***", i+1, envCount, env.canonicalName()))
c.Ui.Error(fmt.Sprintf("[%d/%d] %s: %s", i+1, envCount, env.canonicalName(), report.Summary()))
failedEnvs[env.canonicalName()] = report
}
c.Ui.Output(fmt.Sprintf("[%d/%d] %s: PASSED!", i+1, envCount, env.canonicalName()))
if c.verbose {
c.Ui.Output(fmt.Sprintf("[%d/%d] %s: %s", i+1, envCount, env.canonicalName(), report.Summary()))
}
}
if len(failedEnvs) > 0 {
c.Ui.Error(fmt.Sprintf("The following environments ***FAILED***"))
for name, report := range failedEnvs {
c.Ui.Error(fmt.Sprintf(" [%s]: %d out of %d suite failures",
name, report.TotalFailedSuites, report.TotalSuites))
}
return 1
}
c.Ui.Output("All Environments PASSED!")
return 0
}
func (c *Run) runTest(opts *runOpts) (*TestReport, error) {
goBin, err := exec.LookPath("go")
if err != nil {
return nil, err
}
cmd := exec.Command(goBin, opts.goArgs()...)
cmd.Env = opts.goEnv()
out, err := cmd.StdoutPipe()
defer out.Close()
if err != nil {
return nil, err
}
err = cmd.Start()
if err != nil {
return nil, err
}
dec := NewDecoder(out)
report, err := dec.Decode(c.logger.Named("run.gotest"))
if err != nil {
return nil, err
}
return report, nil
}
// runOpts contains fields used to build the arguments and environment variabled
// nessicary to run go test and initialize the e2e framework
type runOpts struct {
nomadAddr string
consulAddr string
vaultAddr string
provider string
env string
run string
local bool
slow bool
verbose bool
}
// goArgs returns the list of arguments passed to the go command to start the
// e2e test framework
func (opts *runOpts) goArgs() []string {
a := []string{
"test",
"-json",
}
if opts.run != "" {
a = append(a, "-run=TestE2E/"+opts.run)
}
a = append(a, []string{
"github.com/hashicorp/nomad/e2e",
"-env=" + opts.env,
"-env.provider=" + opts.provider,
}...)
if opts.slow {
a = append(a, "-slow")
}
if opts.local {
a = append(a, "-local")
}
return a
}
// goEnv returns the list of environment variabled passed to the go command to start
// the e2e test framework
func (opts *runOpts) goEnv() []string {
env := append(os.Environ(), "NOMAD_E2E=1")
if opts.nomadAddr != "" {
env = append(env, "NOMAD_ADDR="+opts.nomadAddr)
}
if opts.consulAddr != "" {
env = append(env, fmt.Sprintf("%s=%s", capi.HTTPAddrEnvName, opts.consulAddr))
}
if opts.vaultAddr != "" {
env = append(env, fmt.Sprintf("%s=%s", vapi.EnvVaultAddress, opts.consulAddr))
}
return env
}