forked from hashicorp/nomad
-
Notifications
You must be signed in to change notification settings - Fork 0
/
framework.go
209 lines (181 loc) · 6.05 KB
/
framework.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
package framework
import (
"flag"
"fmt"
"reflect"
"strings"
"testing"
)
var fEnv = flag.String("env", "", "name of the environment executing against")
var fProvider = flag.String("env.provider", "", "cloud provider for which environment is executing against")
var fOS = flag.String("env.os", "", "operating system for which the environment is executing against")
var fArch = flag.String("env.arch", "", "cpu architecture for which the environment is executing against")
var fTags = flag.String("env.tags", "", "comma delimited list of tags associated with the environment")
var fLocal = flag.Bool("local", false, "denotes execution is against a local environment")
var fSlow = flag.Bool("slow", false, "toggles execution of slow test suites")
var fForceRun = flag.Bool("forceRun", false, "if set, skips all environment checks when filtering test suites")
var pkgFramework = New()
type Framework struct {
suites []*TestSuite
provisioner Provisioner
env Environment
isLocalRun bool
slow bool
force bool
}
// Environment includes information about the target environment the test
// framework is targeting. During 'go run', these fields are populated by
// the following flags:
//
// -env <string> "name of the environment executing against"
// -env.provider <string> "cloud provider for which the environment is executing against"
// -env.os <string> "operating system of environment"
// -env.arch <string> "cpu architecture of environment"
// -env.tags <string> "comma delimited list of environment tags"
//
// These flags are not needed when executing locally
type Environment struct {
Name string
Provider string
OS string
Arch string
Tags map[string]struct{}
}
// New creates a Framework
func New() *Framework {
env := Environment{
Name: *fEnv,
Provider: *fProvider,
OS: *fOS,
Arch: *fArch,
Tags: map[string]struct{}{},
}
for _, tag := range strings.Split(*fTags, ",") {
env.Tags[tag] = struct{}{}
}
return &Framework{
provisioner: DefaultProvisioner,
env: env,
isLocalRun: *fLocal,
slow: *fSlow,
force: *fForceRun,
}
}
// AddSuites adds a set of test suites to a Framework
func (f *Framework) AddSuites(s ...*TestSuite) *Framework {
f.suites = append(f.suites, s...)
return f
}
// AddSuites adds a set of test suites to the package scoped Framework
func AddSuites(s ...*TestSuite) *Framework {
pkgFramework.AddSuites(s...)
return pkgFramework
}
// Run starts the test framework, running each TestSuite
func (f *Framework) Run(t *testing.T) {
for _, s := range f.suites {
t.Run(s.Component, func(t *testing.T) {
skip, err := f.runSuite(t, s)
if skip {
t.Skipf("skipping suite '%s': %v", s.Component, err)
return
}
if err != nil {
t.Errorf("error starting suite '%s': %v", s.Component, err)
}
})
}
}
// Run starts the package scoped Framework, running each TestSuite
func Run(t *testing.T) {
pkgFramework.Run(t)
}
// runSuite is called from Framework.Run inside of a sub test for each TestSuite.
// If skip is returned as true, the test suite is skipped with the error text added
// to the Skip reason
// If skip is false and an error is returned, the test suite is failed.
func (f *Framework) runSuite(t *testing.T, s *TestSuite) (skip bool, err error) {
// If -forceRun is set, skip all constraint checks
if !f.force {
// If this is a local run, check that the suite supports running locally
if !s.CanRunLocal && f.isLocalRun {
return true, fmt.Errorf("local run detected and suite cannot run locally")
}
// Check that constraints are met
if err := s.Constraints.matches(f.env); err != nil {
return true, fmt.Errorf("constraint failed: %v", err)
}
// Check the slow toggle and if the suite's slow flag is that same
if f.slow != s.Slow {
return true, fmt.Errorf("framework slow suite configuration is %v but suite is %v", f.slow, s.Slow)
}
}
for _, c := range s.Cases {
// The test name is set to the name of the implementing type, including package
name := fmt.Sprintf("%T", c)
// Each TestCase is provisioned a Nomad cluster to test against.
// This varies by the provisioner implementation used, currently
// the default behavior is for every Test case to use a single shared
// Nomad cluster.
info, err := f.provisioner.ProvisionCluster(ProvisionerOptions{
Name: name,
ExpectConsul: s.Consul,
ExpectVault: s.Vault,
})
if err != nil {
return false, fmt.Errorf("could not provision cluster for case: %v", err)
}
defer f.provisioner.DestroyCluster(info.ID)
c.setClusterInfo(info)
// Each TestCase runs as a subtest of the TestSuite
t.Run(c.Name(), func(t *testing.T) {
// If the TestSuite has Parallel set, all cases run in parallel
if s.Parallel {
t.Parallel()
}
f := newF(t)
// Check if the case includes a before all function
if beforeAllTests, ok := c.(BeforeAllTests); ok {
beforeAllTests.BeforeAll(f)
}
// Check if the case includes an after all function at the end
defer func() {
if afterAllTests, ok := c.(AfterAllTests); ok {
afterAllTests.AfterAll(f)
}
}()
// Here we need to iterate through the methods of the case to find
// ones that are test functions
reflectC := reflect.TypeOf(c)
for i := 0; i < reflectC.NumMethod(); i++ {
method := reflectC.Method(i)
if ok := isTestMethod(method.Name); !ok {
continue
}
// Each test is run as its own sub test of the case
// Test cases are never parallel
t.Run(method.Name, func(t *testing.T) {
cF := newFFromParent(f, t)
if BeforeEachTest, ok := c.(BeforeEachTest); ok {
BeforeEachTest.BeforeEach(cF)
}
defer func() {
if afterEachTest, ok := c.(AfterEachTest); ok {
afterEachTest.AfterEach(cF)
}
}()
//Call the method
method.Func.Call([]reflect.Value{reflect.ValueOf(c), reflect.ValueOf(cF)})
})
}
})
}
return false, nil
}
func isTestMethod(m string) bool {
if !strings.HasPrefix(m, "Test") {
return false
}
// THINKING: adding flag to target a specific test or test regex?
return true
}