forked from go-gremlins/gremlins
-
Notifications
You must be signed in to change notification settings - Fork 0
/
executor.go
268 lines (226 loc) · 8.1 KB
/
executor.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
/*
* Copyright 2022 The Gremlins Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package engine
import (
"context"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"sync"
"time"
"github.com/singhnishant94/gremlins/internal/configuration"
"github.com/singhnishant94/gremlins/internal/engine/workdir"
"github.com/singhnishant94/gremlins/internal/engine/workerpool"
"github.com/singhnishant94/gremlins/internal/gomodule"
"github.com/singhnishant94/gremlins/internal/log"
"github.com/singhnishant94/gremlins/internal/mutator"
)
// DefaultTimeoutCoefficient is the default multiplier for the timeout length
// of each test run.
const DefaultTimeoutCoefficient = 3
// ExecutorDealer is the initializer for new workerpool.Executor.
type ExecutorDealer interface {
NewExecutor(mut mutator.Mutator, outCh chan<- mutator.Mutator, wg *sync.WaitGroup) workerpool.Executor
}
// MutantExecutorDealer is a ExecutorDealer for the initialisation of a mutantExecutor.
//
// By default, it sets uses exec.Command to perform the tests on the source
// code. This can be overridden, for example in tests.
//
// The apply and rollback functions are wrappers around the TokenMutator apply and
// rollback. These can be overridden with nop functions in tests. Not an
// ideal setup. In the future we can think of a better way to handle this.
type MutantExecutorDealer struct {
wdDealer workdir.Dealer
execContext execContext
mod gomodule.GoModule
buildTags string
testExecutionTime time.Duration
dryRun bool
integrationMode bool
testCPU int
}
// ExecutorDealerOption is the defining option for the initialisation of a ExecutorDealer.
type ExecutorDealerOption func(j MutantExecutorDealer) MutantExecutorDealer
// WithExecContext overrides the default exec.Command with a custom executor.
func WithExecContext(c execContext) ExecutorDealerOption {
return func(m MutantExecutorDealer) MutantExecutorDealer {
m.execContext = c
return m
}
}
// NewExecutorDealer initialises a MutantExecutorDealer.
func NewExecutorDealer(mod gomodule.GoModule, wdd workdir.Dealer, elapsed time.Duration, opts ...ExecutorDealerOption) *MutantExecutorDealer {
buildTags := configuration.Get[string](configuration.UnleashTagsKey)
dryRun := configuration.Get[bool](configuration.UnleashDryRunKey)
integrationMode := configuration.Get[bool](configuration.UnleashIntegrationMode)
testCPU := configuration.Get[int](configuration.UnleashTestCPUKey)
tCoefficient := configuration.Get[int](configuration.UnleashTimeoutCoefficientKey)
coefficient := DefaultTimeoutCoefficient
if tCoefficient != 0 {
coefficient = tCoefficient
}
if testCPU != 0 && integrationMode {
testCPU /= testCPU
}
jd := MutantExecutorDealer{
mod: mod,
wdDealer: wdd,
buildTags: buildTags,
dryRun: dryRun,
integrationMode: integrationMode,
testCPU: testCPU,
testExecutionTime: elapsed * time.Duration(coefficient),
execContext: exec.CommandContext,
}
for _, opt := range opts {
jd = opt(jd)
}
return &jd
}
// NewExecutor returns a new workerpool.Executor for the given mutator.Mutator.
// It gets an output channel of mutator.Mutator and a sync.WaitGroup. The channel
// will stream the results of the executor, and the wait group will be done when the
// executor is complete.
func (m MutantExecutorDealer) NewExecutor(mut mutator.Mutator, outCh chan<- mutator.Mutator, wg *sync.WaitGroup) workerpool.Executor {
mj := mutantExecutor{
mutant: mut,
outCh: outCh,
wg: wg,
wdDealer: m.wdDealer,
module: m.mod,
dryRun: m.dryRun,
integrationMode: m.integrationMode,
buildTags: m.buildTags,
execContext: m.execContext,
testCPU: m.testCPU,
testExecutionTime: m.testExecutionTime,
}
return &mj
}
type execContext = func(ctx context.Context, name string, args ...string) *exec.Cmd
type mutantExecutor struct {
mutant mutator.Mutator
wdDealer workdir.Dealer
outCh chan<- mutator.Mutator
wg *sync.WaitGroup
execContext execContext
module gomodule.GoModule
buildTags string
testExecutionTime time.Duration
dryRun bool
integrationMode bool
testCPU int
}
// Start is the implementation of the workerpool.Executor definition and is the
// method responsible for performing the actual mutation testing.
// The executor runs on its mutator.Mutator.
// If it is RUNNABLE, and it is not in dry-run mode, it will apply the mutation,
// run the tests and mark the TokenMutator as either KILLED or LIVED depending
// on the result. If the tests pass, it means the TokenMutator survived, so it
// will be LIVED, if the tests fail, the TokenMutator will be KILLED.
// The timeout of the test is managed outside the run of the test, using
// a context with timeout. This is done because the Go test command doesn't
// make it easy to distinguish failures from timeouts.
func (m *mutantExecutor) Start(w *workerpool.Worker) {
defer m.wg.Done()
workerName := fmt.Sprintf("%s-%d", w.Name, w.ID)
rootDir, err := m.wdDealer.Get(workerName)
if err != nil {
panic("error, this is temporary")
}
workingDir := filepath.Join(rootDir, m.module.CallingDir)
m.mutant.SetWorkdir(workingDir)
if m.mutant.Status() == mutator.NotCovered || m.mutant.Status() == mutator.Skipped || m.dryRun {
m.outCh <- m.mutant
return
}
if err := m.mutant.Apply(); err != nil {
log.Errorf("failed to apply mutation at %s - %s\n\t%v\n", m.mutant.Position(), m.mutant.Status(), err)
return
}
m.mutant.SetStatus(m.runTests(rootDir, m.mutant.Pkg()))
if err := m.mutant.Rollback(); err != nil {
// What should we do now?
log.Errorf("failed to restore mutation at %s - %s\n\t%v", m.mutant.Position(), m.mutant.Status(), err)
}
m.outCh <- m.mutant
}
func (m *mutantExecutor) runTests(rootDir, pkg string) mutator.Status {
ctx, cancel := context.WithTimeout(context.Background(), m.testExecutionTime)
defer cancel()
cmd := m.execContext(ctx, "go", m.getTestArgs(pkg)...)
cmd.Dir = m.mutant.Workdir()
if m.integrationMode {
cmd.Dir = rootDir
}
cmd.Env = append(cmd.Env, os.Environ()...)
cmd.Env = append(cmd.Env, fmt.Sprintf("GOTMPDIR=%s", m.wdDealer.WorkDir()))
rel, err := run(cmd)
defer rel()
m.mutant.SetTestExecutionError(err)
if errors.Is(ctx.Err(), context.DeadlineExceeded) {
return mutator.TimedOut
}
var exitErr *exec.ExitError
if errors.As(err, &exitErr) {
return getTestFailedStatus(exitErr.ExitCode())
}
return mutator.Lived
}
func (m *mutantExecutor) getTestArgs(pkg string) []string {
args := []string{"test"}
if m.buildTags != "" {
args = append(args, "-tags", m.buildTags)
}
// Here we add some seconds to the timeout to be sure it's gremlins that catches the test
// timeout and not the test itself. The timeout on the test prevents the test.* processes
// from hanging forever.
args = append(args, "-timeout", (2*time.Second + m.testExecutionTime).String())
args = append(args, "-failfast")
if m.testCPU != 0 {
args = append(args, fmt.Sprintf("-cpu %d", m.testCPU))
}
path := pkg
if m.integrationMode {
path = "./..."
}
args = append(args, path)
return args
}
func run(cmd *exec.Cmd) (func(), error) {
if err := cmd.Run(); err != nil {
return func() {}, err
}
return func() {
err := cmd.Process.Release()
if err != nil {
_ = cmd.Process.Kill()
}
}, nil
}
func getTestFailedStatus(exitCode int) mutator.Status {
switch exitCode {
case 1:
return mutator.Killed
case 2:
return mutator.NotViable
default:
return mutator.Lived
}
}