-
Notifications
You must be signed in to change notification settings - Fork 154
/
ginkgo.go
263 lines (215 loc) · 7.57 KB
/
ginkgo.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
/*
Copyright 2022 The Kubermatic Kubernetes Platform contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"bufio"
"context"
"encoding/xml"
"errors"
"fmt"
"os"
"os/exec"
"path"
"strings"
"time"
"github.com/onsi/ginkgo/reporters"
"go.uber.org/zap"
"k8s.io/apimachinery/pkg/util/wait"
ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
)
type GinkgoResult struct {
Logfile string
Report *reporters.JUnitTestSuite
Duration time.Duration
}
const (
argSeparator = ` \
`
)
type GinkgoRun struct {
Name string
Cmd *exec.Cmd
ReportsDir string
Timeout time.Duration
}
func (r *GinkgoRun) Run(ctx context.Context, parentLog *zap.SugaredLogger, client ctrlruntimeclient.Client) (*GinkgoResult, error) {
log := parentLog.With("reports-dir", r.ReportsDir)
timedCtx, cancel := context.WithTimeout(ctx, r.Timeout)
defer cancel()
// We're clearing up the temp dir on every run
if err := os.RemoveAll(r.ReportsDir); err != nil {
log.Errorw("Failed to remove temporary reports directory", zap.Error(err))
}
if err := os.MkdirAll(r.ReportsDir, os.ModePerm); err != nil {
return nil, fmt.Errorf("failed to create temporary reports directory: %w", err)
}
// Make sure we write to a file instead of a byte buffer as the logs are pretty big
file, err := os.CreateTemp("/tmp", r.Name+"-log")
if err != nil {
return nil, fmt.Errorf("failed to open logfile: %w", err)
}
defer file.Close()
log = log.With("ginkgo-log", file.Name())
writer := bufio.NewWriter(file)
defer writer.Flush()
started := time.Now()
// Copy the command as we cannot execute a command twice
cmd := exec.CommandContext(timedCtx, "")
cmd.Path = r.Cmd.Path
cmd.Args = r.Cmd.Args
cmd.Env = r.Cmd.Env
cmd.Dir = r.Cmd.Dir
cmd.ExtraFiles = r.Cmd.ExtraFiles
if _, err := writer.Write([]byte(strings.Join(cmd.Args, argSeparator))); err != nil {
return nil, fmt.Errorf("failed to write command to log: %w", err)
}
log.Infof("Starting Ginkgo run '%s'...", r.Name)
// Flush to disk so we can actually watch logs
stopCh := make(chan struct{}, 1)
defer close(stopCh)
go wait.Until(func() {
if err := writer.Flush(); err != nil {
log.Warnw("Failed to flush log writer", zap.Error(err))
}
if err := file.Sync(); err != nil {
log.Warnw("Failed to sync log file", zap.Error(err))
}
}, 1*time.Second, stopCh)
cmd.Stdout = writer
cmd.Stderr = writer
if err := cmd.Run(); err != nil {
// did the context's timeout kick in?
if ctxErr := timedCtx.Err(); ctxErr != nil {
return nil, ctxErr
}
var exitErr *exec.ExitError
if errors.As(err, &exitErr) {
log.Debugf("Ginkgo exited with a non-zero return code %d: %v", exitErr.ExitCode(), exitErr)
} else {
return nil, fmt.Errorf("ginkgo failed to start: %w", err)
}
}
log.Debug("Ginkgo run completed, collecting reports...")
// When running ginkgo in parallel, each ginkgo worker creates a own report, thus we must combine them
combinedReport, err := collectReports(r.Name, r.ReportsDir)
if err != nil {
return nil, err
}
// If we have no junit files, we cannot return a valid report
if len(combinedReport.TestCases) == 0 {
return nil, errors.New("Ginkgo report is empty, it seems no tests where executed")
}
combinedReport.Time = time.Since(started).Seconds()
log.Infof("Ginkgo run '%s' took %s", r.Name, time.Since(started))
return &GinkgoResult{
Logfile: file.Name(),
Report: combinedReport,
Duration: time.Since(started),
}, nil
}
func collectReports(name, reportsDir string) (*reporters.JUnitTestSuite, error) {
files, err := os.ReadDir(reportsDir)
if err != nil {
return nil, fmt.Errorf("failed to list files in reportsDir '%s': %w", reportsDir, err)
}
resultSuite := &reporters.JUnitTestSuite{Name: name}
var individualReportFiles []string
for _, f := range files {
if f.IsDir() {
continue
}
if !strings.HasPrefix(f.Name(), "junit_") || !strings.HasSuffix(f.Name(), ".xml") {
continue
}
absName := path.Join(reportsDir, f.Name())
individualReportFiles = append(individualReportFiles, absName)
suite, err := parseGinkgoJUnitReport(absName)
if err != nil {
return nil, fmt.Errorf("failed to read file '%s': %w", absName, err)
}
AppendReport(resultSuite, suite)
}
for _, f := range individualReportFiles {
if err := os.Remove(f); err != nil {
return nil, fmt.Errorf("failed to remove report file: %w", err)
}
}
return resultSuite, nil
}
// JUnitTestSuites is copied from Ginkgo 2.x because KKP otherwise
// depends on Ginkgo 1.x for now. Once we migrate everything, this
// struct can be removed.
// Source: https://github.com/onsi/ginkgo/blob/9c9f45ef886e23dcb006aa245f67e91140e1e60e/reporters/junit_report.go#L24-L39
type JUnitTestSuites struct {
XMLName xml.Name `xml:"testsuites"`
Tests int `xml:"tests,attr"`
Disabled int `xml:"disabled,attr"`
Errors int `xml:"errors,attr"`
Failures int `xml:"failures,attr"`
Time float64 `xml:"time,attr"`
TestSuites []reporters.JUnitTestSuite `xml:"testsuite"`
}
func parseGinkgoJUnitReport(junitFile string) (*reporters.JUnitTestSuite, error) {
b, err := os.ReadFile(junitFile)
if err != nil {
return nil, fmt.Errorf("failed to read file: %w", err)
}
// Ginkgo 1.x creates reports that only have a <testsuite> element at their root.
// Ginkgo 2.x (Kubernetes 1.25+) creates reports that have a <testsuites>
// element at the root, containing a single <testsuite> child.
// As we have other dependencies on Ginkgo, this code cannot rely on
// reporters.JUnitTestSuites (plural) and instead uses a copy.
// try the 2.x version first
suites := &JUnitTestSuites{}
if err := xml.Unmarshal(b, suites); err == nil {
if count := len(suites.TestSuites); count != 1 {
return nil, fmt.Errorf("expected exactly one <testsuite> element in the report, but found %d", count)
}
return &suites.TestSuites[0], nil
}
// not a 2.x, so maybe it's a 1.x file
suite := &reporters.JUnitTestSuite{}
if err := xml.Unmarshal(b, suite); err != nil {
return nil, fmt.Errorf("failed to unmarshal: %w", err)
}
return suite, nil
}
// AppendReport appends a reporters.JUnitTestSuite to another.
// During that process, test cases are deduplicated (identified via their name).
// Successful or failing test cases always take precedence over skipped test cases.
func AppendReport(report, toAppend *reporters.JUnitTestSuite) {
report.Errors += toAppend.Errors
for _, testCase := range toAppend.TestCases {
exists := false
for i, existingCase := range report.TestCases {
if testCase.Name == existingCase.Name && testCase.ClassName == existingCase.ClassName {
if testCase.Skipped == nil && existingCase.Skipped != nil {
// override existing test case
report.TestCases[i] = testCase
if testCase.FailureMessage != nil {
report.Failures += 1
}
}
// skip over adding this test case
exists = true
}
}
if !exists {
report.TestCases = append(report.TestCases, testCase)
if testCase.FailureMessage != nil {
report.Failures += 1
}
report.Tests += 1
}
}
}