-
Notifications
You must be signed in to change notification settings - Fork 1
/
run_policy.go
392 lines (351 loc) · 14.4 KB
/
run_policy.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
package builds
import (
"strings"
"time"
g "github.com/onsi/ginkgo"
o "github.com/onsi/gomega"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
e2e "k8s.io/kubernetes/test/e2e/framework"
buildapi "github.com/openshift/origin/pkg/build/api"
buildclient "github.com/openshift/origin/pkg/build/client"
buildutil "github.com/openshift/origin/pkg/build/util"
exutil "github.com/openshift/origin/test/extended/util"
)
var _ = g.Describe("[builds][Slow] using build configuration runPolicy", func() {
defer g.GinkgoRecover()
var (
// Use invalid source here as we don't care about the result
oc = exutil.NewCLI("cli-build-run-policy", exutil.KubeConfigPath())
)
g.JustBeforeEach(func() {
g.By("waiting for builder service account")
err := exutil.WaitForBuilderAccount(oc.KubeClient().Core().ServiceAccounts(oc.Namespace()))
o.Expect(err).NotTo(o.HaveOccurred())
// Create all fixtures
oc.Run("create").Args("-f", exutil.FixturePath("testdata", "run_policy")).Execute()
})
g.Describe("build configuration with Parallel build run policy", func() {
g.It("runs the builds in parallel", func() {
g.By("starting multiple builds")
var (
startedBuilds []string
counter int
)
bcName := "sample-parallel-build"
buildWatch, err := oc.Client().Builds(oc.Namespace()).Watch(metav1.ListOptions{
LabelSelector: buildutil.BuildConfigSelector(bcName).String(),
})
defer buildWatch.Stop()
// Start first build
stdout, _, err := exutil.StartBuild(oc, bcName, "-o=name")
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(strings.TrimSpace(stdout)).ShouldNot(o.HaveLen(0))
// extract build name from "build/buildName" resource id
startedBuilds = append(startedBuilds, strings.TrimSpace(strings.Split(stdout, "/")[1]))
// Wait for it to become running
for {
event := <-buildWatch.ResultChan()
build := event.Object.(*buildapi.Build)
o.Expect(buildutil.IsBuildComplete(build)).Should(o.BeFalse())
if build.Name == startedBuilds[0] && build.Status.Phase == buildapi.BuildPhaseRunning {
break
}
}
for i := 0; i < 2; i++ {
stdout, _, err = exutil.StartBuild(oc, bcName, "-o=name")
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(strings.TrimSpace(stdout)).ShouldNot(o.HaveLen(0))
startedBuilds = append(startedBuilds, strings.TrimSpace(strings.Split(stdout, "/")[1]))
}
o.Expect(err).NotTo(o.HaveOccurred())
for {
event := <-buildWatch.ResultChan()
build := event.Object.(*buildapi.Build)
if build.Name == startedBuilds[0] {
if buildutil.IsBuildComplete(build) {
break
}
continue
}
// When the the other two builds we started after waiting for the first
// build to become running are Pending, verify the first build is still
// running (so the other two builds are started in parallel with first
// build).
// TODO: This might introduce flakes in case the first build complete
// sooner or fail.
if build.Status.Phase == buildapi.BuildPhasePending {
c := buildclient.NewOSClientBuildClient(oc.Client())
firstBuildRunning := false
_, err := buildutil.BuildConfigBuilds(c, oc.Namespace(), bcName, func(b buildapi.Build) bool {
if b.Name == startedBuilds[0] && b.Status.Phase == buildapi.BuildPhaseRunning {
firstBuildRunning = true
}
return false
})
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(firstBuildRunning).Should(o.BeTrue())
counter++
}
// When the build failed or completed prematurely, fail the test
o.Expect(buildutil.IsBuildComplete(build)).Should(o.BeFalse())
if counter == 2 {
break
}
}
o.Expect(counter).Should(o.BeEquivalentTo(2))
})
})
g.Describe("build configuration with Serial build run policy", func() {
g.It("runs the builds in serial order", func() {
g.By("starting multiple builds")
var (
startedBuilds []string
counter int
)
bcName := "sample-serial-build"
buildVerified := map[string]bool{}
for i := 0; i < 3; i++ {
stdout, _, err := exutil.StartBuild(oc, bcName, "-o=name")
o.Expect(err).NotTo(o.HaveOccurred())
startedBuilds = append(startedBuilds, strings.TrimSpace(strings.Split(stdout, "/")[1]))
}
buildWatch, err := oc.Client().Builds(oc.Namespace()).Watch(metav1.ListOptions{
LabelSelector: buildutil.BuildConfigSelector(bcName).String(),
})
defer buildWatch.Stop()
o.Expect(err).NotTo(o.HaveOccurred())
sawCompletion := false
for {
event := <-buildWatch.ResultChan()
build := event.Object.(*buildapi.Build)
var lastCompletion time.Time
if build.Status.Phase == buildapi.BuildPhaseComplete {
lastCompletion = time.Now()
o.Expect(build.Status.StartTimestamp).ToNot(o.BeNil(), "completed builds should have a valid start time")
o.Expect(build.Status.CompletionTimestamp).ToNot(o.BeNil(), "completed builds should have a valid completion time")
sawCompletion = true
}
if build.Status.Phase == buildapi.BuildPhaseRunning {
latency := lastCompletion.Sub(time.Now())
o.Expect(latency).To(o.BeNumerically("<", 10*time.Second), "next build should have started less than 10s after last completed build")
// Ignore events from complete builds (if there are any) if we already
// verified the build.
if _, exists := buildVerified[build.Name]; exists {
continue
}
// Verify there are no other running or pending builds than this
// build as serial build always runs alone.
c := buildclient.NewOSClientBuildClient(oc.Client())
builds, err := buildutil.BuildConfigBuilds(c, oc.Namespace(), bcName, func(b buildapi.Build) bool {
if b.Name == build.Name {
return false
}
if b.Status.Phase == buildapi.BuildPhaseRunning || b.Status.Phase == buildapi.BuildPhasePending {
return true
}
return false
})
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(builds.Items).Should(o.BeEmpty())
// The builds should start in the same order as they were created.
o.Expect(build.Name).Should(o.BeEquivalentTo(startedBuilds[counter]))
buildVerified[build.Name] = true
counter++
}
if counter == len(startedBuilds) {
break
}
}
o.Expect(sawCompletion).To(o.BeTrue(), "should have seen at least one build complete")
})
})
g.Describe("build configuration with Serial build run policy handling cancellation", func() {
g.It("starts the next build immediately after one is canceled", func() {
g.By("starting multiple builds")
bcName := "sample-serial-build"
for i := 0; i < 3; i++ {
_, _, err := exutil.StartBuild(oc, bcName)
o.Expect(err).NotTo(o.HaveOccurred())
}
buildWatch, err := oc.Client().Builds(oc.Namespace()).Watch(metav1.ListOptions{
LabelSelector: buildutil.BuildConfigSelector(bcName).String(),
})
defer buildWatch.Stop()
o.Expect(err).NotTo(o.HaveOccurred())
var cancelTime, cancelTime2 time.Time
for {
event := <-buildWatch.ResultChan()
build := event.Object.(*buildapi.Build)
if build.Status.Phase == buildapi.BuildPhasePending {
if build.Name == "sample-serial-build-1" {
err := oc.Run("cancel-build").Args("sample-serial-build-1").Execute()
o.Expect(err).ToNot(o.HaveOccurred())
cancelTime = time.Now()
}
if build.Name == "sample-serial-build-2" {
duration := time.Now().Sub(cancelTime)
o.Expect(duration).To(o.BeNumerically("<", 10*time.Second), "next build should have started less than 10s after canceled build")
err := oc.Run("cancel-build").Args("sample-serial-build-2").Execute()
o.Expect(err).ToNot(o.HaveOccurred())
cancelTime2 = time.Now()
}
if build.Name == "sample-serial-build-3" {
duration := time.Now().Sub(cancelTime2)
o.Expect(duration).To(o.BeNumerically("<", 10*time.Second), "next build should have started less than 10s after canceled build")
break
}
}
}
})
})
g.Describe("build configuration with Serial build run policy handling failure", func() {
g.It("starts the next build immediately after one fails", func() {
g.By("starting multiple builds")
bcName := "sample-serial-build-fail"
for i := 0; i < 3; i++ {
_, _, err := exutil.StartBuild(oc, bcName)
o.Expect(err).NotTo(o.HaveOccurred())
}
buildWatch, err := oc.Client().Builds(oc.Namespace()).Watch(metav1.ListOptions{
LabelSelector: buildutil.BuildConfigSelector(bcName).String(),
})
defer buildWatch.Stop()
o.Expect(err).NotTo(o.HaveOccurred())
var failTime, failTime2 time.Time
done, timestamps1, timestamps2, timestamps3 := false, false, false, false
for done == false {
select {
case event := <-buildWatch.ResultChan():
build := event.Object.(*buildapi.Build)
if build.Status.Phase == buildapi.BuildPhaseFailed {
if build.Name == "sample-serial-build-fail-1" {
// this may not be set on the first build modified to failed event because
// the build gets marked failed by the build pod, but the timestamps get
// set by the buildpod controller.
if build.Status.CompletionTimestamp != nil {
o.Expect(build.Status.StartTimestamp).ToNot(o.BeNil(), "failed builds should have a valid start time")
o.Expect(build.Status.CompletionTimestamp).ToNot(o.BeNil(), "failed builds should have a valid completion time")
timestamps1 = true
}
failTime = time.Now()
}
if build.Name == "sample-serial-build-fail-2" {
duration := time.Now().Sub(failTime)
o.Expect(duration).To(o.BeNumerically("<", 10*time.Second), "next build should have started less than 10s after failed build")
if build.Status.CompletionTimestamp != nil {
o.Expect(build.Status.StartTimestamp).ToNot(o.BeNil(), "failed builds should have a valid start time")
o.Expect(build.Status.CompletionTimestamp).ToNot(o.BeNil(), "failed builds should have a valid completion time")
timestamps2 = true
}
failTime2 = time.Now()
}
if build.Name == "sample-serial-build-fail-3" {
duration := time.Now().Sub(failTime2)
o.Expect(duration).To(o.BeNumerically("<", 10*time.Second), "next build should have started less than 10s after failed build")
if build.Status.CompletionTimestamp != nil {
o.Expect(build.Status.StartTimestamp).ToNot(o.BeNil(), "failed builds should have a valid start time")
o.Expect(build.Status.CompletionTimestamp).ToNot(o.BeNil(), "failed builds should have a valid completion time")
timestamps3 = true
}
}
}
// once we have all the expected timestamps, or we run out of time, we can bail.
if timestamps1 && timestamps2 && timestamps3 {
done = true
}
case <-time.After(2 * time.Minute):
// we've waited as long as we dare, go see if we got all the timestamp data we expected.
// if we have the timestamp data, we also know that we checked the next build start latency.
done = true
}
}
o.Expect(timestamps1).To(o.BeTrue(), "failed builds should have start and completion timestamps set")
o.Expect(timestamps2).To(o.BeTrue(), "failed builds should have start and completion timestamps set")
o.Expect(timestamps3).To(o.BeTrue(), "failed builds should have start and completion timestamps set")
})
})
g.Describe("build configuration with SerialLatestOnly build run policy", func() {
g.It("runs the builds in serial order but cancel previous builds", func() {
g.By("starting multiple builds")
var (
startedBuilds []string
expectedRunningBuild int
wasCancelled bool
)
bcName := "sample-serial-latest-only-build"
buildVerified := map[string]bool{}
buildWatch, err := oc.Client().Builds(oc.Namespace()).Watch(metav1.ListOptions{
LabelSelector: buildutil.BuildConfigSelector(bcName).String(),
})
defer buildWatch.Stop()
o.Expect(err).NotTo(o.HaveOccurred())
stdout, _, err := exutil.StartBuild(oc, bcName, "-o=name")
o.Expect(err).NotTo(o.HaveOccurred())
startedBuilds = append(startedBuilds, strings.TrimSpace(strings.Split(stdout, "/")[1]))
// Wait for the first build to become running
for {
event := <-buildWatch.ResultChan()
build := event.Object.(*buildapi.Build)
if build.Name == startedBuilds[0] {
if build.Status.Phase == buildapi.BuildPhaseRunning {
buildVerified[build.Name] = true
// now expect the last build to be run.
expectedRunningBuild = 2
break
}
o.Expect(buildutil.IsBuildComplete(build)).Should(o.BeFalse())
}
}
// Trigger two more builds
for i := 0; i < 2; i++ {
stdout, _, err = exutil.StartBuild(oc, bcName, "-o=name")
o.Expect(err).NotTo(o.HaveOccurred())
startedBuilds = append(startedBuilds, strings.TrimSpace(strings.Split(stdout, "/")[1]))
}
// Verify that the first build will complete and the next build to run
// will be the last build created.
for {
event := <-buildWatch.ResultChan()
build := event.Object.(*buildapi.Build)
e2e.Logf("got event for build %s with phase %s", build.Name, build.Status.Phase)
// The second build should be cancelled
if build.Status.Phase == buildapi.BuildPhaseCancelled {
if build.Name == startedBuilds[1] {
buildVerified[build.Name] = true
wasCancelled = true
}
}
// Only first and third build should actually run (serially).
if build.Status.Phase == buildapi.BuildPhaseRunning {
// Ignore events from complete builds (if there are any) if we already
// verified the build.
if _, exists := buildVerified[build.Name]; exists {
continue
}
// Verify there are no other running or pending builds than this
// build as serial build always runs alone.
c := buildclient.NewOSClientBuildClient(oc.Client())
builds, err := buildutil.BuildConfigBuilds(c, oc.Namespace(), bcName, func(b buildapi.Build) bool {
e2e.Logf("[%s] build %s is %s", build.Name, b.Name, b.Status.Phase)
if b.Name == build.Name {
return false
}
if b.Status.Phase == buildapi.BuildPhaseRunning || b.Status.Phase == buildapi.BuildPhasePending {
return true
}
return false
})
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(builds.Items).Should(o.BeEmpty())
// The builds should start in the same order as they were created.
o.Expect(build.Name).Should(o.BeEquivalentTo(startedBuilds[expectedRunningBuild]))
buildVerified[build.Name] = true
}
if len(buildVerified) == len(startedBuilds) {
break
}
}
o.Expect(wasCancelled).Should(o.BeEquivalentTo(true))
})
})
})