forked from evergreen-ci/evergreen
-
Notifications
You must be signed in to change notification settings - Fork 0
/
repotracker.go
520 lines (475 loc) · 19.2 KB
/
repotracker.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
package repotracker
import (
"fmt"
"time"
"github.com/10gen-labs/slogger/v1"
"github.com/evergreen-ci/evergreen"
"github.com/evergreen-ci/evergreen/model"
"github.com/evergreen-ci/evergreen/model/build"
"github.com/evergreen-ci/evergreen/model/version"
"github.com/evergreen-ci/evergreen/notify"
"github.com/evergreen-ci/evergreen/thirdparty"
"github.com/evergreen-ci/evergreen/util"
"github.com/evergreen-ci/evergreen/validator"
"gopkg.in/yaml.v2"
)
const (
// determines the default maximum number of revisions to fetch for a newly tracked repo
// if not specified in configuration file
DefaultNumNewRepoRevisionsToFetch = 200
DefaultMaxRepoRevisionsToSearch = 50
)
// RepoTracker is used to manage polling repository changes and storing such
// changes. It contains a number of interfaces that specify behavior required by
// client implementations
type RepoTracker struct {
*evergreen.Settings
*model.ProjectRef
RepoPoller
}
// The RepoPoller interface specifies behavior required of all repository poller
// implementations
type RepoPoller interface {
// Fetches the contents of a remote repository's configuration data as at
// the given revision.
GetRemoteConfig(revision string) (*model.Project, error)
// Fetches a list of all filepaths modified by a given revision.
GetChangedFiles(revision string) ([]string, error)
// Fetches all changes since the 'revision' specified - with the most recent
// revision appearing as the first element in the slice.
//
// 'maxRevisionsToSearch' determines the maximum number of revisions we
// allow to search through - in order to find 'revision' - before we give
// up. A value <= 0 implies we allow to search through till we hit the first
// revision for the project.
GetRevisionsSince(sinceRevision string, maxRevisions int) ([]model.Revision, error)
// Fetches the most recent 'numNewRepoRevisionsToFetch' revisions for a
// project - with the most recent revision appearing as the first element in
// the slice.
GetRecentRevisions(numNewRepoRevisionsToFetch int) ([]model.Revision, error)
}
type projectConfigError struct {
Errors []string
Warnings []string
}
func (p projectConfigError) Error() string {
return "Invalid project configuration"
}
// The FetchRevisions method is used by a RepoTracker to run the pipeline for
// tracking repositories. It performs everything from polling the repository to
// persisting any changes retrieved from the repository reference.
func (repoTracker *RepoTracker) FetchRevisions(numNewRepoRevisionsToFetch int) (
err error) {
settings := repoTracker.Settings
projectRef := repoTracker.ProjectRef
projectIdentifier := projectRef.String()
if !projectRef.Enabled {
evergreen.Logger.Logf(slogger.INFO, "Skipping disabled project “%v”", projectRef)
return nil
}
repository, err := model.FindRepository(projectIdentifier)
if err != nil {
return fmt.Errorf("error finding repository '%v': %v",
projectIdentifier, err)
}
var revisions []model.Revision
var lastRevision string
if repository != nil {
lastRevision = repository.LastRevision
}
if lastRevision == "" {
// if this is the first time we're running the tracker for this project,
// fetch the most recent `numNewRepoRevisionsToFetch` revisions
evergreen.Logger.Logf(slogger.INFO, "No last recorded repository revision "+
"for “%v”. Proceeding to fetch most recent %v revisions",
projectRef, numNewRepoRevisionsToFetch)
revisions, err = repoTracker.GetRecentRevisions(numNewRepoRevisionsToFetch)
} else {
evergreen.Logger.Logf(slogger.INFO, "Last recorded repository revision for "+
"“%v” is “%v”", projectRef, lastRevision)
// if the projectRef has a repotracker error then don't get the revisions
if projectRef.RepotrackerError != nil {
if projectRef.RepotrackerError.Exists {
evergreen.Logger.Logf(slogger.ERROR, "repotracker error for base revision, %v",
projectRef.RepotrackerError.InvalidRevision)
return nil
}
}
max := settings.RepoTracker.MaxRepoRevisionsToSearch
if max <= 0 {
max = DefaultMaxRepoRevisionsToSearch
}
revisions, err = repoTracker.GetRevisionsSince(lastRevision, max)
}
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "error fetching revisions for "+
"repository “%v”: %v", projectRef, err)
repoTracker.sendFailureNotification(lastRevision, err)
return nil
}
if len(revisions) > 0 {
lastVersion, err := repoTracker.StoreRevisions(revisions)
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "error storing revisions for "+
"repository %v: %v", projectRef, err)
return err
}
err = model.UpdateLastRevision(lastVersion.Identifier, lastVersion.Revision)
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "error updating last revision for "+
"repository %v: %v", projectRef, err)
return err
}
}
// fetch the most recent, non-ignored version version to activate
activateVersion, err := version.FindOne(version.ByMostRecentNonignored(projectIdentifier))
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "error getting most recent version for "+
"repository %v: %v", projectRef, err)
return err
}
if activateVersion == nil {
evergreen.Logger.Logf(slogger.WARN, "no version to activate for repository %v", projectIdentifier)
return nil
}
err = repoTracker.activateElapsedBuilds(activateVersion)
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "error activating variants: %v", err)
return err
}
return nil
}
// Activates any builds if their BatchTimes have elapsed.
func (repoTracker *RepoTracker) activateElapsedBuilds(v *version.Version) (err error) {
projectId := repoTracker.ProjectRef.Identifier
hasActivated := false
now := time.Now()
for i, status := range v.BuildVariants {
// last comparison is to check that ActivateAt is actually set
if !status.Activated && now.After(status.ActivateAt) && !status.ActivateAt.IsZero() {
evergreen.Logger.Logf(slogger.INFO, "activating variant %v for project %v, revision %v",
status.BuildVariant, projectId, v.Revision)
// Go copies the slice value, we want to modify the actual value
status.Activated = true
status.ActivateAt = now
v.BuildVariants[i] = status
b, err := build.FindOne(build.ById(status.BuildId))
if err != nil {
evergreen.Logger.Logf(slogger.ERROR,
"error retrieving build for project %v, variant %v, build %v: %v",
projectId, status.BuildVariant, status.BuildId, err)
continue
}
evergreen.Logger.Logf(slogger.INFO, "activating build %v for project %v, variant %v",
status.BuildId, projectId, status.BuildVariant)
// Don't need to set the version in here since we do it ourselves in a single update
if err = model.SetBuildActivation(b.Id, true, evergreen.DefaultTaskActivator); err != nil {
evergreen.Logger.Logf(slogger.ERROR, "error activating build %v for project %v, variant %v: %v",
b.Id, projectId, status.BuildVariant, err)
continue
}
hasActivated = true
}
}
// If any variants were activated, update the stored version so that we don't
// attempt to activate them again
if hasActivated {
return v.UpdateBuildVariants()
}
return nil
}
// sendFailureNotification sends a notification to the MCI Team when the
// repotracker is unable to fetch revisions from a given project ref
func (repoTracker *RepoTracker) sendFailureNotification(lastRevision string,
err error) {
// Send a notification to the MCI team
settings := repoTracker.Settings
max := settings.RepoTracker.MaxRepoRevisionsToSearch
if max <= 0 {
max = DefaultMaxRepoRevisionsToSearch
}
projectRef := repoTracker.ProjectRef
subject := fmt.Sprintf(notify.RepotrackerFailurePreface,
projectRef.Identifier, lastRevision)
url := fmt.Sprintf("%v/%v/%v/commits/%v", thirdparty.GithubBase,
projectRef.Owner, projectRef.Repo, projectRef.Branch)
message := fmt.Sprintf("Could not find last known revision '%v' "+
"within the most recent %v revisions at %v: %v", lastRevision, max, url, err)
nErr := notify.NotifyAdmins(subject, message, settings)
if nErr != nil {
evergreen.Logger.Logf(slogger.ERROR, "error sending email: %v", nErr)
}
}
// Verifies that the given revision order number is higher than the latest number stored for the project.
func sanityCheckOrderNum(revOrderNum int, projectId string) error {
latest, err := version.FindOne(version.ByMostRecentForRequester(projectId, evergreen.RepotrackerVersionRequester))
if err != nil {
return fmt.Errorf("Error getting latest version: %v", err.Error())
}
// When there are no versions in the db yet, sanity check is moot
if latest != nil {
if revOrderNum <= latest.RevisionOrderNumber {
return fmt.Errorf("Commit order number isn't greater than last stored version's: %v <= %v",
revOrderNum, latest.RevisionOrderNumber)
}
}
return nil
}
// Constructs all versions stored from recent repository revisions
// The additional complexity is due to support for project modifications on patch builds.
// We need to parse the remote config as it existed when each revision was created.
// The return value is the most recent version created as a result of storing the revisions.
// This function is idempotent with regard to storing the same version multiple times.
func (repoTracker *RepoTracker) StoreRevisions(revisions []model.Revision) (newestVersion *version.Version, err error) {
defer func() {
if newestVersion != nil {
// Fetch the updated version doc, so that we include buildvariants in the result
newestVersion, err = version.FindOne(version.ById(newestVersion.Id))
}
}()
ref := repoTracker.ProjectRef
for i := len(revisions) - 1; i >= 0; i-- {
revision := revisions[i].Revision
evergreen.Logger.Logf(slogger.INFO, "Processing revision %v in project %v", revision, ref.Identifier)
// We check if the version exists here so we can avoid fetching the github config unnecessarily
existingVersion, err := version.FindOne(version.ByProjectIdAndRevision(ref.Identifier, revisions[i].Revision))
if err != nil {
evergreen.Logger.Logf(slogger.ERROR,
"Error looking up version at %v for project %v: %v", ref.Identifier, revision, err)
}
if existingVersion != nil {
evergreen.Logger.Logf(slogger.INFO,
"Skipping creation of version for project %v, revision %v since"+
" we already have a record for it", ref.Identifier, revision)
// We bind newestVersion here since we still need to return the most recent
// version, even if it already exists
newestVersion = existingVersion
continue
}
// Create the stub of the version (not stored in DB yet)
v, err := NewVersionFromRevision(ref, revisions[i])
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "Error creating version for project %v: %v", ref.Identifier, err)
}
err = sanityCheckOrderNum(v.RevisionOrderNumber, ref.Identifier)
if err != nil { // something seriously wrong (bad data in db?) so fail now
panic(err)
}
project, err := repoTracker.GetProjectConfig(revision)
if err != nil {
projectError, isProjectError := err.(projectConfigError)
if isProjectError {
if len(projectError.Warnings) > 0 {
// Store the warnings and keep going. If we don't have
// any true errors, the version will still be created.
v.Warnings = projectError.Warnings
}
if len(projectError.Errors) > 0 {
// Store just the stub version with the project errors
v.Errors = projectError.Errors
if err := v.Insert(); err != nil {
evergreen.Logger.Logf(slogger.ERROR,
"Failed storing stub version for project %v: %v", ref.Identifier, err)
return nil, err
}
newestVersion = v
continue
}
} else {
// Fatal error - don't store the stub
evergreen.Logger.Logf(slogger.INFO,
"Failed to get config for project %v at revision %v: %v", ref.Identifier, revision, err)
return nil, err
}
}
// We have a config, so turn it into a usable yaml string to store with the version doc
projectYamlBytes, err := yaml.Marshal(project)
if err != nil {
return nil, fmt.Errorf("Error marshaling config: %v", err)
}
v.Config = string(projectYamlBytes)
// "Ignore" a version if all changes are to ignored files
if len(project.Ignore) > 0 {
filenames, err := repoTracker.GetChangedFiles(revision)
if err != nil {
return nil, fmt.Errorf("error checking GitHub for ignored files: %v", err)
}
if project.IgnoresAllFiles(filenames) {
v.Ignored = true
}
}
// We rebind newestVersion each iteration, so the last binding will be the newest version
err = createVersionItems(v, ref, project)
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "Error creating version items for %v in project %v: %v",
v.Id, ref.Identifier, err)
return nil, err
}
newestVersion = v
}
return newestVersion, nil
}
// GetProjectConfig fetches the project configuration for a given repository
// returning a remote config if the project references a remote repository
// configuration file - via the Identifier. Otherwise it defaults to the local
// project file. An erroneous project file may be returned along with an error.
func (repoTracker *RepoTracker) GetProjectConfig(revision string) (*model.Project, error) {
projectRef := repoTracker.ProjectRef
if projectRef.LocalConfig != "" {
// return the Local config from the project Ref.
return model.FindProject("", projectRef)
}
project, err := repoTracker.GetRemoteConfig(revision)
if err != nil {
// Only create a stub version on API request errors that pertain
// to actually fetching a config. Those errors currently include:
// thirdparty.APIRequestError, thirdparty.FileNotFoundError and
// thirdparty.YAMLFormatError
_, apiReqErr := err.(thirdparty.APIRequestError)
_, ymlFmtErr := err.(thirdparty.YAMLFormatError)
_, noFileErr := err.(thirdparty.FileNotFoundError)
if apiReqErr || noFileErr || ymlFmtErr {
// If there's an error getting the remote config, e.g. because it
// does not exist, we treat this the same as when the remote config
// is invalid - but add a different error message
message := fmt.Sprintf("error fetching project “%v” configuration "+
"data at revision “%v” (remote path=“%v”): %v",
projectRef.Identifier, revision, projectRef.RemotePath, err)
evergreen.Logger.Logf(slogger.ERROR, message)
return nil, projectConfigError{[]string{message}, nil}
}
// If we get here then we have an infrastructural error - e.g.
// a thirdparty.APIUnmarshalError (indicating perhaps an API has
// changed), a thirdparty.ResponseReadError(problem reading an
// API response) or a thirdparty.APIResponseError (nil API
// response) - or encountered a problem in fetching a local
// configuration file. At any rate, this is bad enough that we
// want to send a notification instead of just creating a stub
// version.
var lastRevision string
repository, fErr := model.FindRepository(projectRef.Identifier)
if fErr != nil || repository == nil {
evergreen.Logger.Logf(slogger.ERROR, "error finding "+
"repository '%v': %v", projectRef.Identifier, fErr)
} else {
lastRevision = repository.LastRevision
}
repoTracker.sendFailureNotification(lastRevision, err)
return nil, err
}
// check if project config is valid
verrs, err := validator.CheckProjectSyntax(project)
if err != nil {
return nil, err
}
if len(verrs) != 0 {
// We have syntax errors in the project.
// Format them, as we need to store + display them to the user
var errMessage, warnMessage string
var projectErrors, projectWarnings []string
for _, e := range verrs {
if e.Level == validator.Warning {
warnMessage += fmt.Sprintf("\n\t=> %v", e)
projectWarnings = append(projectWarnings, e.Error())
} else {
errMessage += fmt.Sprintf("\n\t=> %v", e)
projectErrors = append(projectErrors, e.Error())
}
}
if len(projectErrors) > 0 {
evergreen.Logger.Logf(slogger.ERROR, "Error validating project '%v' "+
"configuration at revision '%v': %v", projectRef.Identifier,
revision, errMessage)
}
if len(projectWarnings) > 0 {
evergreen.Logger.Logf(slogger.ERROR, "Warning while validating project '%v' "+
"configuration at revision '%v': %v", projectRef.Identifier,
revision, warnMessage)
}
return project, projectConfigError{projectErrors, projectWarnings}
}
return project, nil
}
// NewVersionFromRevision populates a new Version with metadata from a model.Revision.
// Does not populate its config or store anything in the database.
func NewVersionFromRevision(ref *model.ProjectRef, rev model.Revision) (*version.Version, error) {
number, err := model.GetNewRevisionOrderNumber(ref.Identifier)
if err != nil {
return nil, err
}
v := &version.Version{
Author: rev.Author,
AuthorEmail: rev.AuthorEmail,
Branch: ref.Branch,
CreateTime: rev.CreateTime,
Id: util.CleanName(fmt.Sprintf("%v_%v", ref.String(), rev.Revision)),
Identifier: ref.Identifier,
Message: rev.RevisionMessage,
Owner: ref.Owner,
RemotePath: ref.RemotePath,
Repo: ref.Repo,
RepoKind: ref.RepoKind,
Requester: evergreen.RepotrackerVersionRequester,
Revision: rev.Revision,
Status: evergreen.VersionCreated,
RevisionOrderNumber: number,
}
return v, nil
}
// createVersionItems populates and stores all the tasks and builds for a version according to
// the given project config.
func createVersionItems(v *version.Version, ref *model.ProjectRef, project *model.Project) error {
// generate all task Ids so that we can easily reference them for dependencies
taskIdTable := model.NewTaskIdTable(project, v)
// create all builds for the version
for _, buildvariant := range project.BuildVariants {
if buildvariant.Disabled {
continue
}
buildId, err := model.CreateBuildFromVersion(project, v, taskIdTable, buildvariant.Name, false, nil)
if err != nil {
return err
}
lastActivated, err := version.FindOne(version.ByLastVariantActivation(ref.Identifier, buildvariant.Name))
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "Error getting activation time for bv %v", buildvariant.Name)
return err
}
var lastActivation *time.Time
if lastActivated != nil {
for _, buildStatus := range lastActivated.BuildVariants {
if buildStatus.BuildVariant == buildvariant.Name && buildStatus.Activated {
lastActivation = &buildStatus.ActivateAt
break
}
}
}
var activateAt time.Time
if lastActivation == nil {
// if we don't have a last activation time then prepare to activate it immediately.
activateAt = time.Now()
} else {
activateAt = lastActivation.Add(time.Minute * time.Duration(ref.GetBatchTime(&buildvariant)))
}
evergreen.Logger.Logf(slogger.INFO, "Going to activate bv %v for project %v, version %v at %v",
buildvariant.Name, ref.Identifier, v.Id, activateAt)
v.BuildIds = append(v.BuildIds, buildId)
v.BuildVariants = append(v.BuildVariants, version.BuildStatus{
BuildVariant: buildvariant.Name,
Activated: false,
ActivateAt: activateAt,
BuildId: buildId,
})
}
if err := v.Insert(); err != nil {
evergreen.Logger.Errorf(slogger.ERROR, "Error inserting version %v: %v", v.Id, err)
for _, buildStatus := range v.BuildVariants {
if buildErr := model.DeleteBuild(buildStatus.BuildId); buildErr != nil {
evergreen.Logger.Errorf(slogger.ERROR, "Error deleting build %v: %v",
buildStatus.BuildId, buildErr)
}
}
return err
}
return nil
}