forked from buildkite/agent
-
Notifications
You must be signed in to change notification settings - Fork 8
/
artifact_batch_creator.go
80 lines (63 loc) · 2.03 KB
/
artifact_batch_creator.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
package agent
import (
"time"
"github.com/buildkite/agent/api"
"github.com/buildkite/agent/logger"
"github.com/buildkite/agent/retry"
)
type ArtifactBatchCreator struct {
// The APIClient that will be used when uploading jobs
APIClient *api.Client
// The ID of the Job that these artifacts belong to
JobID string
// All the artifacts that need to be created
Artifacts []*api.Artifact
// Where the artifacts are being uploaded to on the command line
UploadDestination string
}
func (a *ArtifactBatchCreator) Create() ([]*api.Artifact, error) {
length := len(a.Artifacts)
chunks := 30
// Split into the artifacts into chunks so we're not uploading a ton of
// files at once.
for i := 0; i < length; i += chunks {
j := i + chunks
if length < j {
j = length
}
// The artifacts that will be uploaded in this chunk
theseArtiacts := a.Artifacts[i:j]
// An ID is required so Buildkite can ensure this create
// operation is idompotent (if we try and upload the same ID
// twice, it'll just return the previous data and skip the
// upload)
batch := &api.ArtifactBatch{api.NewUUID(), theseArtiacts, a.UploadDestination}
logger.Info("Creating (%d-%d)/%d artifacts", i, j, length)
var creation *api.ArtifactBatchCreateResponse
var resp *api.Response
var err error
// Retry the batch upload a couple of times
err = retry.Do(func(s *retry.Stats) error {
creation, resp, err = a.APIClient.Artifacts.Create(a.JobID, batch)
if resp != nil && (resp.StatusCode == 401 || resp.StatusCode == 404 || resp.StatusCode == 500) {
s.Break()
}
if err != nil {
logger.Warn("%s (%s)", err, s)
}
return err
}, &retry.Config{Maximum: 10, Interval: 5 * time.Second})
// Did the batch creation eventually fail?
if err != nil {
return nil, err
}
// Save the id and instructions to each artifact
index := 0
for _, id := range creation.ArtifactIDs {
theseArtiacts[index].ID = id
theseArtiacts[index].UploadInstructions = creation.UploadInstructions
index += 1
}
}
return a.Artifacts, nil
}