-
Notifications
You must be signed in to change notification settings - Fork 124
/
task_log.go
180 lines (151 loc) · 5.43 KB
/
task_log.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
package taskoutput
import (
"context"
"fmt"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/evergreen-ci/evergreen"
"github.com/evergreen-ci/evergreen/apimodels"
"github.com/evergreen-ci/evergreen/model/log"
"github.com/evergreen-ci/utility"
"github.com/mongodb/grip/send"
"github.com/pkg/errors"
)
// TaskLogType represents the recognized types of task logs.
type TaskLogType string
const (
TaskLogTypeAll TaskLogType = "all_logs"
TaskLogTypeAgent TaskLogType = "agent_log"
TaskLogTypeSystem TaskLogType = "system_log"
TaskLogTypeTask TaskLogType = "task_log"
)
func (t TaskLogType) Validate(writing bool) error {
switch t {
case TaskLogTypeAll, TaskLogTypeAgent, TaskLogTypeSystem, TaskLogTypeTask:
default:
return errors.Errorf("unrecognized task log type '%s'", t)
}
if writing && t == TaskLogTypeAll {
return errors.Errorf("cannot persist task log type '%s'", TaskLogTypeAll)
}
return nil
}
// TaskLogOutput is the versioned entry point for coordinating persistent
// storage of a task run's task log data.
type TaskLogOutput struct {
Version int `bson:"version" json:"version"`
BucketConfig evergreen.BucketConfig `bson:"bucket_config" json:"bucket_config"`
AWSCredentials *credentials.Credentials `bson:"-" json:"-"`
}
// ID returns the unique identifier of the task log output type.
// Note that this is distinct from the task log output type subtype `task_log`.
func (TaskLogOutput) ID() string { return "task_logs" }
// TaskLogGetOptions represents the arguments for fetching task logs belonging
// to a task run.
type TaskLogGetOptions struct {
// LogType is the type of task log to fetch. Must be a valid task log
// type.
LogType TaskLogType
// Start is the start time (inclusive) of the time range filter,
// represented as a Unix timestamp in nanoseconds. Defaults to
// unbounded.
Start *int64
// End is the end time (inclusive) of the time range filter,
// represented as a Unix timestamp in nanoseconds. Defaults to
// unbounded.
End *int64
// LineLimit limits the number of lines read from the log. Ignored if
// less than or equal to 0.
LineLimit int
// TailN is the number of lines to read from the tail of the log.
// Ignored if less than or equal to 0.
TailN int
}
// NewSender returns a new task log sender for the given task run.
func (o TaskLogOutput) NewSender(ctx context.Context, taskOpts TaskOptions, senderOpts EvergreenSenderOptions, logType TaskLogType) (send.Sender, error) {
if err := logType.Validate(true); err != nil {
return nil, err
}
svc, err := o.getLogService(ctx)
if err != nil {
return nil, errors.Wrap(err, "getting log service")
}
senderOpts.appendLines = func(ctx context.Context, lines []log.LogLine) error {
return svc.Append(ctx, o.getLogName(taskOpts, logType), lines)
}
return newEvergreenSender(ctx, fmt.Sprintf("%s-%s", taskOpts.TaskID, logType), senderOpts)
}
// Append appends log lines to the specified task log for the given task run.
func (o TaskLogOutput) Append(ctx context.Context, taskOpts TaskOptions, logType TaskLogType, lines []log.LogLine) error {
if err := logType.Validate(true); err != nil {
return err
}
svc, err := o.getLogService(ctx)
if err != nil {
return errors.Wrap(err, "getting log service")
}
return svc.Append(ctx, o.getLogName(taskOpts, logType), lines)
}
// Get returns task logs belonging to the specified task run.
func (o TaskLogOutput) Get(ctx context.Context, taskOpts TaskOptions, getOpts TaskLogGetOptions) (log.LogIterator, error) {
if err := getOpts.LogType.Validate(false); err != nil {
return nil, err
}
if o.Version == 0 {
return o.getBuildloggerLogs(ctx, taskOpts, getOpts)
}
svc, err := o.getLogService(ctx)
if err != nil {
return nil, errors.Wrap(err, "getting log service")
}
return svc.Get(ctx, log.GetOptions{
LogNames: []string{o.getLogName(taskOpts, getOpts.LogType)},
Start: getOpts.Start,
End: getOpts.End,
LineLimit: getOpts.LineLimit,
TailN: getOpts.TailN,
})
}
func (o TaskLogOutput) getLogName(taskOpts TaskOptions, logType TaskLogType) string {
prefix := fmt.Sprintf("%s/%s/%d/%s", taskOpts.ProjectID, taskOpts.TaskID, taskOpts.Execution, o.ID())
var logTypePrefix string
switch logType {
case TaskLogTypeAgent:
logTypePrefix = "agent"
case TaskLogTypeSystem:
logTypePrefix = "system"
case TaskLogTypeTask:
logTypePrefix = "task"
default:
return prefix
}
return fmt.Sprintf("%s/%s", prefix, logTypePrefix)
}
func (o TaskLogOutput) getLogService(ctx context.Context) (log.LogService, error) {
b, err := newBucket(ctx, o.BucketConfig, o.AWSCredentials)
if err != nil {
return nil, err
}
return log.NewLogServiceV0(b), nil
}
// getBuildloggerLogs makes request to Cedar Buildlogger for logs.
func (o TaskLogOutput) getBuildloggerLogs(ctx context.Context, taskOpts TaskOptions, getOpts TaskLogGetOptions) (log.LogIterator, error) {
opts := apimodels.GetBuildloggerLogsOptions{
BaseURL: evergreen.GetEnvironment().Settings().Cedar.BaseURL,
TaskID: taskOpts.TaskID,
Execution: utility.ToIntPtr(taskOpts.Execution),
Start: utility.FromInt64Ptr(getOpts.Start),
End: utility.FromInt64Ptr(getOpts.End),
Limit: getOpts.LineLimit,
Tail: getOpts.TailN,
}
if getOpts.LogType == TaskLogTypeAll {
opts.Tags = []string{
string(TaskLogTypeAgent),
string(TaskLogTypeSystem),
string(TaskLogTypeTask),
}
} else {
opts.Tags = []string{string(getOpts.LogType)}
}
return apimodels.GetBuildloggerLogs(ctx, opts)
}