-
Notifications
You must be signed in to change notification settings - Fork 3
/
queue.go
289 lines (238 loc) · 6.5 KB
/
queue.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
package axe
import (
"context"
"fmt"
"math/rand"
"sync"
"time"
"gopkg.in/tomb.v2"
"github.com/256dpi/fire"
"github.com/256dpi/fire/coal"
"github.com/256dpi/fire/stick"
)
type board struct {
sync.Mutex
jobs map[coal.ID]*Model
}
// Blueprint describes a queueable job.
type Blueprint struct {
// The job to be enqueued.
Job Job
// The job delay. If specified, the job will not be dequeued until the
// specified duration has passed.
Delay time.Duration
// The job isolation. If specified, the job will only be enqueued if no job
// has been executed in the specified duration.
Isolation time.Duration
}
// Options defines queue options.
type Options struct {
// The store used to manage jobs.
Store *coal.Store
// The maximum amount of lag that should be applied to every dequeue attempt.
//
// By default, multiple workers compete with each other when getting jobs
// from the same queue. An artificial lag limits multiple simultaneous
// dequeue attempts and allows the worker with the smallest lag to dequeue
// the job and inform the other workers to limit parallel dequeue attempts.
//
// Default: 100ms.
MaxLag time.Duration
// The duration after which a job may be returned again from the board.
//
// It may take some time until the board is updated with the new state of
// the job after a dequeue. The block period prevents another worker from
// simultaneously trying to dequeue the job. If the initial worker failed to
// dequeue the job it will be available again after the defined period.
//
// Default: 10s.
BlockPeriod time.Duration
// The callback that is called with job errors.
Reporter func(error)
}
// Queue manages job queueing.
type Queue struct {
options Options
tasks map[string]*Task
boards map[string]*board
tomb tomb.Tomb
}
// NewQueue creates and returns a new queue.
func NewQueue(options Options) *Queue {
// set default max lag
if options.MaxLag == 0 {
options.MaxLag = 100 * time.Millisecond
}
// set default block period
if options.BlockPeriod == 0 {
options.BlockPeriod = 10 * time.Second
}
return &Queue{
options: options,
tasks: make(map[string]*Task),
}
}
// Add will add the specified task to the queue.
func (q *Queue) Add(task *Task) {
// safety check
if q.boards != nil {
panic("axe: unable to add task to running queue")
}
// prepare task
task.prepare()
// get name
name := GetMeta(task.Job).Name
// check existence
if q.tasks[name] != nil {
panic(fmt.Sprintf(`axe: task with name "%s" already exists`, name))
}
// save task
q.tasks[name] = task
}
// Enqueue will enqueue a job. If the context carries a transaction it must be
// associated with the store that is also used by the queue.
func (q *Queue) Enqueue(ctx context.Context, job Job, delay, isolation time.Duration) (bool, error) {
return Enqueue(ctx, q.options.Store, job, delay, isolation)
}
// Callback is a factory to create callbacks that can be used to enqueue jobs
// during request processing.
func (q *Queue) Callback(matcher fire.Matcher, cb func(ctx *fire.Context) Blueprint) *fire.Callback {
return fire.C("axe/Queue.Callback", 0, matcher, func(ctx *fire.Context) error {
// get blueprint
bp := cb(ctx)
// check transaction
ok, ts := coal.GetTransaction(ctx)
// check if transaction store is different
if ok && ts != q.options.Store {
// enqueue job outside of transaction
_, err := q.Enqueue(nil, bp.Job, bp.Delay, bp.Isolation)
if err != nil {
return err
}
} else {
// otherwise enqueue with potential transaction
_, err := q.Enqueue(ctx, bp.Job, bp.Delay, bp.Isolation)
if err != nil {
return err
}
}
return nil
})
}
// Action is a factory to create an action that can be used to enqueue jobs.
func (q *Queue) Action(methods []string, cb func(ctx *fire.Context) Blueprint) *fire.Action {
return fire.A("axe/Queue.Callback", methods, 0, func(ctx *fire.Context) error {
// get blueprint
bp := cb(ctx)
// check transaction
ok, ts := coal.GetTransaction(ctx)
// check if transaction store is different
if ok && ts != q.options.Store {
// enqueue job outside of transaction
_, err := q.Enqueue(nil, bp.Job, bp.Delay, bp.Isolation)
if err != nil {
return err
}
} else {
// otherwise enqueue with potential transaction
_, err := q.Enqueue(ctx, bp.Job, bp.Delay, bp.Isolation)
if err != nil {
return err
}
}
// respond with an empty object
err := ctx.Respond(stick.Map{})
if err != nil {
return err
}
return nil
})
}
// Run will start fetching jobs from the queue and execute them. It will return
// a channel that is closed once the queue has been synced and is available.
func (q *Queue) Run() chan struct{} {
// initialize boards
q.boards = make(map[string]*board)
// create boards
for _, task := range q.tasks {
name := GetMeta(task.Job).Name
q.boards[name] = &board{
jobs: make(map[coal.ID]*Model),
}
}
// prepare channel
synced := make(chan struct{})
// run process
q.tomb.Go(func() error {
return q.process(synced)
})
return synced
}
// Close will close the queue.
func (q *Queue) Close() {
// kill and wait
q.tomb.Kill(nil)
_ = q.tomb.Wait()
}
func (q *Queue) process(synced chan struct{}) error {
// start tasks
for _, task := range q.tasks {
task.start(q)
}
// reconcile jobs
var once sync.Once
stream := coal.Reconcile(q.options.Store, &Model{}, func() {
once.Do(func() {
close(synced)
})
}, func(model coal.Model) {
q.update(model.(*Model))
}, func(model coal.Model) {
q.update(model.(*Model))
}, nil, q.options.Reporter)
// await close
<-q.tomb.Dying()
// close stream
stream.Close()
return tomb.ErrDying
}
func (q *Queue) update(job *Model) {
// get board
board, ok := q.boards[job.Name]
if !ok {
return
}
// lock board
board.Lock()
defer board.Unlock()
// handle job
switch job.State {
case Enqueued, Dequeued, Failed:
// apply random lag
lag := time.Duration(rand.Int63n(int64(q.options.MaxLag)))
job.Available = job.Available.Add(lag)
// update job
board.jobs[job.ID()] = job
case Completed, Cancelled:
// remove job
delete(board.jobs, job.ID())
}
}
func (q *Queue) get(name string) (coal.ID, bool) {
// get board
board := q.boards[name]
// lock board
board.Lock()
defer board.Unlock()
// get time
now := time.Now()
// return first available job
for _, job := range board.jobs {
if job.Available.Before(now) {
// block job until the specified timeout has been reached
job.Available = job.Available.Add(q.options.BlockPeriod)
return job.ID(), true
}
}
return coal.ID{}, false
}