-
Notifications
You must be signed in to change notification settings - Fork 132
/
batch.go
73 lines (67 loc) · 1.56 KB
/
batch.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
package ingest
import (
"context"
"sync"
"time"
log "github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/util/clock"
)
// Batcher batches up events from a channel. Batches are created whenever maxItems have been
// received or maxTimeout has elapsed since the last batch was created (whichever occurs first).
type Batcher[T any] struct {
input chan T
maxItems int
maxTimeout time.Duration
clock clock.Clock
callback func([]T)
buffer []T
mutex sync.Mutex
}
func NewBatcher[T any](input chan T, maxItems int, maxTimeout time.Duration, callback func([]T)) *Batcher[T] {
return &Batcher[T]{
input: input,
maxItems: maxItems,
maxTimeout: maxTimeout,
callback: callback,
clock: clock.RealClock{},
mutex: sync.Mutex{},
}
}
func (b *Batcher[T]) Run(ctx context.Context) {
for {
b.buffer = []T{}
expire := b.clock.After(b.maxTimeout)
for appendToBatch := true; appendToBatch; {
select {
case <-ctx.Done():
log.Info("Batcher: context is done")
// context is finished
return
case value, ok := <-b.input:
if !ok {
// input channel has closed
return
}
b.mutex.Lock()
b.buffer = append(b.buffer, value)
if len(b.buffer) == b.maxItems {
b.callback(b.buffer)
appendToBatch = false
}
b.mutex.Unlock()
case <-expire:
b.mutex.Lock()
if len(b.buffer) > 0 {
b.callback(b.buffer)
}
appendToBatch = false
b.mutex.Unlock()
}
}
}
}
func (b *Batcher[T]) BufferLen() int {
b.mutex.Lock()
defer b.mutex.Unlock()
return len(b.buffer)
}