-
-
Notifications
You must be signed in to change notification settings - Fork 0
/
consumer.go
105 lines (92 loc) · 2.87 KB
/
consumer.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
package gracesarama
import (
"context"
"fmt"
"github.com/Shopify/sarama"
"sync"
)
// NewConsumerGroupRunner returns a runner that controls the start-up and graceful shutdown of a consumer group.
func NewConsumerGroupRunner(
addrs []string, groupID string, config *sarama.Config,
topics []string,
handler sarama.ConsumerGroupHandler,
) *ConsumerGroupRunner {
config.Consumer.Return.Errors = true
return &ConsumerGroupRunner{
addrs: addrs,
groupID: groupID,
config: config,
consumerGroup: nil,
topics: topics,
handler: handler,
shutdownFinishedCh: nil,
}
}
// ConsumerGroupRunner is used to run and gracefully shutdown a consumer group.
type ConsumerGroupRunner struct {
addrs []string
groupID string
config *sarama.Config
consumerGroup sarama.ConsumerGroup
topics []string
handler sarama.ConsumerGroupHandler
shutdownFinishedCh chan struct{}
// ErrorHandlerFn handles any errors found while consuming.
// If it is nil errors are ignored.
ErrorHandlerFn func(err error)
// LogFn is used to log any debug/info messages from the runner.
// Leave as nil if you do not want any log messages.
LogFn func(format string, a ...interface{})
}
// Run starts up the consumer group.
func (cgr *ConsumerGroupRunner) Run(ctx context.Context) error {
group, err := sarama.NewConsumerGroup(cgr.addrs, cgr.groupID, cgr.config)
if err != nil {
return fmt.Errorf("could not create new consumer group: %w", err)
}
cgr.consumerGroup = group
go cgr.handleCtxDone(ctx)
wg := &sync.WaitGroup{}
wg.Add(1)
go cgr.handleErrors(ctx, wg)
consumeLoop:
for {
// `Consume` should be called inside an infinite loop, when a
// server-side rebalance happens, the consumer session will need to be
// recreated to get the new claims
if cgr.LogFn != nil {
cgr.LogFn("gracesarama: consumer group [%s]: starting to consume topics: %s", cgr.groupID, cgr.topics)
}
if err := cgr.consumerGroup.Consume(ctx, cgr.topics, cgr.handler); err != nil {
if err == sarama.ErrClosedConsumerGroup {
// Wait for consumerGroup.Close() to return before exiting.
if cgr.shutdownFinishedCh != nil {
<-cgr.shutdownFinishedCh
}
break consumeLoop
}
return err
}
if cgr.LogFn != nil {
cgr.LogFn("gracesarama: consumer group [%s]: stopped consuming topics: %s", cgr.groupID, cgr.topics)
}
}
wg.Wait()
return nil
}
func (cgr *ConsumerGroupRunner) handleCtxDone(ctx context.Context) {
<-ctx.Done()
if cgr.consumerGroup != nil {
cgr.shutdownFinishedCh = make(chan struct{})
cgr.consumerGroup.Close()
close(cgr.shutdownFinishedCh)
}
}
func (cgr *ConsumerGroupRunner) handleErrors(ctx context.Context, wg *sync.WaitGroup) {
defer wg.Done()
for err := range cgr.consumerGroup.Errors() {
if cgr.ErrorHandlerFn != nil {
cgr.ErrorHandlerFn(err)
}
}
}