-
Notifications
You must be signed in to change notification settings - Fork 321
/
sink.go
159 lines (142 loc) · 3.26 KB
/
sink.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
package rule
import (
"sync/atomic"
"time"
"github.com/baidu/openedge/logger"
"github.com/baidu/openedge/openedge-hub/common"
"github.com/baidu/openedge/openedge-hub/router"
"github.com/baidu/openedge/openedge-hub/utils"
)
type sink struct {
id string
offset uint64
broker broker
msgchan *msgchan
trieq0 *router.Trie
trieq1 *router.Trie
tomb utils.Tomb
log logger.Logger
}
func newSink(id string, b broker, r *router.Trie, msgchan *msgchan) *sink {
s := &sink{
id: id,
broker: b,
trieq0: r,
trieq1: router.NewTrie(),
msgchan: msgchan,
log: logger.WithField("sink", id),
}
return s
}
func (s *sink) getOffset() uint64 {
return atomic.LoadUint64(&s.offset)
}
func (s *sink) setOffset(v uint64) {
atomic.StoreUint64(&s.offset, v)
}
// Register adds a subscription
func (s *sink) register(sub *sinksub) {
s.trieq0.Add(sub)
s.trieq1.Add(sub)
}
// Remove removes a subscription
func (s *sink) remove(id, topic string) {
s.trieq0.Remove(id, topic)
s.trieq1.Remove(id, topic)
}
// RemoveAll removes all subscriptions by id
func (s *sink) removeAll(id string) {
s.trieq0.RemoveAll(id)
}
func (s *sink) start() error {
if s.id == common.RuleMsgQ0 {
return s.tomb.Gos(s.goRoutingQ0)
}
offset, err := s.broker.InitOffset(s.id, s.msgchan.persist != nil)
if err != nil {
return err
}
s.setOffset(offset)
return s.tomb.Gos(s.goRoutingQ1)
}
func (s *sink) stop() {
s.log.Debugf("sink stopping")
s.trieq0.RemoveAll(s.id)
s.tomb.Kill()
}
func (s *sink) wait() {
err := s.tomb.Wait()
s.log.WithError(err).Debugf("sink stopped")
}
func (s *sink) goRoutingQ0() error {
s.log.Debugf("task of routing message (Q0) begins")
defer s.log.Debugf("task of routing message (Q0) stopped")
var msg *common.Message
for {
select {
case <-s.tomb.Dying():
return nil
case msg = <-s.broker.MsgQ0Chan():
matches := s.trieq0.MatchUnique(msg.Topic)
for _, sub := range matches {
sub.Flow(*msg)
}
}
}
}
func (s *sink) goRoutingQ1() error {
s.log.Debugf("task of routing message (Q1) begins with offset=%d", s.getOffset())
defer s.log.Debugf("task of routing message (Q1) stopped")
var (
err error
msg *common.Message
msgs []*common.Message
length int
)
ticker := time.NewTicker(time.Millisecond * 10)
maxBatchSize := s.broker.Config().Message.Egress.Qos1.Batch.Max
for {
if !s.tomb.Alive() {
return nil
}
msgs, err = s.broker.FetchQ1(s.getOffset(), maxBatchSize)
if err != nil {
s.log.WithError(err).Errorf("failed to fetch message")
select {
case <-s.tomb.Dying():
return nil
case <-time.After(time.Second):
continue
}
}
length = len(msgs)
if length == 0 {
select {
case <-s.tomb.Dying():
return nil
case <-ticker.C:
continue
}
}
s.log.Debugf("%d message(s) fetched", length)
if length != 1 {
for _, msg = range msgs[:length-1] {
matches := s.trieq1.MatchUnique(msg.Topic)
for _, sub := range matches {
sub.Flow(*msg)
}
}
}
msg = msgs[length-1]
matches := s.trieq1.MatchUnique(msg.Topic)
for _, sub := range matches {
sub.Flow(*msg)
}
if len(matches) == 0 {
// put barrier to make sure offset update in db even no message routed
msg.Barrier = true
s.msgchan.putQ0(msg)
}
s.setOffset(msg.SequenceID + 1)
}
}