-
Notifications
You must be signed in to change notification settings - Fork 264
/
retry_queue.go
125 lines (107 loc) · 2.79 KB
/
retry_queue.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
package server
import (
"errors"
"fmt"
"sync"
"time"
"go.uber.org/zap"
)
// RetryQueue 重试队列
type RetryQueue struct {
inFlightPQ inFlightPqueue
inFlightMessages map[string]*Message
inFlightMutex sync.Mutex
s *Server
fakeMessageID int64
}
// NewRetryQueue NewRetryQueue
func NewRetryQueue(s *Server) *RetryQueue {
return &RetryQueue{
inFlightPQ: newInFlightPqueue(1024),
inFlightMessages: make(map[string]*Message),
s: s,
fakeMessageID: 10000,
}
}
func (r *RetryQueue) startInFlightTimeout(msg *Message) {
now := time.Now()
msg.pri = now.Add(r.s.opts.MessageRetry.Interval).UnixNano()
r.pushInFlightMessage(msg)
r.addToInFlightPQ(msg)
r.s.monitor.RetryQueueMsgInc()
}
func (r *RetryQueue) addToInFlightPQ(msg *Message) {
r.inFlightMutex.Lock()
defer r.inFlightMutex.Unlock()
r.inFlightPQ.Push(msg)
}
func (r *RetryQueue) pushInFlightMessage(msg *Message) {
r.inFlightMutex.Lock()
defer r.inFlightMutex.Unlock()
key := fmt.Sprintf("%d_%d", msg.MessageID, msg.toClientID)
_, ok := r.inFlightMessages[key]
if ok {
return
}
r.inFlightMessages[key] = msg
}
func (r *RetryQueue) popInFlightMessage(clientID int64, messageID int64) (*Message, error) {
r.inFlightMutex.Lock()
defer r.inFlightMutex.Unlock()
key := fmt.Sprintf("%d_%d", messageID, clientID)
msg, ok := r.inFlightMessages[key]
if !ok {
return nil, errors.New("ID not in flight")
}
delete(r.inFlightMessages, key)
return msg, nil
}
func (r *RetryQueue) finishMessage(clientID int64, messageID int64) error {
msg, err := r.popInFlightMessage(clientID, messageID)
if err != nil {
return err
}
r.removeFromInFlightPQ(msg)
r.s.monitor.RetryQueueMsgDec()
return nil
}
func (r *RetryQueue) removeFromInFlightPQ(msg *Message) {
r.inFlightMutex.Lock()
if msg.index == -1 {
// this item has already been popped off the pqueue
r.inFlightMutex.Unlock()
return
}
r.inFlightPQ.Remove(msg.index)
r.inFlightMutex.Unlock()
}
func (r *RetryQueue) processInFlightQueue(t int64) {
for {
r.inFlightMutex.Lock()
msg, _ := r.inFlightPQ.PeekAndShift(t)
r.inFlightMutex.Unlock()
if msg == nil {
break
}
err := r.finishMessage(msg.toClientID, msg.MessageID)
if err != nil {
r.s.Error("processInFlightQueue-finishMessage失败", zap.Error(err), zap.Int64("toClientID", msg.toClientID), zap.Int64("messageID", msg.MessageID))
break
}
r.s.deliveryManager.startRetryDeliveryMsg(msg)
}
}
// Start 开始运行重试
func (r *RetryQueue) Start() {
r.s.Schedule(r.s.opts.MessageRetry.ScanInterval, func() {
now := time.Now().UnixNano()
r.processInFlightQueue(now)
})
r.s.Schedule(time.Minute*5, func() {
r.inFlightMutex.Lock()
defer r.inFlightMutex.Unlock()
r.s.monitor.InFlightMessagesSet(len(r.inFlightMessages))
})
}
func (r *RetryQueue) Stop() {
}