-
Notifications
You must be signed in to change notification settings - Fork 921
/
memberfetcher.go
301 lines (247 loc) · 7.03 KB
/
memberfetcher.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
package bot
import (
"errors"
"sync"
"time"
"github.com/jonas747/discordgo"
"github.com/jonas747/dstate"
"github.com/jonas747/yagpdb/bot/eventsystem"
"github.com/jonas747/yagpdb/common"
"github.com/karlseguin/ccache"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)
var (
MemberFetcher = &memberFetcher{
fetching: make(map[int64]*MemberFetchGuildQueue),
notFetching: make(map[int64]*MemberFetchGuildQueue),
}
failedUsersCache = ccache.New(ccache.Configure())
)
// GetMember will either return a member from state or fetch one from the member fetcher and then put it in state
func GetMember(guildID, userID int64) (*dstate.MemberState, error) {
gs := State.Guild(true, guildID)
if gs == nil {
return nil, ErrGuildNotFound
}
cop := gs.MemberCopy(true, userID)
if cop != nil && cop.MemberSet {
return cop, nil
}
result := <-MemberFetcher.RequestMember(guildID, userID, false)
return result.Member, result.Err
}
// GetMembers is the same as GetMember but with multiple members
func GetMembers(guildID int64, userIDs ...int64) ([]*dstate.MemberState, error) {
resultChan := make(chan *dstate.MemberState)
for _, v := range userIDs {
go func(id int64) {
m, _ := GetMember(guildID, id)
resultChan <- m
}(v)
}
result := make([]*dstate.MemberState, 0, len(userIDs))
for i := 0; i < len(userIDs); i++ {
m := <-resultChan
if m != nil {
result = append(result, m)
}
}
return result, nil
}
// GetMemberJoinedAt is the same as GetMember but it ensures the JoinedAt field is present
func GetMemberJoinedAt(guildID, userID int64) (*dstate.MemberState, error) {
gs := State.Guild(true, guildID)
if gs == nil {
return nil, ErrGuildNotFound
}
cop := gs.MemberCopy(true, userID)
if cop != nil && cop.MemberSet && !cop.JoinedAt.IsZero() {
return cop, nil
}
result := <-MemberFetcher.RequestMember(guildID, userID, true)
return result.Member, result.Err
}
// memberFetcher handles a per guild queue for fetching members
// This is probably overkill as the root cause for the flood of member requests (state being flushed upon guilds becoming unavailble) was fixed
// But it's here, for better or for worse
type memberFetcher struct {
sync.RWMutex
// Queue of guilds to user id's to fetch
fetching map[int64]*MemberFetchGuildQueue
notFetching map[int64]*MemberFetchGuildQueue
// Signal to run immediately
RunChan chan bool
Stop chan bool
}
type MemberFetchGuildQueue struct {
Queue []*MemberFetchRequest
}
type MemberFetchRequest struct {
Member int64
Guild int64
NeedJoinedAt bool
WaitingChannels []chan MemberFetchResult
}
func (req *MemberFetchRequest) sendResult(result MemberFetchResult) {
for _, ch := range req.WaitingChannels {
go func(channel chan MemberFetchResult) {
channel <- result
}(ch)
}
}
type MemberFetchResult struct {
Err error
Member *dstate.MemberState
}
func (m *memberFetcher) RequestMember(guildID, userID int64, requireJoinedAt bool) <-chan MemberFetchResult {
m.Lock()
var req *MemberFetchRequest
var q *MemberFetchGuildQueue
// Check to see if this guild is already in the queue
q, ok := m.notFetching[guildID]
if !ok {
q, ok = m.fetching[guildID]
}
if ok {
// The guild's queue already exist
for _, elem := range q.Queue {
if elem.Member == userID && (!requireJoinedAt || elem.NeedJoinedAt) {
// The member is already queued up
req = elem
break
}
}
}
// Request is nil, member was not already requests before
if req == nil {
req = &MemberFetchRequest{
Member: userID,
Guild: guildID,
NeedJoinedAt: requireJoinedAt,
}
if q == nil {
// Qeueu is nil, this guild does not currently have a queue, create one
q = &MemberFetchGuildQueue{
Queue: make([]*MemberFetchRequest, 0, 1),
}
m.notFetching[guildID] = q
}
q.Queue = append(q.Queue, req)
}
// Add the result channel to the request waiting channel
resultChan := make(chan MemberFetchResult)
req.WaitingChannels = append(req.WaitingChannels, resultChan)
m.Unlock()
return resultChan
}
func (m *memberFetcher) Run() {
ticker := time.NewTicker(time.Second)
for {
select {
case <-ticker.C:
m.check()
case <-m.RunChan:
m.check()
}
}
}
func (m *memberFetcher) check() {
m.Lock()
for k, v := range m.notFetching {
m.fetching[k] = v
delete(m.notFetching, k)
go m.runGuild(k)
}
m.Unlock()
}
var metricsMemberFetcherRequests = promauto.NewCounter(prometheus.CounterOpts{
Name: "yagpdb_memberfetcher_requests_total",
Help: "The total number of processed events",
})
func (m *memberFetcher) runGuild(guildID int64) {
for {
if !m.next(guildID) {
break
}
}
}
func (m *memberFetcher) next(guildID int64) (more bool) {
m.Lock()
if len(m.fetching[guildID].Queue) < 1 {
// Done processing this guild queue
delete(m.fetching, guildID)
m.Unlock()
return false
}
q := m.fetching[guildID]
elem := q.Queue[0]
m.Unlock()
logger.WithField("guild", guildID).WithField("user", elem.Member).Debug("Requesting guild member")
if gs := State.Guild(true, guildID); gs != nil {
if member := gs.MemberCopy(true, elem.Member); member != nil && member.MemberSet && (!elem.NeedJoinedAt || !member.JoinedAt.IsZero()) {
// Member is already in state, no need to request it
m.Lock()
result := MemberFetchResult{
Member: member,
}
elem.sendResult(result)
q.Queue = q.Queue[1:]
m.Unlock()
return true
}
}
var ms *dstate.MemberState
var err error
// Check if this was previously attempted and failed
failedCacheKey := discordgo.StrID(guildID) + ":" + discordgo.StrID(elem.Member)
if failedUsersCache.Get(failedCacheKey) == nil {
metricsMemberFetcherRequests.Inc()
var member *discordgo.Member
member, err = common.BotSession.GuildMember(guildID, elem.Member)
if err == nil && (member != nil && member.User == nil) {
err = errors.New("User is nil")
}
if err != nil {
logger.WithField("guild", guildID).WithField("user", elem.Member).WithError(err).Debug("Failed fetching member")
code, _ := common.DiscordError(err)
if code == discordgo.ErrCodeUnknownUser {
failedUsersCache.Set(failedCacheKey, 1, time.Hour)
}
} else {
member.GuildID = guildID
go eventsystem.EmitEvent(eventsystem.NewEventData(nil, eventsystem.EventMemberFetched, &discordgo.GuildMemberAdd{Member: member}), eventsystem.EventMemberFetched)
if member == nil {
panic("nil member")
}
if member.User == nil {
panic("nil user")
}
if gs := State.Guild(true, guildID); gs != nil {
gs.MemberAddUpdate(true, member)
ms = gs.MemberCopy(true, member.User.ID)
}
}
} else {
err = errors.New("Member is in failed fetching cache")
}
if ms == nil && err == nil {
err = errors.New("Member not found (left guild perhaps?)")
}
m.Lock()
result := MemberFetchResult{
Err: err,
Member: ms,
}
elem.sendResult(result)
q.Queue = q.Queue[1:]
m.Unlock()
return true
}
func (m *memberFetcher) Status() (fetching, notFetching int) {
m.RLock()
fetching = len(m.fetching)
notFetching = len(m.notFetching)
m.RUnlock()
return
}