-
Notifications
You must be signed in to change notification settings - Fork 210
/
broker.go
352 lines (291 loc) · 9.24 KB
/
broker.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
package hare
import (
"errors"
"github.com/spacemeshos/go-spacemesh/log"
"github.com/spacemeshos/go-spacemesh/p2p/service"
"github.com/spacemeshos/go-spacemesh/priorityq"
"sync"
)
const inboxCapacity = 1024 // inbox size per instance
type startInstanceError error
type syncStateFunc func() bool
type validator interface {
Validate(m *Msg) bool
}
// Closer adds the ability to close objects.
type Closer struct {
channel chan struct{} // closeable go routines listen to this channel
}
// NewCloser creates a new (not closed) closer.
func NewCloser() Closer {
return Closer{make(chan struct{})}
}
// Close signals all listening instances to close.
// Note: should be called only once.
func (closer *Closer) Close() {
close(closer.channel)
}
// CloseChannel returns the channel to wait on for close signal.
func (closer *Closer) CloseChannel() chan struct{} {
return closer.channel
}
// Broker is the dispatcher of incoming Hare messages.
// The broker validates that the sender is eligible and active and forwards the message to the corresponding outbox.
type Broker struct {
Closer
log.Log
network NetworkService
eValidator validator // provides eligibility validation
stateQuerier StateQuerier // provides activeness check
isNodeSynced syncStateFunc // provider function to check if the node is currently synced
layersPerEpoch uint16
inbox chan service.GossipMessage
syncState map[instanceId]bool
outbox map[instanceId]chan *Msg
pending map[instanceId][]*Msg // the buffer of pending messages for the next layer
tasks chan func() // a channel to synchronize tasks (register/unregister) with incoming messages handling
latestLayer instanceId // the latest layer to attempt register (successfully or unsuccessfully)
isStarted bool
minDeleted instanceId
limit int // max number of consensus processes simultaneously
}
func newBroker(networkService NetworkService, eValidator validator, stateQuerier StateQuerier, syncState syncStateFunc, layersPerEpoch uint16, limit int, closer Closer, log log.Log) *Broker {
return &Broker{
Closer: closer,
Log: log,
network: networkService,
eValidator: eValidator,
stateQuerier: stateQuerier,
isNodeSynced: syncState,
layersPerEpoch: layersPerEpoch,
syncState: make(map[instanceId]bool),
outbox: make(map[instanceId]chan *Msg),
pending: make(map[instanceId][]*Msg),
tasks: make(chan func()),
latestLayer: 0,
minDeleted: 0,
limit: limit,
}
}
// Start listening to Hare messages (non-blocking).
func (b *Broker) Start() error {
if b.isStarted { // Start has been called at least twice
b.Error("Could not start instance")
return startInstanceError(errors.New("instance already started"))
}
b.isStarted = true
b.inbox = b.network.RegisterGossipProtocol(protoName, priorityq.Mid)
go b.eventLoop()
return nil
}
var (
errUnregistered = errors.New("layer is unregistered")
errNotSynced = errors.New("layer is not synced")
errFutureMsg = errors.New("future message")
errRegistration = errors.New("failed during registration")
errTooMany = errors.New("too many consensus process")
errInstanceNotSynced = errors.New("instance not synchronized")
)
// validate the message is contextually valid and that the target layer is synced.
// note: it is important to check synchronicity after contextual to avoid memory leak in syncState.
func (b *Broker) validate(m *Message) error {
msgInstId := m.InnerMsg.InstanceId
_, exist := b.outbox[msgInstId]
if !exist {
// prev layer, must be unregistered
if msgInstId < b.latestLayer {
return errUnregistered
}
// current layer
if msgInstId == b.latestLayer {
return errRegistration
}
// early msg
if msgInstId == b.latestLayer+1 {
return errEarlyMsg
}
// future msg
return errFutureMsg
}
// exist, check synchronicity
if !b.isSynced(msgInstId) {
return errNotSynced
}
// synced and has instance
return nil
}
// listens to incoming messages and incoming tasks
func (b *Broker) eventLoop() {
for {
select {
case msg := <-b.inbox:
if msg == nil {
b.With().Error("Broker message validation failed: called with nil",
log.Uint64("latest_layer", uint64(b.latestLayer)))
continue
}
hareMsg, err := MessageFromBuffer(msg.Bytes())
if err != nil {
b.Error("Could not build message err=%v", err)
continue
}
if hareMsg.InnerMsg == nil {
b.With().Error("Broker message validation failed",
log.Err(errNilInner), log.Uint64("latest_layer", uint64(b.latestLayer)))
continue
}
msgInstId := hareMsg.InnerMsg.InstanceId
// TODO: fix metrics
//metrics.MessageTypeCounter.With("type_id", hareMsg.InnerMsg.Type.String(), "layer", strconv.FormatUint(uint64(msgInstId), 10), "reporter", "brokerHandler").Add(1)
isEarly := false
if err := b.validate(hareMsg); err != nil {
if err != errEarlyMsg {
// not early, validation failed
b.With().Debug("Broker received a message to a CP that is not registered",
log.Err(err),
log.Uint64("msg_layer_id", uint64(msgInstId)),
log.Uint64("latest_layer", uint64(b.latestLayer)))
continue
}
b.With().Debug("early message detected",
log.Err(err),
log.Uint64("msg_layer_id", uint64(msgInstId)),
log.Uint64("latest_layer", uint64(b.latestLayer)))
isEarly = true
}
// the msg is either early or has instance
// create msg
iMsg, err := newMsg(hareMsg, b.stateQuerier, b.layersPerEpoch)
if err != nil {
b.Warning("Message validation failed: could not construct msg err=%v", err)
continue
}
// validate msg
if !b.eValidator.Validate(iMsg) {
b.Warning("Message validation failed: eValidator returned false %v", hareMsg)
continue
}
// validation passed, report
msg.ReportValidation(protoName)
if isEarly {
if _, exist := b.pending[msgInstId]; !exist { // create buffer if first msg
b.pending[msgInstId] = make([]*Msg, 0)
}
// we want to write all buffered messages to a chan with InboxCapacity len
// hence, we limit the buffer for pending messages
if len(b.pending[msgInstId]) == inboxCapacity {
b.Error("Reached %v pending messages. Ignoring message for layer %v sent from %v",
inboxCapacity, msgInstId, iMsg.PubKey.ShortString())
continue
}
b.pending[msgInstId] = append(b.pending[msgInstId], iMsg)
continue
}
// has instance, just send
out, exist := b.outbox[msgInstId]
if !exist {
b.Panic("broker should have had an instance for layer %v", msgInstId)
}
out <- iMsg
case task := <-b.tasks:
task()
case <-b.CloseChannel():
b.Warning("Broker exiting")
return
}
}
}
func (b *Broker) updateLatestLayer(id instanceId) {
if id <= b.latestLayer { // should expect to update only newer layers
b.Panic("Tried to update a previous layer expected %v > %v", id, b.latestLayer)
return
}
b.latestLayer = id
}
func (b *Broker) cleanOldLayers() {
for i := b.minDeleted + 1; i < b.latestLayer; i++ {
if _, exist := b.outbox[i]; !exist { // unregistered
delete(b.syncState, i) // clean sync state
b.minDeleted++
} else { // encountered first still running layer
break
}
}
}
func (b *Broker) updateSynchronicity(id instanceId) {
if _, ok := b.syncState[id]; ok { // already has result
return
}
// not exist means unknown, check & set
if !b.isNodeSynced() {
b.With().Info("Note: node is not synced. Marking layer as not synced", log.LayerId(uint64(id)))
b.syncState[id] = false // mark not synced
return
}
b.syncState[id] = true // mark valid
}
func (b *Broker) isSynced(id instanceId) bool {
b.updateSynchronicity(id)
synced, ok := b.syncState[id]
if !ok { // not exist means unknown
log.Panic("syncState doesn't contain a value after call to updateSynchronicity")
}
return synced
}
// Register a layer to receive messages
// Note: the registering instance is assumed to be started and accepting messages
func (b *Broker) Register(id instanceId) (chan *Msg, error) {
resErr := make(chan error, 1)
resCh := make(chan chan *Msg, 1)
regRequest := func() {
b.updateLatestLayer(id)
if len(b.outbox) >= b.limit {
resErr <- errTooMany
resCh <- nil
return
}
if b.isSynced(id) {
b.outbox[id] = make(chan *Msg, inboxCapacity)
pendingForInstance := b.pending[id]
if pendingForInstance != nil {
for _, mOut := range pendingForInstance {
b.outbox[id] <- mOut
}
delete(b.pending, id)
}
resErr <- nil
resCh <- b.outbox[id]
return
}
resErr <- errInstanceNotSynced
resCh <- nil
}
b.tasks <- regRequest // send synced task
// wait for result
err := <-resErr
result := <-resCh
if err != nil { // reg failed
return nil, err
}
return result, nil // reg ok
}
// Unregister a layer from receiving messages
func (b *Broker) Unregister(id instanceId) {
wg := sync.WaitGroup{}
wg.Add(1)
b.tasks <- func() {
delete(b.outbox, id) // delete matching outbox
b.cleanOldLayers()
b.Info("Unregistered layer %v ", id)
wg.Done()
}
wg.Wait()
}
// Synced returns true if the given layer is synced, false otherwise
func (b *Broker) Synced(id instanceId) bool {
res := make(chan bool)
b.tasks <- func() {
res <- b.isSynced(id)
}
return <-res
}