-
Notifications
You must be signed in to change notification settings - Fork 175
/
request_handler.go
415 lines (364 loc) · 12.8 KB
/
request_handler.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
package synchronization
import (
"errors"
"fmt"
"github.com/rs/zerolog"
"github.com/onflow/flow-go/engine"
commonsync "github.com/onflow/flow-go/engine/common/synchronization"
"github.com/onflow/flow-go/model/flow"
"github.com/onflow/flow-go/model/messages"
"github.com/onflow/flow-go/module"
"github.com/onflow/flow-go/module/chainsync"
"github.com/onflow/flow-go/module/lifecycle"
"github.com/onflow/flow-go/module/metrics"
"github.com/onflow/flow-go/network"
"github.com/onflow/flow-go/network/channels"
"github.com/onflow/flow-go/state/cluster"
"github.com/onflow/flow-go/storage"
)
// defaultSyncRequestQueueCapacity maximum capacity of sync requests queue
const defaultSyncRequestQueueCapacity = 500
// defaultSyncRequestQueueCapacity maximum capacity of range requests queue
const defaultRangeRequestQueueCapacity = 500
// defaultSyncRequestQueueCapacity maximum capacity of batch requests queue
const defaultBatchRequestQueueCapacity = 500
// defaultEngineRequestsWorkers number of workers to dispatch events for requests
const defaultEngineRequestsWorkers = 8
type RequestHandlerEngine struct {
unit *engine.Unit
lm *lifecycle.LifecycleManager
me module.Local
log zerolog.Logger
metrics module.EngineMetrics
blocks storage.ClusterBlocks
core module.SyncCore
state cluster.State
con network.Conduit // used for sending responses to requesters
pendingSyncRequests engine.MessageStore // message store for *message.SyncRequest
pendingBatchRequests engine.MessageStore // message store for *message.BatchRequest
pendingRangeRequests engine.MessageStore // message store for *message.RangeRequest
requestMessageHandler *engine.MessageHandler // message handler responsible for request processing
}
func NewRequestHandlerEngine(
log zerolog.Logger,
metrics module.EngineMetrics,
con network.Conduit,
me module.Local,
blocks storage.ClusterBlocks,
core module.SyncCore,
state cluster.State,
) *RequestHandlerEngine {
r := &RequestHandlerEngine{
unit: engine.NewUnit(),
lm: lifecycle.NewLifecycleManager(),
me: me,
log: log.With().Str("engine", "cluster_synchronization").Logger(),
metrics: metrics,
blocks: blocks,
core: core,
state: state,
con: con,
}
r.setupRequestMessageHandler()
return r
}
// SubmitLocal submits an event originating on the local node.
func (r *RequestHandlerEngine) SubmitLocal(event interface{}) {
err := r.ProcessLocal(event)
if err != nil {
r.log.Fatal().Err(err).Msg("internal error processing event")
}
}
// Submit submits the given event from the node with the given origin ID
// for processing in a non-blocking manner. It returns instantly and logs
// a potential processing error internally when done.
func (r *RequestHandlerEngine) Submit(channel channels.Channel, originID flow.Identifier, event interface{}) {
err := r.Process(channel, originID, event)
if err != nil {
r.log.Fatal().Err(err).Msg("internal error processing event")
}
}
// ProcessLocal processes an event originating on the local node.
func (r *RequestHandlerEngine) ProcessLocal(event interface{}) error {
return r.process(r.me.NodeID(), event)
}
// Process processes the given event from the node with the given origin ID in
// a blocking manner. It returns the potential processing error when done.
func (r *RequestHandlerEngine) Process(channel channels.Channel, originID flow.Identifier, event interface{}) error {
err := r.process(originID, event)
if err != nil {
if engine.IsIncompatibleInputTypeError(err) {
r.log.Warn().Msgf("%v delivered unsupported message %T through %v", originID, event, channel)
return nil
}
return fmt.Errorf("unexpected error while processing engine message: %w", err)
}
return nil
}
// process processes events for the synchronization request handler engine.
// Error returns:
// - IncompatibleInputTypeError if input has unexpected type
// - All other errors are potential symptoms of internal state corruption or bugs (fatal).
func (r *RequestHandlerEngine) process(originID flow.Identifier, event interface{}) error {
return r.requestMessageHandler.Process(originID, event)
}
// setupRequestMessageHandler initializes the inbound queues and the MessageHandler for UNTRUSTED requests.
func (r *RequestHandlerEngine) setupRequestMessageHandler() {
// RequestHeap deduplicates requests by keeping only one sync request for each requester.
r.pendingSyncRequests = commonsync.NewRequestHeap(defaultSyncRequestQueueCapacity)
r.pendingRangeRequests = commonsync.NewRequestHeap(defaultRangeRequestQueueCapacity)
r.pendingBatchRequests = commonsync.NewRequestHeap(defaultBatchRequestQueueCapacity)
// define message queueing behaviour
r.requestMessageHandler = engine.NewMessageHandler(
r.log,
engine.NewNotifier(),
engine.Pattern{
Match: func(msg *engine.Message) bool {
_, ok := msg.Payload.(*messages.SyncRequest)
if ok {
r.metrics.MessageReceived(metrics.EngineClusterSynchronization, metrics.MessageSyncRequest)
}
return ok
},
Store: r.pendingSyncRequests,
},
engine.Pattern{
Match: func(msg *engine.Message) bool {
_, ok := msg.Payload.(*messages.RangeRequest)
if ok {
r.metrics.MessageReceived(metrics.EngineClusterSynchronization, metrics.MessageRangeRequest)
}
return ok
},
Store: r.pendingRangeRequests,
},
engine.Pattern{
Match: func(msg *engine.Message) bool {
_, ok := msg.Payload.(*messages.BatchRequest)
if ok {
r.metrics.MessageReceived(metrics.EngineClusterSynchronization, metrics.MessageBatchRequest)
}
return ok
},
Store: r.pendingBatchRequests,
},
)
}
// onSyncRequest processes an outgoing handshake; if we have a higher height, we
// inform the other node of it, so they can organize their block downloads. If
// we have a lower height, we add the difference to our own download queue.
func (r *RequestHandlerEngine) onSyncRequest(originID flow.Identifier, req *messages.SyncRequest) error {
final, err := r.state.Final().Head()
if err != nil {
return fmt.Errorf("could not get last finalized header: %w", err)
}
// queue any missing heights as needed
r.core.HandleHeight(final, req.Height)
// don't bother sending a response if we're within tolerance or if we're
// behind the requester
if r.core.WithinTolerance(final, req.Height) || req.Height > final.Height {
return nil
}
// if we're sufficiently ahead of the requester, send a response
res := &messages.SyncResponse{
Height: final.Height,
Nonce: req.Nonce,
}
err = r.con.Unicast(res, originID)
if err != nil {
r.log.Warn().Err(err).Msg("sending sync response failed")
return nil
}
r.metrics.MessageSent(metrics.EngineClusterSynchronization, metrics.MessageSyncResponse)
return nil
}
// onRangeRequest processes a request for a range of blocks by height.
func (r *RequestHandlerEngine) onRangeRequest(originID flow.Identifier, req *messages.RangeRequest) error {
r.log.Debug().Str("origin_id", originID.String()).Msg("received new range request")
// get the latest final state to know if we can fulfill the request
head, err := r.state.Final().Head()
if err != nil {
return fmt.Errorf("could not get last finalized header: %w", err)
}
// if we don't have anything to send, we can bail right away
if head.Height < req.FromHeight || req.FromHeight > req.ToHeight {
return nil
}
// enforce client-side max request size
var maxSize uint
// TODO: clean up this logic
if core, ok := r.core.(*chainsync.Core); ok {
maxSize = core.Config.MaxSize
} else {
maxSize = chainsync.DefaultConfig().MaxSize
}
maxHeight := req.FromHeight + uint64(maxSize)
if maxHeight < req.ToHeight {
r.log.Warn().
Uint64("from", req.FromHeight).
Uint64("to", req.ToHeight).
Uint64("size", (req.ToHeight-req.FromHeight)+1).
Uint("max_size", maxSize).
Msg("range request is too large")
req.ToHeight = maxHeight
}
// get all of the blocks, one by one
blocks := make([]messages.UntrustedClusterBlock, 0, req.ToHeight-req.FromHeight+1)
for height := req.FromHeight; height <= req.ToHeight; height++ {
block, err := r.blocks.ByHeight(height)
if errors.Is(err, storage.ErrNotFound) {
r.log.Error().Uint64("height", height).Msg("skipping unknown heights")
break
}
if err != nil {
return fmt.Errorf("could not get block for height (%d): %w", height, err)
}
blocks = append(blocks, messages.UntrustedClusterBlockFromInternal(block))
}
// if there are no blocks to send, skip network message
if len(blocks) == 0 {
r.log.Debug().Msg("skipping empty range response")
return nil
}
// send the response
res := &messages.ClusterBlockResponse{
Nonce: req.Nonce,
Blocks: blocks,
}
err = r.con.Unicast(res, originID)
if err != nil {
r.log.Warn().Err(err).Hex("origin_id", originID[:]).Msg("sending range response failed")
return nil
}
r.metrics.MessageSent(metrics.EngineClusterSynchronization, metrics.MessageBlockResponse)
return nil
}
// onBatchRequest processes a request for a specific block by block ID.
func (r *RequestHandlerEngine) onBatchRequest(originID flow.Identifier, req *messages.BatchRequest) error {
r.log.Debug().Str("origin_id", originID.String()).Msg("received new batch request")
// we should bail and send nothing on empty request
if len(req.BlockIDs) == 0 {
return nil
}
// TODO: clean up this logic
var maxSize uint
if core, ok := r.core.(*chainsync.Core); ok {
maxSize = core.Config.MaxSize
} else {
maxSize = chainsync.DefaultConfig().MaxSize
}
if len(req.BlockIDs) > int(maxSize) {
r.log.Warn().
Int("size", len(req.BlockIDs)).
Uint("max_size", maxSize).
Msg("batch request is too large")
}
// deduplicate the block IDs in the batch request
blockIDs := make(map[flow.Identifier]struct{})
for _, blockID := range req.BlockIDs {
blockIDs[blockID] = struct{}{}
// enforce client-side max request size
if len(blockIDs) == int(maxSize) {
break
}
}
// try to get all the blocks by ID
blocks := make([]messages.UntrustedClusterBlock, 0, len(blockIDs))
for blockID := range blockIDs {
block, err := r.blocks.ByID(blockID)
if errors.Is(err, storage.ErrNotFound) {
r.log.Debug().Hex("block_id", blockID[:]).Msg("skipping unknown block")
continue
}
if err != nil {
return fmt.Errorf("could not get block by ID (%s): %w", blockID, err)
}
blocks = append(blocks, messages.UntrustedClusterBlockFromInternal(block))
}
// if there are no blocks to send, skip network message
if len(blocks) == 0 {
r.log.Debug().Msg("skipping empty batch response")
return nil
}
// send the response
res := &messages.ClusterBlockResponse{
Nonce: req.Nonce,
Blocks: blocks,
}
err := r.con.Unicast(res, originID)
if err != nil {
r.log.Warn().Err(err).Hex("origin_id", originID[:]).Msg("sending batch response failed")
return nil
}
r.metrics.MessageSent(metrics.EngineClusterSynchronization, metrics.MessageBlockResponse)
return nil
}
// processAvailableRequests is processor of pending events which drives events from networking layer to business logic.
func (r *RequestHandlerEngine) processAvailableRequests() error {
for {
select {
case <-r.unit.Quit():
return nil
default:
}
msg, ok := r.pendingSyncRequests.Get()
if ok {
err := r.onSyncRequest(msg.OriginID, msg.Payload.(*messages.SyncRequest))
if err != nil {
return fmt.Errorf("processing sync request failed: %w", err)
}
continue
}
msg, ok = r.pendingRangeRequests.Get()
if ok {
err := r.onRangeRequest(msg.OriginID, msg.Payload.(*messages.RangeRequest))
if err != nil {
return fmt.Errorf("processing range request failed: %w", err)
}
continue
}
msg, ok = r.pendingBatchRequests.Get()
if ok {
err := r.onBatchRequest(msg.OriginID, msg.Payload.(*messages.BatchRequest))
if err != nil {
return fmt.Errorf("processing batch request failed: %w", err)
}
continue
}
// when there is no more messages in the queue, back to the loop to wait
// for the next incoming message to arrive.
return nil
}
}
// requestProcessingLoop is a separate goroutine that performs processing of queued requests
func (r *RequestHandlerEngine) requestProcessingLoop() {
notifier := r.requestMessageHandler.GetNotifier()
for {
select {
case <-r.unit.Quit():
return
case <-notifier:
err := r.processAvailableRequests()
if err != nil {
r.log.Fatal().Err(err).Msg("internal error processing queued requests")
}
}
}
}
// Ready returns a ready channel that is closed once the engine has fully started.
func (r *RequestHandlerEngine) Ready() <-chan struct{} {
r.lm.OnStart(func() {
for i := 0; i < defaultEngineRequestsWorkers; i++ {
r.unit.Launch(r.requestProcessingLoop)
}
})
return r.lm.Started()
}
// Done returns a done channel that is closed once the engine has fully stopped.
func (r *RequestHandlerEngine) Done() <-chan struct{} {
r.lm.OnStop(func() {
// wait for all request processing workers to exit
<-r.unit.Done()
})
return r.lm.Stopped()
}