-
Notifications
You must be signed in to change notification settings - Fork 175
/
consumer.go
83 lines (65 loc) · 2.17 KB
/
consumer.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
package chunkconsumer
import (
"fmt"
"github.com/rs/zerolog"
"github.com/onflow/flow-go/engine/verification/fetcher"
"github.com/onflow/flow-go/module"
"github.com/onflow/flow-go/module/jobqueue"
"github.com/onflow/flow-go/storage"
)
const (
DefaultJobIndex = uint64(0)
DefaultChunkWorkers = uint64(5)
)
// ChunkConsumer consumes the jobs from the job queue, and pass it to the
// Worker for processing.
// It wraps the generic job consumer in order to be used as a ReadyDoneAware
// on startup
type ChunkConsumer struct {
consumer module.JobConsumer
chunkProcessor fetcher.AssignedChunkProcessor
metrics module.VerificationMetrics
}
func NewChunkConsumer(
log zerolog.Logger,
metrics module.VerificationMetrics,
processedIndex storage.ConsumerProgress, // to persist the processed index
chunksQueue storage.ChunksQueue, // to read jobs (chunks) from
chunkProcessor fetcher.AssignedChunkProcessor, // to process jobs (chunks)
maxProcessing uint64, // max number of jobs to be processed in parallel
) *ChunkConsumer {
worker := NewWorker(chunkProcessor)
chunkProcessor.WithChunkConsumerNotifier(worker)
jobs := &ChunkJobs{locators: chunksQueue}
lg := log.With().Str("module", "chunk_consumer").Logger()
consumer := jobqueue.NewConsumer(lg, jobs, processedIndex, worker, maxProcessing)
chunkConsumer := &ChunkConsumer{
consumer: consumer,
chunkProcessor: chunkProcessor,
metrics: metrics,
}
worker.consumer = chunkConsumer
return chunkConsumer
}
func (c *ChunkConsumer) NotifyJobIsDone(jobID module.JobID) {
processedIndex := c.consumer.NotifyJobIsDone(jobID)
c.metrics.OnChunkConsumerJobDone(processedIndex)
}
// Size returns number of in-memory chunk jobs that chunk consumer is processing.
func (c *ChunkConsumer) Size() uint {
return c.consumer.Size()
}
func (c ChunkConsumer) Check() {
c.consumer.Check()
}
func (c *ChunkConsumer) Ready() <-chan struct{} {
err := c.consumer.Start(DefaultJobIndex)
if err != nil {
panic(fmt.Errorf("could not start the chunk consumer for match engine: %w", err))
}
return c.chunkProcessor.Ready()
}
func (c *ChunkConsumer) Done() <-chan struct{} {
c.consumer.Stop()
return c.chunkProcessor.Done()
}