-
Notifications
You must be signed in to change notification settings - Fork 166
/
consumer.go
74 lines (57 loc) · 1.78 KB
/
consumer.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
package chunkconsumer
import (
"fmt"
"github.com/rs/zerolog"
"github.com/onflow/flow-go/engine/verification/fetcher"
"github.com/onflow/flow-go/module"
"github.com/onflow/flow-go/module/jobqueue"
"github.com/onflow/flow-go/storage"
)
const (
DefaultJobIndex = uint64(0)
DefaultChunkWorkers = uint64(20)
)
// ChunkConsumer consumes the jobs from the job queue, and pass it to the
// Worker for processing.
// It wraps the generic job consumer in order to be used as a ReadyDoneAware
// on startup
type ChunkConsumer struct {
consumer module.JobConsumer
}
func NewChunkConsumer(
log zerolog.Logger,
processedIndex storage.ConsumerProgress, // to persist the processed index
chunksQueue storage.ChunksQueue, // to read jobs (chunks) from
engine fetcher.AssignedChunkProcessor, // to process jobs (chunks)
maxProcessing uint64, // max number of jobs to be processed in parallel
) *ChunkConsumer {
worker := NewWorker(engine)
engine.WithChunkConsumerNotifier(worker)
jobs := &ChunkJobs{locators: chunksQueue}
lg := log.With().Str("module", "chunk_consumer").Logger()
consumer := jobqueue.NewConsumer(lg, jobs, processedIndex, worker, maxProcessing)
chunkConsumer := &ChunkConsumer{consumer}
worker.consumer = chunkConsumer
return chunkConsumer
}
func (c *ChunkConsumer) NotifyJobIsDone(jobID module.JobID) {
c.consumer.NotifyJobIsDone(jobID)
}
func (c ChunkConsumer) Check() {
c.consumer.Check()
}
func (c *ChunkConsumer) Ready() <-chan struct{} {
err := c.consumer.Start(DefaultJobIndex)
if err != nil {
panic(fmt.Errorf("could not start the chunk consumer for match engine: %w", err))
}
ready := make(chan struct{})
close(ready)
return ready
}
func (c *ChunkConsumer) Done() <-chan struct{} {
c.consumer.Stop()
ready := make(chan struct{})
close(ready)
return ready
}