forked from ava-labs/ortelius
-
Notifications
You must be signed in to change notification settings - Fork 0
/
consumer_db.go
159 lines (133 loc) · 4.78 KB
/
consumer_db.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
// (c) 2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.
package stream
import (
"context"
"fmt"
"github.com/corpetty/ortelius/services/db"
"github.com/corpetty/ortelius/services/metrics"
"github.com/corpetty/ortelius/cfg"
"github.com/corpetty/ortelius/services"
)
// consumer takes events from db and sends them to a service consumer
type consumerDB struct {
id string
eventType EventType
chainID string
consumer services.Consumer
sc *services.Control
// metrics
metricProcessedCountKey string
metricFailureCountKey string
metricProcessMillisCounterKey string
metricSuccessCountKey string
topicName string
}
type serviceConsumerFactory func(uint32, string, string) (services.Consumer, error)
// NewConsumerFactory returns a processorFactory for the given service consumer
func NewConsumerDBFactory(factory serviceConsumerFactory, eventType EventType) ProcessorFactoryChainDB {
return func(sc *services.Control, conf cfg.Config, chainVM string, chainID string) (ProcessorDB, error) {
c := &consumerDB{
eventType: eventType,
chainID: chainID,
sc: sc,
}
switch eventType {
case EventTypeDecisions:
c.metricProcessedCountKey = fmt.Sprintf("consume_records_processed_%s", chainID)
c.metricProcessMillisCounterKey = fmt.Sprintf("consume_records_process_millis_%s", chainID)
c.metricSuccessCountKey = fmt.Sprintf("consume_records_success_%s", chainID)
c.metricFailureCountKey = fmt.Sprintf("consume_records_failure_%s", chainID)
c.id = fmt.Sprintf("consumer %d %s %s", conf.NetworkID, chainVM, chainID)
case EventTypeConsensus:
c.metricProcessedCountKey = fmt.Sprintf("consume_consensus_records_processed_%s", chainID)
c.metricProcessMillisCounterKey = fmt.Sprintf("consume_consensus_records_process_millis_%s", chainID)
c.metricSuccessCountKey = fmt.Sprintf("consume_consensus_records_success_%s", chainID)
c.metricFailureCountKey = fmt.Sprintf("consume_consensus_records_failure_%s", chainID)
c.id = fmt.Sprintf("consumer_consensus %d %s %s", conf.NetworkID, chainVM, chainID)
}
metrics.Prometheus.CounterInit(c.metricProcessedCountKey, "records processed")
metrics.Prometheus.CounterInit(c.metricProcessMillisCounterKey, "records processed millis")
metrics.Prometheus.CounterInit(c.metricSuccessCountKey, "records success")
metrics.Prometheus.CounterInit(c.metricFailureCountKey, "records failure")
sc.InitConsumeMetrics()
var err error
c.consumer, err = factory(conf.NetworkID, chainVM, chainID)
if err != nil {
_ = c.Close()
return nil, err
}
c.topicName = GetTopicName(conf.NetworkID, chainID, c.eventType)
return c, nil
}
}
func (c *consumerDB) ID() string {
return c.id
}
func (c *consumerDB) Topic() []string {
return []string{c.topicName}
}
// Close closes the consumer
func (c *consumerDB) Close() error {
return nil
}
func (c *consumerDB) Process(conns *services.Connections, row *services.TxPool) error {
msg := &Message{
id: row.MsgKey,
chainID: c.chainID,
body: row.Serialization,
timestamp: row.CreatedAt.UTC().Unix(),
nanosecond: int64(row.CreatedAt.UTC().Nanosecond()),
}
return c.Consume(conns, msg)
}
func (c *consumerDB) Consume(conns *services.Connections, msg *Message) error {
collectors := metrics.NewCollectors(
metrics.NewCounterIncCollect(c.metricProcessedCountKey),
metrics.NewCounterObserveMillisCollect(c.metricProcessMillisCounterKey),
metrics.NewCounterIncCollect(services.MetricConsumeProcessedCountKey),
metrics.NewCounterObserveMillisCollect(services.MetricConsumeProcessMillisCounterKey),
)
defer func() {
err := collectors.Collect()
if err != nil {
c.sc.Log.Error("collectors.Collect: %s", err)
}
}()
var err error
for {
err = c.persistConsume(conns, msg)
if !db.ErrIsLockError(err) {
break
}
}
if err != nil {
c.Failure()
collectors.Error()
c.sc.Log.Error("consumer.Consume: %s", err)
return err
}
c.Success()
c.sc.BalanceAccumulatorManager.Run(c.sc)
return err
}
func (c *consumerDB) persistConsume(conns *services.Connections, msg *Message) error {
ctx, cancelFn := context.WithTimeout(context.Background(), cfg.DefaultConsumeProcessWriteTimeout)
defer cancelFn()
switch c.eventType {
case EventTypeDecisions:
return c.consumer.Consume(ctx, conns, msg, c.sc.Persist)
case EventTypeConsensus:
return c.consumer.ConsumeConsensus(ctx, conns, msg, c.sc.Persist)
default:
return fmt.Errorf("invalid eventType %v", c.eventType)
}
}
func (c *consumerDB) Failure() {
_ = metrics.Prometheus.CounterInc(c.metricFailureCountKey)
_ = metrics.Prometheus.CounterInc(services.MetricConsumeFailureCountKey)
}
func (c *consumerDB) Success() {
_ = metrics.Prometheus.CounterInc(c.metricSuccessCountKey)
_ = metrics.Prometheus.CounterInc(services.MetricConsumeSuccessCountKey)
}