/
subscription.go
277 lines (239 loc) · 8.67 KB
/
subscription.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
package services
import (
"context"
"fmt"
"math/big"
"time"
"github.com/smartcontractkit/chainlink/core/logger"
"github.com/smartcontractkit/chainlink/core/services/eth"
strpkg "github.com/smartcontractkit/chainlink/core/store"
"github.com/smartcontractkit/chainlink/core/store/models"
"github.com/smartcontractkit/chainlink/core/store/orm"
"github.com/smartcontractkit/chainlink/core/store/presenters"
"github.com/smartcontractkit/chainlink/core/utils"
ethereum "github.com/ethereum/go-ethereum"
"github.com/pkg/errors"
"go.uber.org/multierr"
)
// Unsubscriber is the interface for all subscriptions, allowing one to unsubscribe.
type Unsubscriber interface {
Unsubscribe()
}
// JobSubscription listens to event logs being pushed from the Ethereum Node to a job.
type JobSubscription struct {
Job models.JobSpec
unsubscribers []Unsubscriber
}
// StartJobSubscription constructs a JobSubscription which listens for and
// tracks event logs corresponding to the specified job. Ignores any errors if
// there is at least one successful subscription to an initiator log.
func StartJobSubscription(job models.JobSpec, head *models.Head, store *strpkg.Store, runManager RunManager) (JobSubscription, error) {
var merr error
var unsubscribers []Unsubscriber
initrs := job.InitiatorsFor(models.LogBasedChainlinkJobInitiators...)
nextHead := head.NextInt() // Exclude current block from subscription
if replayFromBlock := store.Config.ReplayFromBlock(); replayFromBlock >= 0 {
if replayFromBlock >= nextHead.Int64() {
logger.Infof("StartJobSubscription: Next head was supposed to be %v but ReplayFromBlock flag manually overrides to %v, will subscribe from blocknum %v", nextHead, replayFromBlock, replayFromBlock)
replayFromBlockBN := big.NewInt(replayFromBlock)
nextHead = replayFromBlockBN
}
logger.Warnf("StartJobSubscription: ReplayFromBlock was set to %v which is older than the next head of %v, will subscribe from blocknum %v", replayFromBlock, nextHead, nextHead)
}
for _, initr := range initrs {
unsubscriber, err := NewInitiatorSubscription(initr, store.EthClient, runManager, nextHead, store.Config, ReceiveLogRequest)
if err == nil {
unsubscribers = append(unsubscribers, unsubscriber)
} else {
merr = multierr.Append(merr, err)
}
}
if len(unsubscribers) == 0 {
return JobSubscription{}, multierr.Append(
merr, errors.New(
"unable to subscribe to any logs, check earlier errors in this message, and the initiator types"))
}
return JobSubscription{Job: job, unsubscribers: unsubscribers}, merr
}
// Unsubscribe stops the subscription and cleans up associated resources.
func (js JobSubscription) Unsubscribe() {
for _, sub := range js.unsubscribers {
sub.Unsubscribe()
}
}
// InitiatorSubscription encapsulates all functionality needed to wrap an ethereum subscription
// for use with a Chainlink Initiator. Initiator specific functionality is delegated
// to the callback.
type InitiatorSubscription struct {
*ManagedSubscription
runManager RunManager
Initiator models.Initiator
callback func(RunManager, models.LogRequest)
}
// NewInitiatorSubscription creates a new InitiatorSubscription that feeds received
// logs to the callback func parameter.
func NewInitiatorSubscription(
initr models.Initiator,
client eth.Client,
runManager RunManager,
nextHead *big.Int,
config orm.ConfigReader,
callback func(RunManager, models.LogRequest),
) (InitiatorSubscription, error) {
filter, err := models.FilterQueryFactory(initr, nextHead, config.OperatorContractAddress())
if err != nil {
return InitiatorSubscription{}, errors.Wrap(err, "NewInitiatorSubscription#FilterQueryFactory")
}
sub := InitiatorSubscription{
runManager: runManager,
Initiator: initr,
callback: callback,
}
managedSub, err := NewManagedSubscription(client, filter, sub.dispatchLog)
if err != nil {
return sub, errors.Wrap(err, "NewInitiatorSubscription#NewManagedSubscription")
}
sub.ManagedSubscription = managedSub
loggerLogListening(initr, filter.FromBlock)
return sub, nil
}
func (sub InitiatorSubscription) dispatchLog(log models.Log) {
logger.Debugw(fmt.Sprintf("Log for %v initiator for job %s", sub.Initiator.Type, sub.Initiator.JobSpecID.String()),
"txHash", log.TxHash.Hex(), "logIndex", log.Index, "blockNumber", log.BlockNumber, "job", sub.Initiator.JobSpecID.String())
base := models.InitiatorLogEvent{
Initiator: sub.Initiator,
Log: log,
}
sub.callback(sub.runManager, base.LogRequest())
}
func loggerLogListening(initr models.Initiator, blockNumber *big.Int) {
msg := fmt.Sprintf("Listening for %v from block %v", initr.Type, presenters.FriendlyBigInt(blockNumber))
logger.Infow(msg, "address", utils.LogListeningAddress(initr.Address), "jobID", initr.JobSpecID.String())
}
// ReceiveLogRequest parses the log and runs the job it indicated by its
// GetJobSpecID method
func ReceiveLogRequest(runManager RunManager, le models.LogRequest) {
if !le.Validate() {
logger.Debugw("discarding INVALID EVENT LOG", "log", le.GetLog())
return
}
if le.GetLog().Removed {
logger.Debugw("Skipping run for removed log", "log", le.GetLog(), "jobId", le.GetJobSpecID().String())
return
}
le.ToDebug()
runJob(runManager, le)
}
func runJob(runManager RunManager, le models.LogRequest) {
jobSpecID := le.GetJobSpecID()
initiator := le.GetInitiator()
if err := le.ValidateRequester(); err != nil {
if _, e := runManager.CreateErrored(jobSpecID, initiator, err); e != nil {
logger.Errorw(e.Error())
}
logger.Errorw(err.Error(), le.ForLogger()...)
return
}
rr, err := le.RunRequest()
if err != nil {
if _, e := runManager.CreateErrored(jobSpecID, initiator, err); e != nil {
logger.Errorw(e.Error())
}
logger.Errorw(err.Error(), le.ForLogger()...)
return
}
_, err = runManager.Create(jobSpecID, &initiator, le.BlockNumber(), &rr)
if err != nil {
logger.Errorw(err.Error(), le.ForLogger()...)
}
}
// ManagedSubscription encapsulates the connecting, backfilling, and clean up of an
// ethereum node subscription.
type ManagedSubscription struct {
logSubscriber eth.Client
logs chan models.Log
ethSubscription ethereum.Subscription
callback func(models.Log)
}
// NewManagedSubscription subscribes to the ethereum node with the passed filter
// and delegates incoming logs to callback.
func NewManagedSubscription(
logSubscriber eth.Client,
filter ethereum.FilterQuery,
callback func(models.Log),
) (*ManagedSubscription, error) {
ctx := context.Background()
logs := make(chan models.Log)
es, err := logSubscriber.SubscribeFilterLogs(ctx, filter, logs)
if err != nil {
return nil, err
}
sub := &ManagedSubscription{
logSubscriber: logSubscriber,
callback: callback,
logs: logs,
ethSubscription: es,
}
go sub.listenToLogs(filter)
return sub, nil
}
// Unsubscribe closes channels and cleans up resources.
func (sub ManagedSubscription) Unsubscribe() {
if sub.ethSubscription != nil {
timedUnsubscribe(sub.ethSubscription)
}
close(sub.logs)
}
// timedUnsubscribe attempts to unsubscribe but aborts abruptly after a time delay
// unblocking the application. This is an effort to mitigate the occasional
// indefinite block described here from go-ethereum:
// https://chainlink/pull/600#issuecomment-426320971
func timedUnsubscribe(unsubscriber Unsubscriber) {
unsubscribed := make(chan struct{})
go func() {
unsubscriber.Unsubscribe()
close(unsubscribed)
}()
select {
case <-unsubscribed:
case <-time.After(100 * time.Millisecond):
logger.Warnf("Subscription %T Unsubscribe timed out.", unsubscriber)
}
}
func (sub ManagedSubscription) listenToLogs(q ethereum.FilterQuery) {
backfilledSet := sub.backfillLogs(q)
for {
select {
case log, open := <-sub.logs:
if !open {
return
}
if _, present := backfilledSet[log.BlockHash.String()]; !present {
sub.callback(log)
}
case err, ok := <-sub.ethSubscription.Err():
if ok {
logger.Errorw(fmt.Sprintf("Error in log subscription: %s", err.Error()), "err", err)
}
}
}
}
// Manually retrieve old logs since SubscribeFilterLogs(ctx, filter, chLogs) only returns newly
// imported blocks: https://github.com/ethereum/go-ethereum/wiki/RPC-PUB-SUB#logs
// Therefore TxManager.FilterLogs does a one time retrieval of old logs.
func (sub ManagedSubscription) backfillLogs(q ethereum.FilterQuery) map[string]bool {
backfilledSet := map[string]bool{}
if q.FromBlock == nil {
return backfilledSet
}
logs, err := sub.logSubscriber.FilterLogs(context.TODO(), q)
if err != nil {
logger.Errorw("Unable to backfill logs", "err", err, "fromBlock", q.FromBlock.String(), "toBlock", q.ToBlock.String())
return backfilledSet
}
for _, log := range logs {
backfilledSet[log.BlockHash.String()] = true
sub.callback(log)
}
return backfilledSet
}