-
Notifications
You must be signed in to change notification settings - Fork 1.7k
/
listener_v2.go
1750 lines (1584 loc) · 60.7 KB
/
listener_v2.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
package v2
import (
"context"
"database/sql"
"fmt"
"math"
"math/big"
"strings"
"sync"
"time"
"github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/accounts/abi/bind"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/rpc"
"github.com/google/uuid"
"github.com/pkg/errors"
heaps "github.com/theodesp/go-heaps"
"github.com/theodesp/go-heaps/pairing"
"go.uber.org/multierr"
"golang.org/x/exp/slices"
txmgrcommon "github.com/smartcontractkit/chainlink/v2/common/txmgr"
txmgrtypes "github.com/smartcontractkit/chainlink/v2/common/txmgr/types"
"github.com/smartcontractkit/chainlink/v2/core/assets"
evmclient "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client"
httypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/headtracker/types"
"github.com/smartcontractkit/chainlink/v2/core/chains/evm/log"
"github.com/smartcontractkit/chainlink/v2/core/chains/evm/txmgr"
evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types"
"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/aggregator_v3_interface"
"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/batch_vrf_coordinator_v2"
"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/batch_vrf_coordinator_v2plus"
"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/vrf_coordinator_v2"
"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/vrf_coordinator_v2plus"
"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/vrf_owner"
"github.com/smartcontractkit/chainlink/v2/core/logger"
"github.com/smartcontractkit/chainlink/v2/core/services/job"
"github.com/smartcontractkit/chainlink/v2/core/services/keystore"
"github.com/smartcontractkit/chainlink/v2/core/services/pg"
"github.com/smartcontractkit/chainlink/v2/core/services/pipeline"
"github.com/smartcontractkit/chainlink/v2/core/services/vrf/vrfcommon"
"github.com/smartcontractkit/chainlink/v2/core/utils"
bigmath "github.com/smartcontractkit/chainlink/v2/core/utils/big_math"
)
var (
_ log.Listener = &listenerV2{}
_ job.ServiceCtx = &listenerV2{}
coordinatorV2ABI = evmtypes.MustGetABI(vrf_coordinator_v2.VRFCoordinatorV2ABI)
coordinatorV2PlusABI = evmtypes.MustGetABI(vrf_coordinator_v2plus.VRFCoordinatorV2PlusABI)
batchCoordinatorV2ABI = evmtypes.MustGetABI(batch_vrf_coordinator_v2.BatchVRFCoordinatorV2ABI)
batchCoordinatorV2PlusABI = evmtypes.MustGetABI(batch_vrf_coordinator_v2plus.BatchVRFCoordinatorV2PlusABI)
vrfOwnerABI = evmtypes.MustGetABI(vrf_owner.VRFOwnerMetaData.ABI)
)
const (
// GasAfterPaymentCalculation is the gas used after computing the payment
GasAfterPaymentCalculation = 21000 + // base cost of the transaction
100 + 5000 + // warm subscription balance read and update. See https://eips.ethereum.org/EIPS/eip-2929
2*2100 + 20000 - // cold read oracle address and oracle balance and first time oracle balance update, note first time will be 20k, but 5k subsequently
4800 + // request delete refund (refunds happen after execution), note pre-london fork was 15k. See https://eips.ethereum.org/EIPS/eip-3529
6685 // Positive static costs of argument encoding etc. note that it varies by +/- x*12 for every x bytes of non-zero data in the proof.
// BatchFulfillmentIterationGasCost is the cost of a single iteration of the batch coordinator's
// loop. This is used to determine the gas allowance for a batch fulfillment call.
BatchFulfillmentIterationGasCost = 52_000
// backoffFactor is the factor by which to increase the delay each time a request fails.
backoffFactor = 1.3
V2ReservedLinkQuery = `SELECT SUM(CAST(meta->>'MaxLink' AS NUMERIC(78, 0)))
FROM eth_txes
WHERE meta->>'MaxLink' IS NOT NULL
AND evm_chain_id = $1
AND CAST(meta->>'SubId' AS NUMERIC) = $2
AND state IN ('unconfirmed', 'unstarted', 'in_progress')
GROUP BY meta->>'SubId'`
V2PlusReservedLinkQuery = `SELECT SUM(CAST(meta->>'MaxLink' AS NUMERIC(78, 0)))
FROM eth_txes
WHERE meta->>'MaxLink' IS NOT NULL
AND evm_chain_id = $1
AND CAST(meta->>'GlobalSubId' AS NUMERIC) = $2
AND state IN ('unconfirmed', 'unstarted', 'in_progress')
GROUP BY meta->>'GlobalSubId'`
V2PlusReservedEthQuery = `SELECT SUM(CAST(meta->>'MaxEth' AS NUMERIC(78, 0)))
FROM eth_txes
WHERE meta->>'MaxEth' IS NOT NULL
AND evm_chain_id = $1
AND CAST(meta->>'GlobalSubId' AS NUMERIC) = $2
AND state IN ('unconfirmed', 'unstarted', 'in_progress')
GROUP BY meta->>'GlobalSubId'`
CouldNotDetermineIfLogConsumedMsg = "Could not determine if log was already consumed"
)
type errPossiblyInsufficientFunds struct{}
func (errPossiblyInsufficientFunds) Error() string {
return "Simulation errored, possibly insufficient funds. Request will remain unprocessed until funds are available"
}
type errBlockhashNotInStore struct{}
func (errBlockhashNotInStore) Error() string {
return "Blockhash not in store"
}
func New(
cfg vrfcommon.Config,
feeCfg vrfcommon.FeeConfig,
l logger.Logger,
ethClient evmclient.Client,
chainID *big.Int,
logBroadcaster log.Broadcaster,
q pg.Q,
coordinator CoordinatorV2_X,
batchCoordinator batch_vrf_coordinator_v2.BatchVRFCoordinatorV2Interface,
vrfOwner vrf_owner.VRFOwnerInterface,
aggregator *aggregator_v3_interface.AggregatorV3Interface,
txm txmgr.TxManager,
pipelineRunner pipeline.Runner,
gethks keystore.Eth,
job job.Job,
mailMon *utils.MailboxMonitor,
reqLogs *utils.Mailbox[log.Broadcast],
reqAdded func(),
respCount map[string]uint64,
headBroadcaster httypes.HeadBroadcasterRegistry,
deduper *vrfcommon.LogDeduper,
) job.ServiceCtx {
return &listenerV2{
cfg: cfg,
feeCfg: feeCfg,
l: logger.Sugared(l),
ethClient: ethClient,
chainID: chainID,
logBroadcaster: logBroadcaster,
txm: txm,
mailMon: mailMon,
coordinator: coordinator,
batchCoordinator: batchCoordinator,
vrfOwner: vrfOwner,
pipelineRunner: pipelineRunner,
job: job,
q: q,
gethks: gethks,
reqLogs: reqLogs,
chStop: make(chan struct{}),
reqAdded: reqAdded,
respCount: respCount,
blockNumberToReqID: pairing.New(),
headBroadcaster: headBroadcaster,
latestHeadMu: sync.RWMutex{},
wg: &sync.WaitGroup{},
aggregator: aggregator,
deduper: deduper,
}
}
type pendingRequest struct {
confirmedAtBlock uint64
req RandomWordsRequested
lb log.Broadcast
utcTimestamp time.Time
// used for exponential backoff when retrying
attempts int
lastTry time.Time
}
type vrfPipelineResult struct {
err error
// maxFee indicates how much juels (link) or wei (ether) would be paid for the VRF request
// if it were to be fulfilled at the maximum gas price (i.e gas lane gas price).
maxFee *big.Int
// fundsNeeded indicates a "minimum balance" in juels or wei that must be held in the
// subscription's account in order to fulfill the request.
fundsNeeded *big.Int
run pipeline.Run
payload string
gasLimit uint32
req pendingRequest
proof VRFProof
reqCommitment RequestCommitment
}
type listenerV2 struct {
utils.StartStopOnce
cfg vrfcommon.Config
feeCfg vrfcommon.FeeConfig
l logger.SugaredLogger
ethClient evmclient.Client
chainID *big.Int
logBroadcaster log.Broadcaster
txm txmgr.TxManager
mailMon *utils.MailboxMonitor
coordinator CoordinatorV2_X
batchCoordinator batch_vrf_coordinator_v2.BatchVRFCoordinatorV2Interface
vrfOwner vrf_owner.VRFOwnerInterface
pipelineRunner pipeline.Runner
job job.Job
q pg.Q
gethks keystore.Eth
reqLogs *utils.Mailbox[log.Broadcast]
chStop utils.StopChan
// We can keep these pending logs in memory because we
// only mark them confirmed once we send a corresponding fulfillment transaction.
// So on node restart in the middle of processing, the lb will resend them.
reqsMu sync.Mutex // Both the log listener and the request handler write to reqs
reqs []pendingRequest
reqAdded func() // A simple debug helper
// Data structures for reorg attack protection
// We want a map so we can do an O(1) count update every fulfillment log we get.
respCountMu sync.Mutex
respCount map[string]uint64
// This auxiliary heap is used when we need to purge the
// respCount map - we repeatedly want to remove the minimum log.
// You could use a sorted list if the completed logs arrive in order, but they may not.
blockNumberToReqID *pairing.PairHeap
// head tracking data structures
headBroadcaster httypes.HeadBroadcasterRegistry
latestHeadMu sync.RWMutex
latestHeadNumber uint64
// Wait group to wait on all goroutines to shut down.
wg *sync.WaitGroup
// aggregator client to get link/eth feed prices from chain. Can be nil for VRF V2 plus
aggregator aggregator_v3_interface.AggregatorV3InterfaceInterface
// deduper prevents processing duplicate requests from the log broadcaster.
deduper *vrfcommon.LogDeduper
}
// Start starts listenerV2.
func (lsn *listenerV2) Start(ctx context.Context) error {
return lsn.StartOnce("VRFListenerV2", func() error {
// Check gas limit configuration
confCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
conf, err := lsn.coordinator.GetConfig(&bind.CallOpts{Context: confCtx})
gasLimit := lsn.feeCfg.LimitDefault()
vrfLimit := lsn.feeCfg.LimitJobType().VRF()
if vrfLimit != nil {
gasLimit = *vrfLimit
}
if err != nil {
lsn.l.Criticalw("Error getting coordinator config for gas limit check, starting anyway.", "err", err)
} else if conf.MaxGasLimit()+(GasProofVerification*2) > uint32(gasLimit) {
lsn.l.Criticalw("Node gas limit setting may not be high enough to fulfill all requests; it should be increased. Starting anyway.",
"currentGasLimit", gasLimit,
"neededGasLimit", conf.MaxGasLimit()+(GasProofVerification*2),
"callbackGasLimit", conf.MaxGasLimit(),
"proofVerificationGas", GasProofVerification)
}
spec := job.LoadEnvConfigVarsVRF(lsn.cfg, *lsn.job.VRFSpec)
unsubscribeLogs := lsn.logBroadcaster.Register(lsn, log.ListenerOpts{
Contract: lsn.coordinator.Address(),
ParseLog: lsn.coordinator.ParseLog,
LogsWithTopics: lsn.coordinator.LogsWithTopics(spec.PublicKey.MustHash()),
// Specify a min incoming confirmations of 1 so that we can receive a request log
// right away. We set the real number of confirmations on a per-request basis in
// the getConfirmedAt method.
MinIncomingConfirmations: 1,
ReplayStartedCallback: lsn.ReplayStartedCallback,
})
latestHead, unsubscribeHeadBroadcaster := lsn.headBroadcaster.Subscribe(lsn)
if latestHead != nil {
lsn.setLatestHead(latestHead)
}
// Log listener gathers request logs
lsn.wg.Add(1)
go func() {
lsn.runLogListener([]func(){unsubscribeLogs, unsubscribeHeadBroadcaster}, spec.MinIncomingConfirmations, lsn.wg)
}()
// Request handler periodically computes a set of logs which can be fulfilled.
lsn.wg.Add(1)
go func() {
lsn.runRequestHandler(spec.PollPeriod, lsn.wg)
}()
lsn.mailMon.Monitor(lsn.reqLogs, "VRFListenerV2", "RequestLogs", fmt.Sprint(lsn.job.ID))
return nil
})
}
func (lsn *listenerV2) setLatestHead(head *evmtypes.Head) {
lsn.latestHeadMu.Lock()
defer lsn.latestHeadMu.Unlock()
num := uint64(head.Number)
if num > lsn.latestHeadNumber {
lsn.latestHeadNumber = num
}
}
// OnNewLongestChain is called by the head broadcaster when a new head is available.
func (lsn *listenerV2) OnNewLongestChain(ctx context.Context, head *evmtypes.Head) {
lsn.setLatestHead(head)
}
func (lsn *listenerV2) getLatestHead() uint64 {
lsn.latestHeadMu.RLock()
defer lsn.latestHeadMu.RUnlock()
return uint64(lsn.latestHeadNumber)
}
// Returns all the confirmed logs from
// the pending queue by subscription
func (lsn *listenerV2) getAndRemoveConfirmedLogsBySub(latestHead uint64) map[string][]pendingRequest {
lsn.reqsMu.Lock()
defer lsn.reqsMu.Unlock()
vrfcommon.UpdateQueueSize(lsn.job.Name.ValueOrZero(), lsn.job.ExternalJobID, lsn.coordinator.Version(), uniqueReqs(lsn.reqs))
var toProcess = make(map[string][]pendingRequest)
var toKeep []pendingRequest
for i := 0; i < len(lsn.reqs); i++ {
if r := lsn.reqs[i]; lsn.ready(r, latestHead) {
toProcess[r.req.SubID().String()] = append(toProcess[r.req.SubID().String()], r)
} else {
toKeep = append(toKeep, lsn.reqs[i])
}
}
lsn.reqs = toKeep
return toProcess
}
func (lsn *listenerV2) ready(req pendingRequest, latestHead uint64) bool {
// Request is not eligible for fulfillment yet
if req.confirmedAtBlock > latestHead {
return false
}
if lsn.job.VRFSpec.BackoffInitialDelay == 0 || req.attempts == 0 {
// Backoff is disabled, or this is the first try
return true
}
return time.Now().UTC().After(
nextTry(
req.attempts,
lsn.job.VRFSpec.BackoffInitialDelay,
lsn.job.VRFSpec.BackoffMaxDelay,
req.lastTry))
}
func nextTry(retries int, initial, max time.Duration, last time.Time) time.Time {
expBackoffFactor := math.Pow(backoffFactor, float64(retries-1))
var delay time.Duration
if expBackoffFactor > float64(max/initial) {
delay = max
} else {
delay = time.Duration(float64(initial) * expBackoffFactor)
}
return last.Add(delay)
}
// Remove all entries 10000 blocks or older
// to avoid a memory leak.
func (lsn *listenerV2) pruneConfirmedRequestCounts() {
lsn.respCountMu.Lock()
defer lsn.respCountMu.Unlock()
min := lsn.blockNumberToReqID.FindMin()
for min != nil {
m := min.(fulfilledReqV2)
if m.blockNumber > (lsn.getLatestHead() - 10000) {
break
}
delete(lsn.respCount, m.reqID)
lsn.blockNumberToReqID.DeleteMin()
min = lsn.blockNumberToReqID.FindMin()
}
}
// Determine a set of logs that are confirmed
// and the subscription has sufficient balance to fulfill,
// given a eth call with the max gas price.
// Note we have to consider the pending reqs already in the txm as already "spent" link or native,
// using a max link or max native consumed in their metadata.
// A user will need a minBalance capable of fulfilling a single req at the max gas price or nothing will happen.
// This is acceptable as users can choose different keyhashes which have different max gas prices.
// Other variables which can change the bill amount between our eth call simulation and tx execution:
// - Link/eth price fluctuation
// - Falling back to BHS
// However the likelihood is vanishingly small as
// 1) the window between simulation and tx execution is tiny.
// 2) the max gas price provides a very large buffer most of the time.
// Its easier to optimistically assume it will go though and in the rare case of a reversion
// we simply retry TODO: follow up where if we see a fulfillment revert, return log to the queue.
func (lsn *listenerV2) processPendingVRFRequests(ctx context.Context) {
confirmed := lsn.getAndRemoveConfirmedLogsBySub(lsn.getLatestHead())
processed := make(map[string]struct{})
start := time.Now()
// Add any unprocessed requests back to lsn.reqs after request processing is complete.
defer func() {
var toKeep []pendingRequest
for _, subReqs := range confirmed {
for _, req := range subReqs {
if _, ok := processed[req.req.RequestID().String()]; !ok {
req.attempts++
req.lastTry = time.Now().UTC()
toKeep = append(toKeep, req)
if lsn.job.VRFSpec.BackoffInitialDelay != 0 {
lsn.l.Infow("Request failed, next retry will be delayed.",
"reqID", req.req.RequestID().String(),
"subID", req.req.SubID(),
"attempts", req.attempts,
"lastTry", req.lastTry.String(),
"nextTry", nextTry(
req.attempts,
lsn.job.VRFSpec.BackoffInitialDelay,
lsn.job.VRFSpec.BackoffMaxDelay,
req.lastTry))
}
} else {
lsn.markLogAsConsumed(req.lb)
}
}
}
// There could be logs accumulated to this slice while request processor is running,
// so we merged the new ones with the ones that need to be requeued.
lsn.reqsMu.Lock()
lsn.reqs = append(lsn.reqs, toKeep...)
lsn.l.Infow("Finished processing pending requests",
"totalProcessed", len(processed),
"totalFailed", len(toKeep),
"total", len(lsn.reqs),
"time", time.Since(start).String())
lsn.reqsMu.Unlock() // unlock here since len(lsn.reqs) is a read, to avoid a data race.
}()
if len(confirmed) == 0 {
lsn.l.Infow("No pending requests ready for processing")
return
}
for subID, reqs := range confirmed {
l := lsn.l.With("subID", subID, "startTime", time.Now(), "numReqsForSub", len(reqs))
// Get the balance of the subscription and also it's active status.
// The reason we need both is that we cannot determine if a subscription
// is active solely by it's balance, since an active subscription could legitimately
// have a zero balance.
var (
startLinkBalance *big.Int
startEthBalance *big.Int
subIsActive bool
)
sID, ok := new(big.Int).SetString(subID, 10)
if !ok {
l.Criticalw("Unable to convert %s to Int", subID)
continue
}
sub, err := lsn.coordinator.GetSubscription(&bind.CallOpts{
Context: ctx}, sID)
if err != nil {
if strings.Contains(err.Error(), "execution reverted") {
// "execution reverted" indicates that the subscription no longer exists.
// We can no longer just mark these as processed and continue,
// since it could be that the subscription was canceled while there
// were still unfulfilled requests.
// The simplest approach to handle this is to enter the processRequestsPerSub
// loop rather than create a bunch of largely duplicated code
// to handle this specific situation, since we need to run the pipeline to get
// the VRF proof, abi-encode it, etc.
l.Warnw("Subscription not found - setting start balance to zero", "subID", subID, "err", err)
startLinkBalance = big.NewInt(0)
} else {
// Most likely this is an RPC error, so we re-try later.
l.Errorw("Unable to read subscription balance", "err", err)
continue
}
} else {
// Happy path - sub is active.
startLinkBalance = sub.Balance()
if sub.Version() == vrfcommon.V2Plus {
startEthBalance = sub.EthBalance()
}
subIsActive = true
}
// Sort requests in ascending order by CallbackGasLimit
// so that we process the "cheapest" requests for each subscription
// first. This allows us to break out of the processing loop as early as possible
// in the event that a subscription is too underfunded to have it's
// requests processed.
slices.SortFunc(reqs, func(a, b pendingRequest) bool {
return a.req.CallbackGasLimit() < b.req.CallbackGasLimit()
})
p := lsn.processRequestsPerSub(ctx, sID, startLinkBalance, startEthBalance, reqs, subIsActive)
for reqID := range p {
processed[reqID] = struct{}{}
}
}
lsn.pruneConfirmedRequestCounts()
}
// MaybeSubtractReservedLink figures out how much LINK is reserved for other VRF requests that
// have not been fully confirmed yet on-chain, and subtracts that from the given startBalance,
// and returns that value if there are no errors.
func MaybeSubtractReservedLink(q pg.Q, startBalance *big.Int, chainID uint64, subID *big.Int, vrfVersion vrfcommon.Version) (*big.Int, error) {
var (
reservedLink string
query string
)
if vrfVersion == vrfcommon.V2Plus {
query = V2PlusReservedLinkQuery
} else if vrfVersion == vrfcommon.V2 {
query = V2ReservedLinkQuery
} else {
return nil, errors.Errorf("unsupported vrf version %s", vrfVersion)
}
err := q.Get(&reservedLink, query, chainID, subID.String())
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return nil, errors.Wrap(err, "getting reserved LINK")
}
if reservedLink != "" {
reservedLinkInt, success := big.NewInt(0).SetString(reservedLink, 10)
if !success {
return nil, fmt.Errorf("converting reserved LINK %s", reservedLink)
}
return new(big.Int).Sub(startBalance, reservedLinkInt), nil
}
return new(big.Int).Set(startBalance), nil
}
// MaybeSubtractReservedEth figures out how much ether is reserved for other VRF requests that
// have not been fully confirmed yet on-chain, and subtracts that from the given startBalance,
// and returns that value if there are no errors.
func MaybeSubtractReservedEth(q pg.Q, startBalance *big.Int, chainID uint64, subID *big.Int, vrfVersion vrfcommon.Version) (*big.Int, error) {
var (
reservedEther string
query string
)
if vrfVersion == vrfcommon.V2Plus {
query = V2PlusReservedEthQuery
} else if vrfVersion == vrfcommon.V2 {
// native payment is not supported for v2, so returning 0 reserved ETH
return big.NewInt(0), nil
} else {
return nil, errors.Errorf("unsupported vrf version %s", vrfVersion)
}
err := q.Get(&reservedEther, query, chainID, subID.String())
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return nil, errors.Wrap(err, "getting reserved ether")
}
if reservedEther != "" {
reservedEtherInt, success := big.NewInt(0).SetString(reservedEther, 10)
if !success {
return nil, fmt.Errorf("converting reserved ether %s", reservedEther)
}
return new(big.Int).Sub(startBalance, reservedEtherInt), nil
}
if startBalance != nil {
return new(big.Int).Set(startBalance), nil
}
return big.NewInt(0), nil
}
type fulfilledReqV2 struct {
blockNumber uint64
reqID string
}
func (a fulfilledReqV2) Compare(b heaps.Item) int {
a1 := a
a2 := b.(fulfilledReqV2)
switch {
case a1.blockNumber > a2.blockNumber:
return 1
case a1.blockNumber < a2.blockNumber:
return -1
default:
return 0
}
}
func (lsn *listenerV2) processRequestsPerSubBatchHelper(
ctx context.Context,
subID *big.Int,
startBalance *big.Int,
startBalanceNoReserved *big.Int,
reqs []pendingRequest,
subIsActive bool,
nativePayment bool,
) (processed map[string]struct{}) {
start := time.Now()
processed = make(map[string]struct{})
// Base the max gas for a batch on the max gas limit for a single callback.
// Since the max gas limit for a single callback is usually quite large already,
// we probably don't want to exceed it too much so that we can reliably get
// batch fulfillments included, while also making sure that the biggest gas guzzler
// callbacks are included.
config, err := lsn.coordinator.GetConfig(&bind.CallOpts{
Context: ctx,
})
if err != nil {
lsn.l.Errorw("Couldn't get config from coordinator", "err", err)
return processed
}
// Add very conservative upper bound estimate on verification costs.
batchMaxGas := uint32(config.MaxGasLimit() + 400_000)
l := lsn.l.With(
"subID", subID,
"eligibleSubReqs", len(reqs),
"startBalance", startBalance.String(),
"startBalanceNoReserved", startBalanceNoReserved.String(),
"batchMaxGas", batchMaxGas,
"subIsActive", subIsActive,
"nativePayment", nativePayment,
)
defer func() {
l.Infow("Finished processing for sub",
"endBalance", startBalanceNoReserved.String(),
"totalProcessed", len(processed),
"totalUnique", uniqueReqs(reqs),
"time", time.Since(start).String())
}()
l.Infow("Processing requests for subscription with batching")
// Check for already consumed or expired reqs
unconsumed, processedReqs := lsn.getUnconsumed(l, reqs)
for _, reqID := range processedReqs {
processed[reqID] = struct{}{}
}
// Process requests in chunks in order to kick off as many jobs
// as configured in parallel. Then we can combine into fulfillment
// batches afterwards.
for chunkStart := 0; chunkStart < len(unconsumed); chunkStart += int(lsn.job.VRFSpec.ChunkSize) {
chunkEnd := chunkStart + int(lsn.job.VRFSpec.ChunkSize)
if chunkEnd > len(unconsumed) {
chunkEnd = len(unconsumed)
}
chunk := unconsumed[chunkStart:chunkEnd]
var unfulfilled []pendingRequest
alreadyFulfilled, err := lsn.checkReqsFulfilled(ctx, l, chunk)
if errors.Is(err, context.Canceled) {
l.Infow("Context canceled, stopping request processing", "err", err)
return processed
} else if err != nil {
l.Errorw("Error checking for already fulfilled requests, proceeding anyway", "err", err)
}
for i, a := range alreadyFulfilled {
if a {
processed[chunk[i].req.RequestID().String()] = struct{}{}
} else {
unfulfilled = append(unfulfilled, chunk[i])
}
}
// All fromAddresses passed to the VRFv2 job have the same KeySpecific-MaxPrice value.
fromAddresses := lsn.fromAddresses()
maxGasPriceWei := lsn.feeCfg.PriceMaxKey(fromAddresses[0])
// Cases:
// 1. Never simulated: in this case, we want to observe the time until simulated
// on the utcTimestamp field of the pending request.
// 2. Simulated before: in this case, lastTry will be set to a non-zero time value,
// in which case we'd want to use that as a relative point from when we last tried
// the request.
observeRequestSimDuration(lsn.job.Name.ValueOrZero(), lsn.job.ExternalJobID, lsn.coordinator.Version(), unfulfilled)
pipelines := lsn.runPipelines(ctx, l, maxGasPriceWei, unfulfilled)
batches := newBatchFulfillments(batchMaxGas, lsn.coordinator.Version())
outOfBalance := false
for _, p := range pipelines {
ll := l.With("reqID", p.req.req.RequestID().String(),
"txHash", p.req.req.Raw().TxHash,
"maxGasPrice", maxGasPriceWei.String(),
"fundsNeeded", p.fundsNeeded.String(),
"maxFee", p.maxFee.String(),
"gasLimit", p.gasLimit,
"attempts", p.req.attempts,
"remainingBalance", startBalanceNoReserved.String(),
"consumerAddress", p.req.req.Sender(),
"blockNumber", p.req.req.Raw().BlockNumber,
"blockHash", p.req.req.Raw().BlockHash,
)
fromAddresses := lsn.fromAddresses()
fromAddress, err := lsn.gethks.GetRoundRobinAddress(lsn.chainID, fromAddresses...)
if err != nil {
l.Errorw("Couldn't get next from address", "err", err)
continue
}
ll = ll.With("fromAddress", fromAddress)
if p.err != nil {
if errors.Is(p.err, errBlockhashNotInStore{}) {
// Running the blockhash store feeder in backwards mode will be required to
// resolve this.
ll.Criticalw("Pipeline error", "err", p.err)
} else {
ll.Errorw("Pipeline error", "err", p.err)
if !subIsActive {
ll.Warnw("Force-fulfilling a request with insufficient funds on a cancelled sub")
etx, err := lsn.enqueueForceFulfillment(ctx, p, fromAddress)
if err != nil {
ll.Errorw("Error enqueuing force-fulfillment, re-queueing request", "err", err)
continue
}
ll.Infow("Successfully enqueued force-fulfillment", "ethTxID", etx.ID)
processed[p.req.req.RequestID().String()] = struct{}{}
// Need to put a continue here, otherwise the next if statement will be hit
// and we'd break out of the loop prematurely.
// If a sub is canceled, we want to force-fulfill ALL of it's pending requests
// before saying we're done with it.
continue
}
if startBalanceNoReserved.Cmp(p.fundsNeeded) < 0 && errors.Is(p.err, errPossiblyInsufficientFunds{}) {
ll.Infow("Insufficient balance to fulfill a request based on estimate, breaking", "err", p.err)
outOfBalance = true
// break out of this inner loop to process the currently constructed batch
break
}
// Ensure consumer is valid, otherwise drop the request.
if !lsn.isConsumerValidAfterFinalityDepthElapsed(ctx, p.req) {
lsn.l.Infow(
"Dropping request that was made by an invalid consumer.",
"consumerAddress", p.req.req.Sender(),
"reqID", p.req.req.RequestID(),
"blockNumber", p.req.req.Raw().BlockNumber,
"blockHash", p.req.req.Raw().BlockHash,
)
lsn.markLogAsConsumed(p.req.lb)
}
}
continue
}
if startBalanceNoReserved.Cmp(p.maxFee) < 0 {
// Insufficient funds, have to wait for a user top up.
// Break out of the loop now and process what we are able to process
// in the constructed batches.
ll.Infow("Insufficient balance to fulfill a request, breaking")
break
}
batches.addRun(p, fromAddress)
startBalanceNoReserved.Sub(startBalanceNoReserved, p.maxFee)
}
var processedRequestIDs []string
for _, batch := range batches.fulfillments {
l.Debugw("Processing batch", "batchSize", len(batch.proofs))
p := lsn.processBatch(l, subID, startBalanceNoReserved, batchMaxGas, batch, batch.fromAddress)
processedRequestIDs = append(processedRequestIDs, p...)
}
for _, reqID := range processedRequestIDs {
processed[reqID] = struct{}{}
}
// outOfBalance is set to true if the current sub we are processing
// has run out of funds to process any remaining requests. After enqueueing
// this constructed batch, we break out of this outer loop in order to
// avoid unnecessarily processing the remaining requests.
if outOfBalance {
break
}
}
return
}
func (lsn *listenerV2) processRequestsPerSubBatch(
ctx context.Context,
subID *big.Int,
startLinkBalance *big.Int,
startEthBalance *big.Int,
reqs []pendingRequest,
subIsActive bool,
) map[string]struct{} {
var processed = make(map[string]struct{})
startBalanceNoReserveLink, err := MaybeSubtractReservedLink(
lsn.q, startLinkBalance, lsn.chainID.Uint64(), subID, lsn.coordinator.Version())
if err != nil {
lsn.l.Errorw("Couldn't get reserved LINK for subscription", "sub", reqs[0].req.SubID(), "err", err)
return processed
}
startBalanceNoReserveEth, err := MaybeSubtractReservedEth(
lsn.q, startEthBalance, lsn.chainID.Uint64(), subID, lsn.coordinator.Version())
if err != nil {
lsn.l.Errorw("Couldn't get reserved ether for subscription", "sub", reqs[0].req.SubID(), "err", err)
return processed
}
// Split the requests into native and LINK requests.
var (
nativeRequests []pendingRequest
linkRequests []pendingRequest
)
for _, req := range reqs {
if req.req.NativePayment() {
nativeRequests = append(nativeRequests, req)
} else {
linkRequests = append(linkRequests, req)
}
}
// process the native and link requests in parallel
var wg sync.WaitGroup
var nativeProcessed, linkProcessed map[string]struct{}
wg.Add(2)
go func() {
defer wg.Done()
nativeProcessed = lsn.processRequestsPerSubBatchHelper(ctx, subID, startEthBalance, startBalanceNoReserveEth, nativeRequests, subIsActive, true)
}()
go func() {
defer wg.Done()
linkProcessed = lsn.processRequestsPerSubBatchHelper(ctx, subID, startLinkBalance, startBalanceNoReserveLink, linkRequests, subIsActive, false)
}()
wg.Wait()
// combine the processed link and native requests into the processed map
for k, v := range nativeProcessed {
processed[k] = v
}
for k, v := range linkProcessed {
processed[k] = v
}
return processed
}
// enqueueForceFulfillment enqueues a forced fulfillment through the
// VRFOwner contract. It estimates gas again on the transaction due
// to the extra steps taken within VRFOwner.fulfillRandomWords.
func (lsn *listenerV2) enqueueForceFulfillment(
ctx context.Context,
p vrfPipelineResult,
fromAddress common.Address,
) (etx txmgr.Tx, err error) {
if lsn.job.VRFSpec.VRFOwnerAddress == nil {
err = errors.New("vrf owner address not set in job spec, recreate job and provide it to force-fulfill")
return
}
if p.payload == "" {
// should probably never happen
// a critical log will be logged if this is the case in simulateFulfillment
err = errors.New("empty payload in vrfPipelineResult")
return
}
// fulfill the request through the VRF owner
err = lsn.q.Transaction(func(tx pg.Queryer) error {
if err = lsn.logBroadcaster.MarkConsumed(p.req.lb, pg.WithQueryer(tx)); err != nil {
return err
}
lsn.l.Infow("VRFOwner.fulfillRandomWords vs. VRFCoordinatorV2.fulfillRandomWords",
"vrf_owner.fulfillRandomWords", hexutil.Encode(vrfOwnerABI.Methods["fulfillRandomWords"].ID),
"vrf_coordinator_v2.fulfillRandomWords", hexutil.Encode(coordinatorV2ABI.Methods["fulfillRandomWords"].ID),
)
vrfOwnerAddress1 := lsn.vrfOwner.Address()
vrfOwnerAddressSpec := lsn.job.VRFSpec.VRFOwnerAddress.Address()
lsn.l.Infow("addresses diff", "wrapper_address", vrfOwnerAddress1, "spec_address", vrfOwnerAddressSpec)
lsn.l.Infow("fulfillRandomWords payload", "proof", p.proof, "commitment", p.reqCommitment.Get(), "payload", p.payload)
txData := hexutil.MustDecode(p.payload)
if err != nil {
return errors.Wrap(err, "abi pack VRFOwner.fulfillRandomWords")
}
estimateGasLimit, err := lsn.ethClient.EstimateGas(ctx, ethereum.CallMsg{
From: fromAddress,
To: &vrfOwnerAddressSpec,
Data: txData,
})
if err != nil {
return errors.Wrap(err, "failed to estimate gas on VRFOwner.fulfillRandomWords")
}
lsn.l.Infow("Estimated gas limit on force fulfillment",
"estimateGasLimit", estimateGasLimit, "pipelineGasLimit", p.gasLimit)
if estimateGasLimit < uint64(p.gasLimit) {
estimateGasLimit = uint64(p.gasLimit)
}
requestID := common.BytesToHash(p.req.req.RequestID().Bytes())
subID := p.req.req.SubID()
requestTxHash := p.req.req.Raw().TxHash
etx, err = lsn.txm.CreateTransaction(txmgr.TxRequest{
FromAddress: fromAddress,
ToAddress: lsn.vrfOwner.Address(),
EncodedPayload: txData,
FeeLimit: uint32(estimateGasLimit),
Strategy: txmgrcommon.NewSendEveryStrategy(),
Meta: &txmgr.TxMeta{
RequestID: &requestID,
SubID: ptr(subID.Uint64()),
RequestTxHash: &requestTxHash,
// No max link since simulation failed
},
}, pg.WithQueryer(tx), pg.WithParentCtx(ctx))
return err
})
return
}
// For an errored pipeline run, wait until the finality depth of the chain to have elapsed,
// then check if the failing request is being called by an invalid sender. Return false if this is the case,
// otherwise true.
func (lsn *listenerV2) isConsumerValidAfterFinalityDepthElapsed(ctx context.Context, req pendingRequest) bool {
latestHead := lsn.getLatestHead()
if latestHead-req.req.Raw().BlockNumber > uint64(lsn.cfg.FinalityDepth()) {
code, err := lsn.ethClient.CodeAt(ctx, req.req.Sender(), big.NewInt(int64(latestHead)))
if err != nil {
lsn.l.Warnw("Failed to fetch contract code", "err", err)
return true // error fetching code, give the benefit of doubt to the consumer
}
if len(code) == 0 {
return false // invalid consumer
}
}
return true // valid consumer, or finality depth has not elapsed
}
// processRequestsPerSubHelper processes a set of pending requests for the provided sub id.
// It returns a set of request IDs that were processed.
// Note that the provided startBalanceNoReserve is the balance of the subscription
// minus any pending requests that have already been processed and not yet fulfilled onchain.
func (lsn *listenerV2) processRequestsPerSubHelper(
ctx context.Context,
subID *big.Int,
startBalance *big.Int,
startBalanceNoReserved *big.Int,
reqs []pendingRequest,
subIsActive bool,
nativePayment bool,
) (processed map[string]struct{}) {
start := time.Now()
processed = make(map[string]struct{})
l := lsn.l.With(
"subID", subID,
"eligibleSubReqs", len(reqs),
"startBalance", startBalance.String(),
"startBalanceNoReserved", startBalanceNoReserved.String(),
"subIsActive", subIsActive,
"nativePayment", nativePayment,
)
defer func() {
l.Infow("Finished processing for sub",
"endBalance", startBalanceNoReserved.String(),
"totalProcessed", len(processed),
"totalUnique", uniqueReqs(reqs),
"time", time.Since(start).String())
}()
l.Infow("Processing requests for subscription")
// Check for already consumed or expired reqs
unconsumed, processedReqs := lsn.getUnconsumed(l, reqs)
for _, reqID := range processedReqs {
processed[reqID] = struct{}{}
}
// Process requests in chunks
for chunkStart := 0; chunkStart < len(unconsumed); chunkStart += int(lsn.job.VRFSpec.ChunkSize) {
chunkEnd := chunkStart + int(lsn.job.VRFSpec.ChunkSize)
if chunkEnd > len(unconsumed) {
chunkEnd = len(unconsumed)
}
chunk := unconsumed[chunkStart:chunkEnd]
var unfulfilled []pendingRequest