/
tx_mgr.go
549 lines (491 loc) · 19.9 KB
/
tx_mgr.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
// We use Optimism's `SimpleTransactionManager` below (as of commit: 02c570c4c05603e1261f664ee8f92502814bfca2).
package txmgr
import (
"context"
"errors"
"math/big"
"strings"
"sync"
"time"
"github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/log"
"github.com/specularL2/specular/services/sidecar/utils/fmt"
"github.com/specularL2/specular/services/sidecar/utils/retry"
)
const (
// Geth requires a minimum fee bump of 10% for tx resubmission
priceBump int64 = 10
// The multiplier applied to fee suggestions to put a hard limit on fee increases
feeLimitMultiplier = 5
)
// new = old * (100 + priceBump) / 100
var priceBumpPercent = big.NewInt(100 + priceBump)
var oneHundred = big.NewInt(100)
// ETHBackend is the set of methods that the transaction manager uses to resubmit gas & determine
// when transactions are included on L1.
type ETHBackend interface {
// BlockNumber returns the most recent block number.
BlockNumber(ctx context.Context) (uint64, error)
// TransactionReceipt queries the backend for a receipt associated with
// txHash. If lookup does not fail, but the transaction is not found,
// nil should be returned for both values.
TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error)
// SendTransaction submits a signed transaction to L1.
SendTransaction(ctx context.Context, tx *types.Transaction) error
// These functions are used to estimate what the basefee & priority fee should be set to.
// TODO(CLI-3318): Maybe need a generic interface to support different RPC providers
HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error)
SuggestGasTipCap(ctx context.Context) (*big.Int, error)
// NonceAt returns the account nonce of the given account.
// The block number can be nil, in which case the nonce is taken from the latest known block.
NonceAt(ctx context.Context, account common.Address, blockNumber *big.Int) (uint64, error)
// PendingNonceAt returns the pending nonce.
PendingNonceAt(ctx context.Context, account common.Address) (uint64, error)
// EstimateGas returns an estimate of the amount of gas needed to execute the given
// transaction against the current pending block.
EstimateGas(ctx context.Context, msg ethereum.CallMsg) (uint64, error)
// CallContract executes an eth_call against the provided contract.
CallContract(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) ([]byte, error)
}
// TxManager performs linear fee bumping of a tx until it confirms.
type TxManager struct {
cfg Config
backend ETHBackend
l log.Logger
signer SignerFn
nonce *uint64
nonceLock sync.RWMutex
}
type SignerFn func(ctx context.Context, address common.Address, tx *types.Transaction) (*types.Transaction, error)
// NewTxManager initializes a new TxManager with the passed Config.
func NewTxManager(l log.Logger, cfg Config, backend ETHBackend, signer SignerFn) *TxManager {
return &TxManager{
cfg: cfg,
backend: backend,
l: l,
signer: signer,
}
}
func (m *TxManager) From() common.Address {
return m.cfg.From
}
func (m *TxManager) BlockNumber(ctx context.Context) (uint64, error) {
return m.backend.BlockNumber(ctx)
}
// TxCandidate is a transaction candidate that can be submitted to ask the
// [TxManager] to construct a transaction with gas price bounds.
type TxCandidate struct {
// TxData is the transaction data to be used in the constructed tx.
TxData []byte
// To is the recipient of the constructed tx. Nil means contract creation.
To *common.Address
// GasLimit is the gas limit to be used in the constructed tx.
GasLimit uint64
// Value is the value to be used in the constructed tx.
Value *big.Int
}
// Send is used to publish a transaction with incrementally higher gas prices
// until the transaction eventually confirms. This method blocks until an
// invocation of sendTx returns (called with differing gas prices). The method
// may be canceled using the passed context.
//
// The transaction manager handles all signing. If and only if the gas limit is 0, the
// transaction manager will do a gas estimation.
//
// NOTE: Send can be called concurrently, the nonce will be managed internally.
func (m *TxManager) Send(ctx context.Context, candidate TxCandidate) (*types.Receipt, error) {
receipt, err := m.send(ctx, candidate)
if err != nil {
m.resetNonce()
}
return receipt, err
}
// Call is used to call a contract.
// Internally, it uses the [ethclient.Client.CallContract] method.
func (m *TxManager) Call(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) {
return m.backend.CallContract(ctx, msg, blockNumber)
}
// send performs the actual transaction creation and sending.
func (m *TxManager) send(ctx context.Context, candidate TxCandidate) (*types.Receipt, error) {
if m.cfg.TxSendTimeout != 0 {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, m.cfg.TxSendTimeout)
defer cancel()
}
tx, err := retry.Do(ctx, 10, retry.Fixed(2*time.Second), func() (*types.Transaction, error) {
tx, err := m.craftTx(ctx, candidate)
if err != nil {
m.l.Warn("Failed to create a transaction, will retry", "err", err)
}
return tx, err
})
if err != nil {
return nil, fmt.Errorf("failed to create the tx: %w", err)
}
return m.sendTx(ctx, tx)
}
// craftTx creates the signed transaction
// It queries L1 for the current fee market conditions as well as for the nonce.
// NOTE: This method SHOULD NOT publish the resulting transaction.
// NOTE: If the [TxCandidate.GasLimit] is non-zero, it will be used as the transaction's gas.
// NOTE: Otherwise, the [TxManager] will query the specified backend for an estimate.
func (m *TxManager) craftTx(ctx context.Context, candidate TxCandidate) (*types.Transaction, error) {
gasTipCap, basefee, err := m.suggestGasPriceCaps(ctx)
if err != nil {
return nil, fmt.Errorf("failed to get gas price info: %w", err)
}
gasFeeCap := calcGasFeeCap(basefee, gasTipCap)
rawTx := &types.DynamicFeeTx{
ChainID: m.cfg.ChainID,
To: candidate.To,
GasTipCap: gasTipCap,
GasFeeCap: gasFeeCap,
Data: candidate.TxData,
Value: candidate.Value,
}
m.l.Info("Creating tx", "to", rawTx.To, "from", m.cfg.From)
// If the gas limit is set, we can use that as the gas
if candidate.GasLimit != 0 {
rawTx.Gas = candidate.GasLimit
} else {
// Calculate the intrinsic gas for the transaction
gas, err := m.backend.EstimateGas(ctx, ethereum.CallMsg{
From: m.cfg.From,
To: candidate.To,
GasFeeCap: gasFeeCap,
GasTipCap: gasTipCap,
Data: rawTx.Data,
Value: rawTx.Value,
})
if err != nil {
return nil, fmt.Errorf("failed to estimate gas: %w", err)
}
rawTx.Gas = gas
}
// Avoid bumping the nonce if the gas estimation fails.
nonce, err := m.nextNonce(ctx)
if err != nil {
return nil, err
}
rawTx.Nonce = nonce
ctx, cancel := context.WithTimeout(ctx, m.cfg.NetworkTimeout)
defer cancel()
return m.signer(ctx, m.cfg.From, types.NewTx(rawTx))
}
// nextNonce returns a nonce to use for the next transaction. It uses
// eth_getTransactionCount with "latest" once, and then subsequent calls simply
// increment this number. If the transaction manager is reset, it will query the
// eth_getTransactionCount nonce again.
func (m *TxManager) nextNonce(ctx context.Context) (uint64, error) {
m.nonceLock.Lock()
defer m.nonceLock.Unlock()
if m.nonce == nil {
childCtx, cancel := context.WithTimeout(ctx, m.cfg.NetworkTimeout)
defer cancel()
nonce, err := m.backend.NonceAt(childCtx, m.cfg.From, nil)
if err != nil {
return 0, fmt.Errorf("failed to get nonce: %w", err)
}
m.nonce = &nonce
} else {
*m.nonce++
}
return *m.nonce, nil
}
// resetNonce resets the internal nonce tracking. This is called if any pending send
// returns an error.
func (m *TxManager) resetNonce() {
m.nonceLock.Lock()
defer m.nonceLock.Unlock()
m.nonce = nil
}
// send submits the same transaction several times with increasing gas prices as necessary.
// It waits for the transaction to be confirmed on chain.
func (m *TxManager) sendTx(ctx context.Context, tx *types.Transaction) (*types.Receipt, error) {
var wg sync.WaitGroup
defer wg.Wait()
ctx, cancel := context.WithCancel(ctx)
defer cancel()
sendState := NewSendState(m.cfg.SafeAbortNonceTooLowCount, m.cfg.TxNotInMempoolTimeout)
receiptChan := make(chan *types.Receipt, 1)
sendTxAsync := func(tx *types.Transaction) {
defer wg.Done()
m.publishAndWaitForTx(ctx, tx, sendState, receiptChan)
}
// Immediately publish a transaction before starting the resumbission loop
wg.Add(1)
go sendTxAsync(tx)
ticker := time.NewTicker(m.cfg.ResubmissionTimeout)
defer ticker.Stop()
for {
select {
case <-ticker.C:
// Don't resubmit a transaction if it has been mined, but we are waiting for the conf depth.
if sendState.IsWaitingForConfirmation() {
continue
}
// If we see lots of unrecoverable errors (and no pending transactions) abort sending the transaction.
if sendState.ShouldAbortImmediately() {
m.l.Warn("Aborting transaction submission")
return nil, errors.New("aborted transaction sending")
}
// Increase the gas price & submit the new transaction
newTx, err := m.increaseGasPrice(ctx, tx)
if err != nil || sendState.IsWaitingForConfirmation() {
// there is a chance the previous tx goes into "waiting for confirmation" state
// during the increaseGasPrice call. In some (but not all) cases increaseGasPrice
// will error out during gas estimation. In either case we should continue waiting
// rather than resubmit the tx.
continue
}
tx = newTx
wg.Add(1)
go sendTxAsync(tx)
case <-ctx.Done():
return nil, ctx.Err()
case receipt := <-receiptChan:
return receipt, nil
}
}
}
// publishAndWaitForTx publishes the transaction to the transaction pool and then waits for it with [waitMined].
// It should be called in a new go-routine. It will send the receipt to receiptChan in a non-blocking way if a receipt is found
// for the transaction.
func (m *TxManager) publishAndWaitForTx(ctx context.Context, tx *types.Transaction, sendState *SendState, receiptChan chan *types.Receipt) {
log := m.l.New("hash", tx.Hash(), "nonce", tx.Nonce(), "gasTipCap", tx.GasTipCap(), "gasFeeCap", tx.GasFeeCap())
log.Info("Publishing transaction")
cCtx, cancel := context.WithTimeout(ctx, m.cfg.NetworkTimeout)
defer cancel()
err := m.backend.SendTransaction(cCtx, tx)
sendState.ProcessSendError(err)
// Properly log & exit if there is an error
if err != nil {
switch {
case errStringMatch(err, core.ErrNonceTooLow):
log.Warn("nonce too low", "err", err)
case errStringMatch(err, context.Canceled):
log.Warn("transaction send cancelled", "err", err)
case errStringMatch(err, core.ErrKnownBlock):
log.Warn("resubmitted already known block", "err", err)
case errStringMatch(err, core.ErrFeeCapTooLow):
log.Warn("transaction fee cap too low", "err", err)
default:
log.Error("unable to publish transaction", "err", err)
}
return
}
log.Info("Transaction successfully published")
// Poll for the transaction to be ready & then send the result to receiptChan
receipt, err := m.waitMined(ctx, tx, sendState)
if err != nil {
// this will happen if the tx was successfully replaced by a tx with bumped fees
log.Info("Transaction receipt not found", "err", err)
return
}
select {
case receiptChan <- receipt:
return
default:
}
}
// waitMined waits for the transaction to be mined or for the context to be cancelled.
func (m *TxManager) waitMined(ctx context.Context, tx *types.Transaction, sendState *SendState) (*types.Receipt, error) {
txHash := tx.Hash()
queryTicker := time.NewTicker(m.cfg.ReceiptQueryInterval)
defer queryTicker.Stop()
for {
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-queryTicker.C:
if receipt := m.queryReceipt(ctx, txHash, sendState); receipt != nil {
return receipt, nil
}
}
}
}
// queryReceipt queries for the receipt and returns the receipt if it has passed the confirmation depth
func (m *TxManager) queryReceipt(ctx context.Context, txHash common.Hash, sendState *SendState) *types.Receipt {
ctx, cancel := context.WithTimeout(ctx, m.cfg.NetworkTimeout)
defer cancel()
receipt, err := m.backend.TransactionReceipt(ctx, txHash)
if errors.Is(err, ethereum.NotFound) {
sendState.TxNotMined(txHash)
m.l.Trace("Transaction not yet mined", "hash", txHash)
return nil
} else if err != nil {
m.l.Info("Receipt retrieval failed", "hash", txHash, "err", err)
return nil
} else if receipt == nil {
m.l.Warn("Receipt and error are both nil", "hash", txHash)
return nil
}
// Receipt is confirmed to be valid from this point on
sendState.TxMined(txHash)
txHeight := receipt.BlockNumber.Uint64()
tipHeight, err := m.backend.BlockNumber(ctx)
if err != nil {
m.l.Error("Unable to fetch block number", "err", err)
return nil
}
m.l.Debug("Transaction mined, checking confirmations", "hash", txHash, "txHeight", txHeight,
"tipHeight", tipHeight, "numConfirmations", m.cfg.NumConfirmations)
// The transaction is considered confirmed when
// txHeight+numConfirmations-1 <= tipHeight. Note that the -1 is
// needed to account for the fact that confirmations have an
// inherent off-by-one, i.e. when using 1 confirmation the
// transaction should be confirmed when txHeight is equal to
// tipHeight. The equation is rewritten in this form to avoid
// underflows.
if txHeight+m.cfg.NumConfirmations <= tipHeight+1 {
m.l.Info("Transaction confirmed", "hash", txHash)
return receipt
}
// Safe to subtract since we know the LHS above is greater.
confsRemaining := (txHeight + m.cfg.NumConfirmations) - (tipHeight + 1)
m.l.Debug("Transaction not yet confirmed", "hash", txHash, "confsRemaining", confsRemaining)
return nil
}
// increaseGasPrice takes the previous transaction, clones it, and returns it with fee values that
// are at least `priceBump` percent higher than the previous ones to satisfy Geth's replacement
// rules, and no lower than the values returned by the fee suggestion algorithm to ensure it
// doesn't linger in the mempool. Finally to avoid runaway price increases, fees are capped at a
// `feeLimitMultiplier` multiple of the suggested values.
func (m *TxManager) increaseGasPrice(ctx context.Context, tx *types.Transaction) (*types.Transaction, error) {
m.l.Info("bumping gas price for tx", "hash", tx.Hash(), "tip", tx.GasTipCap(), "fee", tx.GasFeeCap(), "gaslimit", tx.Gas())
tip, basefee, err := m.suggestGasPriceCaps(ctx)
if err != nil {
m.l.Warn("failed to get suggested gas tip and basefee", "err", err)
return nil, err
}
bumpedTip, bumpedFee := updateFees(tx.GasTipCap(), tx.GasFeeCap(), tip, basefee, m.l)
// Make sure increase is at most 5x the suggested values
maxTip := new(big.Int).Mul(tip, big.NewInt(feeLimitMultiplier))
if bumpedTip.Cmp(maxTip) > 0 {
m.l.Warn(fmt.Sprintf("bumped tip getting capped at %dx multiple of the suggested value", feeLimitMultiplier), "bumped", bumpedTip, "suggestion", tip)
bumpedTip.Set(maxTip)
}
maxFee := calcGasFeeCap(new(big.Int).Mul(basefee, big.NewInt(feeLimitMultiplier)), maxTip)
if bumpedFee.Cmp(maxFee) > 0 {
m.l.Warn("bumped fee getting capped at multiple of the implied suggested value", "bumped", bumpedFee, "suggestion", maxFee)
bumpedFee.Set(maxFee)
}
rawTx := &types.DynamicFeeTx{
ChainID: tx.ChainId(),
Nonce: tx.Nonce(),
GasTipCap: bumpedTip,
GasFeeCap: bumpedFee,
To: tx.To(),
Value: tx.Value(),
Data: tx.Data(),
AccessList: tx.AccessList(),
}
// Re-estimate gaslimit in case things have changed or a previous gaslimit estimate was wrong
gas, err := m.backend.EstimateGas(ctx, ethereum.CallMsg{
From: m.cfg.From,
To: rawTx.To,
GasFeeCap: bumpedTip,
GasTipCap: bumpedFee,
Data: rawTx.Data,
})
if err != nil {
// If this is a transaction resubmission, we sometimes see this outcome because the
// original tx can get included in a block just before the above call. In this case the
// error is due to the tx reverting with message "block number must be equal to next
// expected block number"
m.l.Warn("failed to re-estimate gas", "err", err, "gaslimit", tx.Gas())
return nil, err
}
if tx.Gas() != gas {
m.l.Info("re-estimated gas differs", "oldgas", tx.Gas(), "newgas", gas)
}
rawTx.Gas = gas
ctx, cancel := context.WithTimeout(ctx, m.cfg.NetworkTimeout)
defer cancel()
newTx, err := m.signer(ctx, m.cfg.From, types.NewTx(rawTx))
if err != nil {
m.l.Warn("failed to sign new transaction", "err", err)
return tx, nil
}
return newTx, nil
}
// suggestGasPriceCaps suggests what the new tip & new basefee should be based on the current L1 conditions
func (m *TxManager) suggestGasPriceCaps(ctx context.Context) (*big.Int, *big.Int, error) {
cCtx, cancel := context.WithTimeout(ctx, m.cfg.NetworkTimeout)
defer cancel()
tip, err := m.backend.SuggestGasTipCap(cCtx)
if err != nil {
return nil, nil, fmt.Errorf("failed to fetch the suggested gas tip cap: %w", err)
} else if tip == nil {
return nil, nil, errors.New("the suggested tip was nil")
}
cCtx, cancel = context.WithTimeout(ctx, m.cfg.NetworkTimeout)
defer cancel()
head, err := m.backend.HeaderByNumber(cCtx, nil)
if err != nil {
return nil, nil, fmt.Errorf("failed to fetch the suggested basefee: %w", err)
} else if head.BaseFee == nil {
return nil, nil, errors.New("txmgr does not support pre-london blocks that do not have a basefee")
}
return tip, head.BaseFee, nil
}
// calcThresholdValue returns x * priceBumpPercent / 100
func calcThresholdValue(x *big.Int) *big.Int {
threshold := new(big.Int).Mul(priceBumpPercent, x)
threshold = threshold.Div(threshold, oneHundred)
return threshold
}
// updateFees takes an old transaction's tip & fee cap plus a new tip & basefee, and returns
// a suggested tip and fee cap such that:
//
// (a) each satisfies geth's required tx-replacement fee bumps (we use a 10% increase), and
// (b) gasTipCap is no less than new tip, and
// (c) gasFeeCap is no less than calcGasFee(newBaseFee, newTip)
func updateFees(oldTip, oldFeeCap, newTip, newBaseFee *big.Int, lgr log.Logger) (*big.Int, *big.Int) {
newFeeCap := calcGasFeeCap(newBaseFee, newTip)
lgr = lgr.New("old_tip", oldTip, "old_feecap", oldFeeCap, "new_tip", newTip, "new_feecap", newFeeCap)
thresholdTip := calcThresholdValue(oldTip)
thresholdFeeCap := calcThresholdValue(oldFeeCap)
if newTip.Cmp(thresholdTip) >= 0 && newFeeCap.Cmp(thresholdFeeCap) >= 0 {
lgr.Debug("Using new tip and feecap")
return newTip, newFeeCap
} else if newTip.Cmp(thresholdTip) >= 0 && newFeeCap.Cmp(thresholdFeeCap) < 0 {
// Tip has gone up, but basefee is flat or down.
// TODO(CLI-3714): Do we need to recalculate the FC here?
lgr.Debug("Using new tip and threshold feecap")
return newTip, thresholdFeeCap
} else if newTip.Cmp(thresholdTip) < 0 && newFeeCap.Cmp(thresholdFeeCap) >= 0 {
// Basefee has gone up, but the tip hasn't. Recalculate the feecap because if the tip went up a lot
// not enough of the feecap may be dedicated to paying the basefee.
lgr.Debug("Using threshold tip and recalculated feecap")
return thresholdTip, calcGasFeeCap(newBaseFee, thresholdTip)
} else {
// TODO(CLI-3713): Should we skip the bump in this case?
lgr.Debug("Using threshold tip and threshold feecap")
return thresholdTip, thresholdFeeCap
}
}
// calcGasFeeCap deterministically computes the recommended gas fee cap given
// the base fee and gasTipCap. The resulting gasFeeCap is equal to:
//
// gasTipCap + 2*baseFee.
func calcGasFeeCap(baseFee, gasTipCap *big.Int) *big.Int {
return new(big.Int).Add(
gasTipCap,
new(big.Int).Mul(baseFee, big.NewInt(2)),
)
}
// errStringMatch returns true if err.Error() is a substring in target.Error() or if both are nil.
// It can accept nil errors without issue.
// Note: used where `errors.Is` doesn't work.
func errStringMatch(err, target error) bool {
if err == nil && target == nil {
return true
} else if err == nil || target == nil {
return false
}
return strings.Contains(err.Error(), target.Error())
}