From 6e5807d009cd10964b3e767fb789c89749813912 Mon Sep 17 00:00:00 2001 From: Sam Davies Date: Tue, 12 Jul 2022 11:45:52 -0400 Subject: [PATCH] Exponential backoff retry on EthBroadcaster send transaction timeout --- core/chains/evm/txmgr/eth_broadcaster.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/core/chains/evm/txmgr/eth_broadcaster.go b/core/chains/evm/txmgr/eth_broadcaster.go index db0b8196154..a7689e257a1 100644 --- a/core/chains/evm/txmgr/eth_broadcaster.go +++ b/core/chains/evm/txmgr/eth_broadcaster.go @@ -10,6 +10,7 @@ import ( gethCommon "github.com/ethereum/go-ethereum/common" "github.com/jackc/pgconn" + "github.com/jpillora/backoff" "github.com/pkg/errors" "gopkg.in/guregu/null.v4" @@ -204,20 +205,37 @@ func (eb *EthBroadcaster) ethTxInsertTriggerer() { } } +func (eb *EthBroadcaster) newResendBackoff() backoff.Backoff { + return backoff.Backoff{ + Min: 1 * time.Second, + Max: 15 * time.Second, + Jitter: true, + } +} + func (eb *EthBroadcaster) monitorEthTxs(k ethkey.State, triggerCh chan struct{}) { ctx, cancel := utils.ContextFromChan(eb.chStop) defer cancel() + // errorRetryCh allows retry on exponential backoff in case of timeout or + // other unknown error + var errorRetryCh <-chan time.Time + bf := eb.newResendBackoff() + defer eb.wg.Done() for { pollDBTimer := time.NewTimer(utils.WithJitter(eb.config.TriggerFallbackDBPollInterval())) if err := eb.ProcessUnstartedEthTxs(ctx, k); err != nil { + errorRetryCh = time.After(bf.Duration()) if errors.Is(err, context.DeadlineExceeded) { eb.logger.Errorw("Timed out while handling eth_tx queue in ProcessUnstartedEthTxs", "err", err) } else { eb.logger.Criticalw("Unknown error occurred while handling eth_tx queue in ProcessUnstartedEthTxs. This chain/RPC client may not be supported", "err", err) } + } else { + bf = eb.newResendBackoff() + errorRetryCh = nil } select { @@ -236,6 +254,9 @@ func (eb *EthBroadcaster) monitorEthTxs(k ethkey.State, triggerCh chan struct{}) case <-pollDBTimer.C: // DB poller timed out continue + case <-errorRetryCh: + // Error backoff period reached + continue } } }