app/expbackoff: implement exponential backoff package (#982)

Implements an exponential backoff package based on `google.golang.org/grpc@v1.48.0/backoff`. This is to be used in many places where we have `// TODO(corver): improve backoff`

category: feature
ticket: none

app/expbackoff/expbackoff.go

@@ -0,0 +1,205 @@
+// Copyright © 2022 Obol Labs Inc.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of  MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// Package expbackoff implements exponential backoff. It was copied from google.golang.org/grpc.
+package expbackoff
+
+import (
+	"context"
+	"math/rand"
+	"testing"
+	"time"
+)
+
+// Config defines the configuration options for backoff.
+type Config struct {
+	// BaseDelay is the amount of time to backoff after the first failure.
+	BaseDelay time.Duration
+	// Multiplier is the factor with which to multiply backoffs after a
+	// failed retry. Should ideally be greater than 1.
+	Multiplier float64
+	// Jitter is the factor with which backoffs are randomized.
+	Jitter float64
+	// MaxDelay is the upper bound of backoff delay.
+	MaxDelay time.Duration
+}
+
+// DefaultConfig is a backoff configuration with the default values specified
+// at https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md.
+//
+// This should be useful for callers who want to configure backoff with
+// non-default values only for a subset of the options.
+//
+// Copied from google.golang.org/grpc@v1.48.0/backoff/backoff.go.
+var DefaultConfig = Config{
+	BaseDelay:  1.0 * time.Second,
+	Multiplier: 1.6,
+	Jitter:     0.2,
+	MaxDelay:   120 * time.Second,
+}
+
+// FastConfig is a common configuration for fast backoff.
+var FastConfig = Config{
+	BaseDelay:  100 * time.Millisecond,
+	Multiplier: 1.6,
+	Jitter:     0.2,
+	MaxDelay:   5 * time.Second,
+}
+
+// WithFastConfig configures the backoff with FastConfig.
+func WithFastConfig() func(*Config) {
+	return func(config *Config) {
+		*config = FastConfig
+	}
+}
+
+// WithConfig configures the backoff with the provided config.
+func WithConfig(c Config) func(*Config) {
+	return func(config *Config) {
+		*config = c
+	}
+}
+
+// WithMaxDelay configures the backoff with the provided max delay.
+func WithMaxDelay(d time.Duration) func(*Config) {
+	return func(config *Config) {
+		config.MaxDelay = d
+	}
+}
+
+// WithBaseDelay configures the backoff with the provided max delay.
+func WithBaseDelay(d time.Duration) func(*Config) {
+	return func(config *Config) {
+		config.BaseDelay = d
+	}
+}
+
+// New returns a backoff function configured via functional options applied to DefaultConfig.
+// The backoff function will exponentially sleep longer each time it is called.
+// The backoff function returns immediately after the context is cancelled.
+//
+// Usage:
+//  backoff := expbackoff.New(ctx)
+//  for ctx.Err() == nil {
+//    resp, err := doThing(ctx)
+//    if err != nil {
+//      backoff()
+//      continue
+//    } else {
+//      return resp
+//    }
+//  }
+func New(ctx context.Context, opts ...func(*Config)) (backoff func()) {
+	backoff, _ = NewWithReset(ctx, opts...)
+	return backoff
+}
+
+// NewWithReset returns a backoff and a reset function configured via functional options applied to DefaultConfig.
+// The backoff function will exponentially sleep longer each time it is called.
+// Calling the reset function will reset the backoff sleep duration to Config.BaseDelay.
+// The backoff function returns immediately after the context is cancelled.
+//
+// Usage:
+//  backoff, reset := expbackoff.NewWithReset(ctx)
+//  for ctx.Err() == nil {
+//    resp, err := doThing(ctx)
+//    if err != nil {
+//      backoff()
+//      continue
+//    } else {
+//      reset()
+//      // Do something with the response.
+//    }
+//  }
+func NewWithReset(ctx context.Context, opts ...func(*Config)) (backoff func(), reset func()) {
+	conf := DefaultConfig
+	for _, opt := range opts {
+		opt(&conf)
+	}
+
+	var retries int
+
+	backoff = func() {
+		if ctx.Err() != nil {
+			return
+		}
+
+		select {
+		case <-ctx.Done():
+		case <-after(Backoff(conf, retries)):
+		}
+		retries++
+	}
+
+	reset = func() {
+		retries = 0
+	}
+
+	return backoff, reset
+}
+
+// Backoff returns the amount of time to wait before the next retry given the
+// number of retries.
+// Copied from google.golang.org/grpc@v1.48.0/internal/backoff/backoff.go.
+func Backoff(config Config, retries int) time.Duration {
+	if retries == 0 {
+		return config.BaseDelay
+	}
+
+	backoff := float64(config.BaseDelay)
+	max := float64(config.MaxDelay)
+
+	for backoff < max && retries > 0 {
+		backoff *= config.Multiplier
+		retries--
+	}
+	if backoff > max {
+		backoff = max
+	}
+	// Randomize backoff delays so that if a cluster of requests start at
+	// the same time, they won't operate in lockstep.
+	backoff *= 1 + config.Jitter*(randFloat()*2-1)
+	if backoff < 0 {
+		return 0
+	}
+
+	return time.Duration(backoff)
+}
+
+// after is aliased for testing.
+var after = time.After
+
+// SetAfterForT sets the after internal function for testing.
+func SetAfterForT(t *testing.T, fn func(d time.Duration) <-chan time.Time) {
+	t.Helper()
+	cached := after
+	after = fn
+	t.Cleanup(func() {
+		after = cached
+	})
+}
+
+// randFloat is aliased for testing.
+var randFloat = rand.Float64
+
+// SetRandFloatForT sets the random float internal function for testing.
+func SetRandFloatForT(t *testing.T, fn func() float64) {
+	t.Helper()
+	cached := randFloat
+	randFloat = fn
+	t.Cleanup(func() {
+		randFloat = cached
+	})
+}

app/expbackoff/expbackoff_test.go

@@ -0,0 +1,151 @@
+// Copyright © 2022 Obol Labs Inc.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of  MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package expbackoff_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/obolnetwork/charon/app/expbackoff"
+)
+
+func TestConfigs(t *testing.T) {
+	tests := []struct {
+		name     string
+		config   expbackoff.Config
+		backoffs []string
+		jitter   float64
+	}{
+		{
+			name:   "default",
+			config: expbackoff.DefaultConfig,
+			jitter: 0.5,
+			backoffs: []string{
+				"1s",
+				"1.6s",
+				"2.56s",
+				"4.09s",
+				"6.55s",
+				"10.48s",
+				"16.77s",
+				"26.84s",
+				"42.94s",
+				"1m8.71s",
+				"1m49.95s",
+				"2m0s",
+				"2m0s",
+			},
+		},
+		{
+			name:   "default max jitter",
+			config: expbackoff.DefaultConfig,
+			jitter: 1,
+			backoffs: []string{
+				"1s",
+				"1.92s",
+				"3.07s",
+				"4.91s",
+				"7.86s",
+				"12.58s",
+				"20.13s",
+				"32.21s",
+				"51.53s",
+				"1m22.46s",
+				"2m11.94s",
+				"2m24s",
+				"2m24s",
+			},
+		},
+		{
+			name:   "fast",
+			config: expbackoff.FastConfig,
+			jitter: 0.5,
+			backoffs: []string{
+				"100ms",
+				"160ms",
+				"250ms",
+				"400ms",
+				"650ms",
+				"1.04s",
+				"1.67s",
+				"2.68s",
+				"4.29s",
+				"5s",
+				"5s",
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			expbackoff.SetRandFloatForT(t, func() float64 {
+				return test.jitter
+			})
+
+			var resps []string
+			for i := 0; i < len(test.backoffs); i++ {
+				resp := expbackoff.Backoff(test.config, i)
+				resps = append(resps, resp.Truncate(time.Millisecond*10).String())
+			}
+			require.Equal(t, test.backoffs, resps)
+		})
+	}
+}
+
+func TestNewWithReset(t *testing.T) {
+	t0 := time.Now()
+	now := t0
+	expbackoff.SetAfterForT(t, func(d time.Duration) <-chan time.Time {
+		now = now.Add(d)
+		ch := make(chan time.Time, 1)
+		ch <- now
+
+		return ch
+	})
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	backoff, reset := expbackoff.NewWithReset(ctx, expbackoff.WithConfig(expbackoff.Config{
+		BaseDelay:  time.Second,
+		Multiplier: 2,
+		Jitter:     0,
+		MaxDelay:   time.Hour,
+	}))
+
+	elapsed := func(t *testing.T, expect string) {
+		t.Helper()
+		require.Equal(t, expect, now.Sub(t0).Truncate(time.Millisecond*10).String())
+	}
+
+	backoff()
+	elapsed(t, "1s") // +1s
+	backoff()
+	elapsed(t, "3s") // +2s
+	backoff()
+	elapsed(t, "7s") // +4s
+	backoff()
+	elapsed(t, "15s") // +8s
+
+	reset()
+	backoff()
+	elapsed(t, "16s") // +1s
+
+	cancel()
+	backoff()
+	elapsed(t, "16s") // +0s
+}