Add s3 connection reset retries (#1137)

* Add s3 connection reset retries

* Add test

pkg/storages/s3/retryer.go

@@ -0,0 +1,25 @@
+package s3
+
+import (
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws/request"
+)
+
+func NewConnResetRetryer(baseRetryer request.Retryer) *ConnResetRetryer {
+	return &ConnResetRetryer{
+		baseRetryer,
+	}
+}
+
+type ConnResetRetryer struct {
+	request.Retryer
+}
+
+func (r ConnResetRetryer) ShouldRetry(req *request.Request) bool {
+	if req.Error != nil && strings.Contains(req.Error.Error(), "connection reset by peer") {
+		return true
+	}
+
+	return r.Retryer.ShouldRetry(req)
+}

pkg/storages/s3/retryer_test.go

@@ -0,0 +1,43 @@
+package s3
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"syscall"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws/client"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConnResetRetryerRetry(t *testing.T) {
+	retryer := NewConnResetRetryer(client.DefaultRetryer{})
+	err := &net.OpError{
+		Op:     "mock",
+		Net:    "mock",
+		Source: &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: 1234},
+		Addr:   &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: 12340},
+		Err:    &os.SyscallError{Syscall: "read", Err: syscall.ECONNRESET},
+	}
+
+	req := &request.Request{
+		Error: err,
+	}
+
+	assert.True(t, retryer.ShouldRetry(req))
+}
+
+func TestConnResetRetryerRandomError(t *testing.T) {
+	retryer := NewConnResetRetryer(client.DefaultRetryer{})
+	req := &request.Request{
+		Error: fmt.Errorf("some strange unknown error"),
+	}
+	assert.False(t, retryer.ShouldRetry(req))
+}
+
+func TestConnResetRetryerNoError(t *testing.T) {
+	retryer := NewConnResetRetryer(client.DefaultRetryer{})
+	assert.False(t, retryer.ShouldRetry(&request.Request{}))
+}

pkg/storages/s3/session.go

@@ -87,7 +87,7 @@ func getDefaultConfig(settings map[string]string) *aws.Config {
 	// most services. If you want to implement custom retry logic, you can implement the
 	// request.Retryer interface.
 	config := defaults.Get().Config.WithRegion(settings[RegionSetting])
-	config = request.WithRetryer(config, client.DefaultRetryer{NumMaxRetries: MaxRetries})
+	config = request.WithRetryer(config, NewConnResetRetryer(client.DefaultRetryer{NumMaxRetries: MaxRetries}))
 
 	if logLevel, ok := settings[LogLevel]; ok {
 		config = config.WithLogLevel(func(s string) aws.LogLevelType {