Skip to content

Commit d4af6f7

Browse files
authored
feat(bigquery): augment retry predicate to support additional errors (#4046)
* feat(bigquery): augment retry predicate to support additional errors. Based on similar refactor in storage: #4019 Fixes: #4021
1 parent f9c4ccb commit d4af6f7

File tree

3 files changed

+66
-7
lines changed

3 files changed

+66
-7
lines changed

Diff for: bigquery/bigquery.go

+31-7
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ import (
1919
"fmt"
2020
"io"
2121
"net/http"
22+
"net/url"
23+
"strings"
2224
"time"
2325

2426
"cloud.google.com/go/internal"
@@ -169,6 +171,9 @@ func retryableError(err error) bool {
169171
if err == nil {
170172
return false
171173
}
174+
if err == io.ErrUnexpectedEOF {
175+
return true
176+
}
172177
// Special case due to http2: https://github.com/googleapis/google-cloud-go/issues/1793
173178
// Due to Go's default being higher for streams-per-connection than is accepted by the
174179
// BQ backend, it's possible to get streams refused immediately after a connection is
@@ -177,13 +182,32 @@ func retryableError(err error) bool {
177182
if err.Error() == "http2: stream closed" {
178183
return true
179184
}
180-
e, ok := err.(*googleapi.Error)
181-
if !ok {
182-
return false
185+
186+
switch e := err.(type) {
187+
case *googleapi.Error:
188+
// We received a structured error from backend.
189+
var reason string
190+
if len(e.Errors) > 0 {
191+
reason = e.Errors[0].Reason
192+
}
193+
if e.Code == http.StatusServiceUnavailable || e.Code == http.StatusBadGateway || reason == "backendError" || reason == "rateLimitExceeded" {
194+
return true
195+
}
196+
case *url.Error:
197+
retryable := []string{"connection refused", "connection reset"}
198+
for _, s := range retryable {
199+
if strings.Contains(e.Error(), s) {
200+
return true
201+
}
202+
}
203+
case interface{ Temporary() bool }:
204+
if e.Temporary() {
205+
return true
206+
}
183207
}
184-
var reason string
185-
if len(e.Errors) > 0 {
186-
reason = e.Errors[0].Reason
208+
// Unwrap is only supported in go1.13.x+
209+
if e, ok := err.(interface{ Unwrap() error }); ok {
210+
return retryableError(e.Unwrap())
187211
}
188-
return e.Code == http.StatusServiceUnavailable || e.Code == http.StatusBadGateway || reason == "backendError" || reason == "rateLimitExceeded"
212+
return false
189213
}

Diff for: bigquery/bigquery_test.go

+34
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,12 @@ package bigquery
1616

1717
import (
1818
"errors"
19+
"io"
1920
"net/http"
21+
"net/url"
2022
"testing"
2123

24+
"golang.org/x/xerrors"
2225
"google.golang.org/api/googleapi"
2326
)
2427

@@ -38,6 +41,11 @@ func TestRetryableErrors(t *testing.T) {
3841
errors.New("http2: stream closed"),
3942
true,
4043
},
44+
{
45+
"io ErrUnexpectedEOF",
46+
io.ErrUnexpectedEOF,
47+
true,
48+
},
4149
{
4250
"unavailable",
4351
&googleapi.Error{
@@ -46,6 +54,32 @@ func TestRetryableErrors(t *testing.T) {
4654
},
4755
true,
4856
},
57+
{
58+
"url connection error",
59+
&url.Error{Op: "blah", URL: "blah", Err: errors.New("connection refused")},
60+
true,
61+
},
62+
{
63+
"url other error",
64+
&url.Error{Op: "blah", URL: "blah", Err: errors.New("blah")},
65+
false,
66+
},
67+
{
68+
"wrapped retryable",
69+
xerrors.Errorf("test of wrapped retryable: %w", &googleapi.Error{
70+
Code: http.StatusServiceUnavailable,
71+
Message: "foo",
72+
Errors: []googleapi.ErrorItem{
73+
{Reason: "backendError", Message: "foo"},
74+
},
75+
}),
76+
true,
77+
},
78+
{
79+
"wrapped non-retryable",
80+
xerrors.Errorf("test of wrapped retryable: %w", errors.New("blah")),
81+
false,
82+
},
4983
{
5084
// not retried per https://google.aip.dev/194
5185
"internal error",

Diff for: bigquery/go.mod

+1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ require (
88
github.com/golang/protobuf v1.5.2
99
github.com/google/go-cmp v0.5.5
1010
github.com/googleapis/gax-go/v2 v2.0.5
11+
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
1112
google.golang.org/api v0.46.0
1213
google.golang.org/genproto v0.0.0-20210503173045-b96a97608f20
1314
google.golang.org/grpc v1.37.0

0 commit comments

Comments
 (0)