spanner: only retry certain types of internal errors. (#2460)

* spanner: only retry certain types of internal errors.

spanner/client_test.go

@@ -228,6 +228,25 @@ func TestClient_Single_NonRetryableErrorOnPartialResultSet(t *testing.T) {
 	}
 }
 
+func TestClient_Single_NonRetryableInternalErrors(t *testing.T) {
+	t.Parallel()
+	server, client, teardown := setupMockedTestServer(t)
+	defer teardown()
+
+	server.TestSpanner.AddPartialResultSetError(
+		SelectSingerIDAlbumIDAlbumTitleFromAlbums,
+		PartialResultSetExecutionTime{
+			ResumeToken: EncodeResumeToken(2),
+			Err:         status.Errorf(codes.Internal, "grpc: error while marshaling: string field contains invalid UTF-8"),
+		},
+	)
+	ctx := context.Background()
+	err := executeSingerQuery(ctx, client.Single())
+	if status.Code(err) != codes.Internal {
+		t.Fatalf("Error mismatch:\ngot: %v\nwant: %v", err, codes.Internal)
+	}
+}
+
 func TestClient_Single_DeadlineExceeded_NoErrors(t *testing.T) {
 	t.Parallel()
 	server, client, teardown := setupMockedTestServer(t)

spanner/read.go

@@ -457,7 +457,7 @@ var (
 )
 
 func (d *resumableStreamDecoder) next() bool {
-	retryer := gax.OnCodes([]codes.Code{codes.Unavailable, codes.Internal}, d.backoff)
+	retryer := onCodes(d.backoff, codes.Unavailable, codes.Internal)
 	for {
 		switch d.state {
 		case unConnected:

spanner/retry.go

@@ -18,6 +18,7 @@ package spanner
 
 import (
 	"context"
+	"strings"
 	"time"
 
 	"cloud.google.com/go/internal/trace"
@@ -47,7 +48,8 @@ type spannerRetryer struct {
 }
 
 // onCodes returns a spannerRetryer that will retry on the specified error
-// codes.
+// codes. For Internal errors, only errors that have one of a list of known
+// descriptions should be retried.
 func onCodes(bo gax.Backoff, cc ...codes.Code) gax.Retryer {
 	return &spannerRetryer{
 		Retryer: gax.OnCodes(cc, bo),
@@ -57,6 +59,16 @@ func onCodes(bo gax.Backoff, cc ...codes.Code) gax.Retryer {
 // Retry returns the retry delay returned by Cloud Spanner if that is present.
 // Otherwise it returns the retry delay calculated by the generic gax Retryer.
 func (r *spannerRetryer) Retry(err error) (time.Duration, bool) {
+	if status.Code(err) == codes.Internal &&
+		!strings.Contains(err.Error(), "stream terminated by RST_STREAM") &&
+		// See b/25451313.
+		!strings.Contains(err.Error(), "HTTP/2 error code: INTERNAL_ERROR") &&
+		// See b/27794742.
+		!strings.Contains(err.Error(), "Connection closed with unknown cause") &&
+		!strings.Contains(err.Error(), "Received unexpected EOS on DATA frame from server") {
+		return 0, false
+	}
+
 	delay, shouldRetry := r.Retryer.Retry(err)
 	if !shouldRetry {
 		return 0, false