feat(internal): provide wrapping for retried errors (#4797)

This modifies the retry code to allow both context and service errors to be available when the call was retried until hitting a context deadline or cancelation.

This preserves the error string as it is currently put together by internal/annotate; for example a googleapi error from storage looks like this:

`object.Attrs: retry failed with context deadline exceeded; last error: googleapi: got HTTP response code 503 with body: {"code":"503","message":{"error":{"message":"Retry Test: Caused a 503"}}}`

Go 1.13 error semantics can be used to introspect the underlying context and service errors; here are some examples:

```
_, err = client.Bucket(bucketName).Object(objName).Attrs(timeoutCtx)
if err != nil {
	if e, ok := err.(interface{ Unwrap() error }); ok {
		wrappedErr := e.Unwrap()  // yields googleapi.Error type.
	}

	// errors.As allows unwrapping with access to googleapi.Error fields.
	var errAsGoogleapi *googleapi.Error
	if errors.As(err, &errAsGoogleapi) {
		log.Printf("error code: %v", errAsGoogleapi.Code)
	}

	isDeadlineExceeded := errors.Is(err, context.DeadlineExceeded)  // true
}
```

internal.Retry is used by the storage, datastore and bigquery clients, as well as integration tests for compute and pubsub. I want to make sure to check this with everyone affected.

internal/retry.go

@@ -16,9 +16,11 @@ package internal
 
 import (
 	"context"
+	"fmt"
 	"time"
 
 	gax "github.com/googleapis/gax-go/v2"
+	"google.golang.org/grpc/status"
 )
 
 // Retry calls the supplied function f repeatedly according to the provided
@@ -44,11 +46,40 @@ func retry(ctx context.Context, bo gax.Backoff, f func() (stop bool, err error),
 			lastErr = err
 		}
 		p := bo.Pause()
-		if cerr := sleep(ctx, p); cerr != nil {
+		if ctxErr := sleep(ctx, p); ctxErr != nil {
 			if lastErr != nil {
-				return Annotatef(lastErr, "retry failed with %v; last error", cerr)
+				return wrappedCallErr{ctxErr: ctxErr, wrappedErr: lastErr}
 			}
-			return cerr
+			return ctxErr
 		}
 	}
 }
+
+// Use this error type to return an error which allows introspection of both
+// the context error and the error from the service.
+type wrappedCallErr struct {
+	ctxErr     error
+	wrappedErr error
+}
+
+func (e wrappedCallErr) Error() string {
+	return fmt.Sprintf("retry failed with %v; last error: %v", e.ctxErr, e.wrappedErr)
+}
+
+func (e wrappedCallErr) Unwrap() error {
+	return e.wrappedErr
+}
+
+// Is allows errors.Is to match the error from the call as well as context
+// sentinel errors.
+func (e wrappedCallErr) Is(err error) bool {
+	return e.ctxErr == err || e.wrappedErr == err
+}
+
+// GRPCStatus allows the wrapped error to be used with status.FromError.
+func (e wrappedCallErr) GRPCStatus() *status.Status {
+	if s, ok := status.FromError(e.wrappedErr); ok {
+		return s
+	}
+	return nil
+}

internal/retry_test.go

@@ -17,7 +17,6 @@ package internal
 import (
 	"context"
 	"errors"
-	"fmt"
 	"testing"
 	"time"
 
@@ -75,14 +74,21 @@ func TestRetryPreserveError(t *testing.T) {
 		func(context.Context, time.Duration) error {
 			return context.DeadlineExceeded
 		})
+	if err == nil {
+		t.Fatalf("unexpectedly got nil error")
+	}
+	wantError := "retry failed with context deadline exceeded; last error: rpc error: code = NotFound desc = not found"
+	if g, w := err.Error(), wantError; g != w {
+		t.Errorf("got error %q, want %q", g, w)
+	}
 	got, ok := status.FromError(err)
 	if !ok {
 		t.Fatalf("got %T, wanted a status", got)
 	}
 	if g, w := got.Code(), codes.NotFound; g != w {
 		t.Errorf("got code %v, want %v", g, w)
 	}
-	wantMessage := fmt.Sprintf("retry failed with %v; last error: not found", context.DeadlineExceeded)
+	wantMessage := "not found"
 	if g, w := got.Message(), wantMessage; g != w {
 		t.Errorf("got message %q, want %q", g, w)
 	}