Skip to content


Browse files Browse the repository at this point in the history
feat(internal): provide wrapping for retried errors (#4797)
This modifies the retry code to allow both context and service errors to be available when the call was retried until hitting a context deadline or cancelation.

This preserves the error string as it is currently put together by internal/annotate; for example a googleapi error from storage looks like this:

`object.Attrs: retry failed with context deadline exceeded; last error: googleapi: got HTTP response code 503 with body: {"code":"503","message":{"error":{"message":"Retry Test: Caused a 503"}}}`

Go 1.13 error semantics can be used to introspect the underlying context and service errors; here are some examples:

_, err = client.Bucket(bucketName).Object(objName).Attrs(timeoutCtx)
if err != nil {
	if e, ok := err.(interface{ Unwrap() error }); ok {
		wrappedErr := e.Unwrap()  // yields googleapi.Error type.

	// errors.As allows unwrapping with access to googleapi.Error fields.
	var errAsGoogleapi *googleapi.Error
	if errors.As(err, &errAsGoogleapi) {
		log.Printf("error code: %v", errAsGoogleapi.Code)

	isDeadlineExceeded := errors.Is(err, context.DeadlineExceeded)  // true

internal.Retry is used by the storage, datastore and bigquery clients, as well as integration tests for compute and pubsub. I want to make sure to check this with everyone affected.
  • Loading branch information
tritone committed Sep 23, 2021
1 parent 797a9bd commit ce5f4db
Show file tree
Hide file tree
Showing 2 changed files with 42 additions and 5 deletions.
37 changes: 34 additions & 3 deletions internal/retry.go
Expand Up @@ -16,9 +16,11 @@ package internal

import (

gax ""

// Retry calls the supplied function f repeatedly according to the provided
Expand All @@ -44,11 +46,40 @@ func retry(ctx context.Context, bo gax.Backoff, f func() (stop bool, err error),
lastErr = err
p := bo.Pause()
if cerr := sleep(ctx, p); cerr != nil {
if ctxErr := sleep(ctx, p); ctxErr != nil {
if lastErr != nil {
return Annotatef(lastErr, "retry failed with %v; last error", cerr)
return wrappedCallErr{ctxErr: ctxErr, wrappedErr: lastErr}
return cerr
return ctxErr

// Use this error type to return an error which allows introspection of both
// the context error and the error from the service.
type wrappedCallErr struct {
ctxErr error
wrappedErr error

func (e wrappedCallErr) Error() string {
return fmt.Sprintf("retry failed with %v; last error: %v", e.ctxErr, e.wrappedErr)

func (e wrappedCallErr) Unwrap() error {
return e.wrappedErr

// Is allows errors.Is to match the error from the call as well as context
// sentinel errors.
func (e wrappedCallErr) Is(err error) bool {
return e.ctxErr == err || e.wrappedErr == err

// GRPCStatus allows the wrapped error to be used with status.FromError.
func (e wrappedCallErr) GRPCStatus() *status.Status {
if s, ok := status.FromError(e.wrappedErr); ok {
return s
return nil
10 changes: 8 additions & 2 deletions internal/retry_test.go
Expand Up @@ -17,7 +17,6 @@ package internal
import (

Expand Down Expand Up @@ -75,14 +74,21 @@ func TestRetryPreserveError(t *testing.T) {
func(context.Context, time.Duration) error {
return context.DeadlineExceeded
if err == nil {
t.Fatalf("unexpectedly got nil error")
wantError := "retry failed with context deadline exceeded; last error: rpc error: code = NotFound desc = not found"
if g, w := err.Error(), wantError; g != w {
t.Errorf("got error %q, want %q", g, w)
got, ok := status.FromError(err)
if !ok {
t.Fatalf("got %T, wanted a status", got)
if g, w := got.Code(), codes.NotFound; g != w {
t.Errorf("got code %v, want %v", g, w)
wantMessage := fmt.Sprintf("retry failed with %v; last error: not found", context.DeadlineExceeded)
wantMessage := "not found"
if g, w := got.Message(), wantMessage; g != w {
t.Errorf("got message %q, want %q", g, w)
Expand Down

0 comments on commit ce5f4db

Please sign in to comment.