Skip to content
Permalink
Browse files
fix(profiler): Force gax to retry in case of certificate errors (#3178)
* fix(profiler): force gax.Invoke() to retry cert errors

Co-authored-by: Alexey Alexandrov <aalexand@users.noreply.github.com>
  • Loading branch information
kalyanac and aalexand committed Jan 28, 2021
1 parent b75df35 commit 35dcd725dcd03266ed7439de40c277376b38cd71
Showing with 11 additions and 2 deletions.
  1. +11 −2 profiler/profiler.go
@@ -44,6 +44,7 @@ import (
"regexp"
"runtime"
"runtime/pprof"
"strings"
"sync"
"time"

@@ -67,6 +68,7 @@ var (
config Config
startOnce allowUntilSuccess
mutexEnabled bool
logger *log.Logger
// The functions below are stubbed to be overrideable for testing.
getProjectID = gcemd.ProjectID
getInstanceName = gcemd.InstanceName
@@ -222,6 +224,7 @@ func Start(cfg Config, options ...option.ClientOption) error {
}

func start(cfg Config, options ...option.ClientOption) error {
logger = log.New(os.Stderr, "Cloud Profiler: ", log.LstdFlags)
if err := initializeConfig(cfg); err != nil {
debugLog("failed to initialize config: %v", err)
return err
@@ -261,7 +264,7 @@ func start(cfg Config, options ...option.ClientOption) error {

func debugLog(format string, e ...interface{}) {
if config.DebugLogging {
log.Printf(format, e...)
logger.Printf(format, e...)
}
}

@@ -315,7 +318,8 @@ func (r *retryer) Retry(err error) (time.Duration, bool) {
// createProfile talks to the profiler server to create profile. In
// case of error, the goroutine will sleep and retry. Sleep duration may
// be specified by the server. Otherwise it will be an exponentially
// increasing value, bounded by maxBackoff.
// increasing value, bounded by maxBackoff. Special handling for
// certificate errors is described below.
func (a *agent) createProfile(ctx context.Context) *pb.Profile {
req := pb.CreateProfileRequest{
Parent: "projects/" + a.deployment.ProjectId,
@@ -332,6 +336,11 @@ func (a *agent) createProfile(ctx context.Context) *pb.Profile {
p, err = a.client.CreateProfile(ctx, &req, grpc.Trailer(&md))
if err != nil {
debugLog("failed to create profile, will retry: %v", err)
if strings.Contains(err.Error(), "x509: certificate signed by unknown authority") {
// gax.Invoke does not retry missing certificate error. Force a retry by returning
// a different error. See https://github.com/googleapis/google-cloud-go/issues/3158.
err = errors.New("retry the certificate error")
}
}
return err
}, gax.WithRetry(func() gax.Retryer {

0 comments on commit 35dcd72

Please sign in to comment.