Skip to content

Commit

Permalink
fix(profiler): Force gax to retry in case of certificate errors (#3178)
Browse files Browse the repository at this point in the history
* fix(profiler): force gax.Invoke() to retry cert errors

Co-authored-by: Alexey Alexandrov <aalexand@users.noreply.github.com>
  • Loading branch information
kalyanac and aalexand authored Jan 28, 2021
1 parent b75df35 commit 35dcd72
Showing 1 changed file with 11 additions and 2 deletions.
13 changes: 11 additions & 2 deletions profiler/profiler.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ import (
"regexp"
"runtime"
"runtime/pprof"
"strings"
"sync"
"time"

Expand All @@ -67,6 +68,7 @@ var (
config Config
startOnce allowUntilSuccess
mutexEnabled bool
logger *log.Logger
// The functions below are stubbed to be overrideable for testing.
getProjectID = gcemd.ProjectID
getInstanceName = gcemd.InstanceName
Expand Down Expand Up @@ -222,6 +224,7 @@ func Start(cfg Config, options ...option.ClientOption) error {
}

func start(cfg Config, options ...option.ClientOption) error {
logger = log.New(os.Stderr, "Cloud Profiler: ", log.LstdFlags)
if err := initializeConfig(cfg); err != nil {
debugLog("failed to initialize config: %v", err)
return err
Expand Down Expand Up @@ -261,7 +264,7 @@ func start(cfg Config, options ...option.ClientOption) error {

func debugLog(format string, e ...interface{}) {
if config.DebugLogging {
log.Printf(format, e...)
logger.Printf(format, e...)
}
}

Expand Down Expand Up @@ -315,7 +318,8 @@ func (r *retryer) Retry(err error) (time.Duration, bool) {
// createProfile talks to the profiler server to create profile. In
// case of error, the goroutine will sleep and retry. Sleep duration may
// be specified by the server. Otherwise it will be an exponentially
// increasing value, bounded by maxBackoff.
// increasing value, bounded by maxBackoff. Special handling for
// certificate errors is described below.
func (a *agent) createProfile(ctx context.Context) *pb.Profile {
req := pb.CreateProfileRequest{
Parent: "projects/" + a.deployment.ProjectId,
Expand All @@ -332,6 +336,11 @@ func (a *agent) createProfile(ctx context.Context) *pb.Profile {
p, err = a.client.CreateProfile(ctx, &req, grpc.Trailer(&md))
if err != nil {
debugLog("failed to create profile, will retry: %v", err)
if strings.Contains(err.Error(), "x509: certificate signed by unknown authority") {
// gax.Invoke does not retry missing certificate error. Force a retry by returning
// a different error. See https://github.com/googleapis/google-cloud-go/issues/3158.
err = errors.New("retry the certificate error")
}
}
return err
}, gax.WithRetry(func() gax.Retryer {
Expand Down

0 comments on commit 35dcd72

Please sign in to comment.