ep level timeout and retry support

uber · Sep 21, 2022 · 86a6b90 · 86a6b90
1 parent 63e79be
commit 86a6b90
Show file tree

Hide file tree

Showing 99 changed files with 1,826 additions and 593 deletions.
diff --git a/codegen/template_bundle/template_files.go b/codegen/template_bundle/template_files.go
diff --git a/codegen/templates/http_client.tmpl b/codegen/templates/http_client.tmpl
@@ -45,6 +45,7 @@ type Client interface {
 		{{if ne .RequestType "" -}}
 		args {{.RequestType}},
 		{{end -}}
+		timeoutAndRetryOps *zanzibar.TimeoutAndRetryOptions,
 	) (context.Context, {{- if ne .ResponseType "" -}} {{.ResponseType}}, {{- end -}}map[string]string, error)
 {{- end -}}
 {{- end -}}
@@ -258,6 +259,7 @@ func (c *{{$clientName}}) {{$methodName}}(
 	{{if ne .RequestType "" -}}
 	r {{.RequestType}},
 	{{end -}}
+	timeoutAndRetryOps *zanzibar.TimeoutAndRetryOptions,
 ) (context.Context, {{- if ne .ResponseType "" -}} {{.ResponseType}}, {{- end -}}map[string]string, error) {
 	reqUUID := zanzibar.RequestUUIDFromCtx(ctx)
 	if headers == nil {
@@ -273,7 +275,7 @@ func (c *{{$clientName}}) {{$methodName}}(
 	{{if .ResponseType -}}
 	var defaultRes  {{.ResponseType}}
 	{{end -}}
-	req := zanzibar.NewClientHTTPRequest(ctx, c.clientID, "{{$methodName}}", "{{$serviceMethod}}", c.httpClient)
+	req := zanzibar.NewClientHTTPRequest(ctx, c.clientID, "{{$methodName}}", "{{$serviceMethod}}", c.httpClient,timeoutAndRetryOps)
 
 	{{if .ReqHeaderGoStatements }}
 	{{range $index, $line := .ReqClientHeaderGoStatements -}}

diff --git a/codegen/templates/service_mock.tmpl b/codegen/templates/service_mock.tmpl
@@ -38,6 +38,7 @@ type MockService interface {
 		method string,
 		headers map[string]string,
 		req, resp zanzibar.RWTStruct,
+		timeoutAndRetryOps *zanzibar.TimeoutAndRetryOptions,
 	) (bool, map[string]string, error)
 	{{$mock}}() *{{$mockType}}
 	Server() *zanzibar.Gateway
@@ -188,12 +189,13 @@ func (m *mockService) MakeTChannelRequest(
 	method string,
 	headers map[string]string,
 	req, res zanzibar.RWTStruct,
+	timeoutAndRetryOps *zanzibar.TimeoutAndRetryOptions,
 ) (bool, map[string]string, error) {
 	if !m.started {
 		return false, nil, errors.New("mock server is not started")
 	}
 
 	sc := m.server.ServerTChannel.GetSubChannel(m.server.ServiceName)
 	sc.Peers().Add(m.server.RealTChannelAddr)
-	return m.tChannelClient.Call(ctx, thriftService, method, headers, req, res)
+	return m.tChannelClient.Call(ctx, thriftService, method, headers, req, res, timeoutAndRetryOps)
 }
diff --git a/codegen/templates/tchannel_client.tmpl b/codegen/templates/tchannel_client.tmpl
@@ -49,6 +49,7 @@ type Client interface {
 		{{if ne .RequestType "" -}}
 		args {{.RequestType}},
 		{{end -}}
+		timeoutAndRetryOps *zanzibar.TimeoutAndRetryOptions,
 	) (context.Context, {{- if ne .ResponseType "" -}} {{.ResponseType}}, {{- end -}}map[string]string, error)
 {{- end -}}
 {{- end -}}
@@ -329,6 +330,7 @@ type {{$clientName}} struct {
 		{{if ne .RequestType "" -}}
 		args {{.RequestType}},
 		{{end -}}
+		timeoutAndRetryOps *zanzibar.TimeoutAndRetryOptions,
 	) (context.Context, {{- if ne .ResponseType "" -}} {{.ResponseType}}, {{- end -}}map[string]string, error) {
 		var result {{.GenCodePkgName}}.{{title $svc.Name}}_{{title .Name}}_Result
 		{{if .ResponseType -}}
@@ -345,7 +347,7 @@ type {{$clientName}} struct {
 		var err error
 		if (c.circuitBreakerDisabled) {
 			success, respHeaders, err = c.client.Call(
-				ctx, "{{$svc.Name}}", "{{.Name}}", reqHeaders, args, &result,
+				ctx, "{{$svc.Name}}", "{{.Name}}", reqHeaders, args, &result, timeoutAndRetryOps,
 			)
 		} else {
 			// We want hystrix ckt-breaker to count errors only for system issues
@@ -360,7 +362,7 @@ type {{$clientName}} struct {
 				elapsed := time.Now().Sub(start)
 				scope.Timer("hystrix-timer").Record(elapsed)
 				success, respHeaders, clientErr = c.client.Call(
-					ctx, "{{$svc.Name}}", "{{.Name}}", reqHeaders, args, &result,
+					ctx, "{{$svc.Name}}", "{{.Name}}", reqHeaders, args, &result, timeoutAndRetryOps,
 				)
 				if _, isSysErr := clientErr.(tchannel.SystemError); !isSysErr {
 					// Declare ok if it is not a system-error

diff --git a/codegen/templates/tchannel_endpoint.tmpl b/codegen/templates/tchannel_endpoint.tmpl
@@ -241,6 +241,8 @@ func (h *{{$handlerName}}) redirectToDeputy(
 		h.Deps.Default.Config.MustGetInt("tchannel.deputy.timeoutPerAttempt"),
 	)
 
+	maxAttempts := int(h.Deps.Default.Config.MustGetInt("clients.{{$clientID}}.retryCount"))
+
 	methodNames := map[string]string{
 		"{{.ThriftService}}::{{.Name}}": "{{$methodName}}",
 	}
@@ -266,7 +268,13 @@ func (h *{{$handlerName}}) redirectToDeputy(
 		},
 	)
 
-	success, respHeaders, err := client.Call(ctx, "{{.ThriftService}}", "{{$methodName}}", reqHeaders, req, res)
+	timeoutAndRetryOps := zanzibar.TimeoutAndRetryOptions{
+		OverallTimeoutInMs: 			(timeoutPerAttempt + zanzibar.DefaultBackOffTimeAcrossRetries) * time.Duration(maxAttempts),
+		RequestTimeoutPerAttemptInMs:	timeoutPerAttempt,
+		MaxAttempts:					maxAttempts,
+		BackOffTimeAcrossRetriesInMs: 	zanzibar.DefaultBackOffTimeAcrossRetries,
+	}
+	success, respHeaders, err := client.Call(ctx, "{{.ThriftService}}", "{{$methodName}}", reqHeaders, req, res, &timeoutAndRetryOps)
 	return ctx, success, res, respHeaders, err
 }
 {{end -}}