Allow only system failures to trigger circuit breaker

codegen/templates/http_client.tmpl

@@ -238,10 +238,20 @@ func (c *{{$clientName}}) {{$methodName}}(
 	if (c.circuitBreakerDisabled) {
 		res, err = req.Do()
 	} else {
+		// We want hystrix ckt-breaker to count errors only for system issues
+		var clientErr error
 		err = hystrix.DoC(ctx, "{{$clientID}}", func(ctx context.Context) error {
-			res, err = req.Do()
-			return err
+			res, clientErr = req.Do()
+			if res.StatusCode < 500 {
+				// This is not a system error/issue
+				return nil
+			}
+			return clientErr
 		}, nil)
+		if err == nil {
+			// ckt-breaker was ok, bubble up client error if set
+			err = clientErr
+		}
 	}
 	if err != nil {
 		return {{if eq .ResponseType ""}}nil, err{{else}}defaultRes, nil, err{{end}}

codegen/templates/tchannel_client.tmpl

@@ -264,15 +264,24 @@ type {{$clientName}} struct {
 				ctx, "{{$svc.Name}}", "{{.Name}}", reqHeaders, args, &result,
 			)
 		} else {
+			// We want hystrix ckt-breaker to count errors only for system issues
+			var clientErr error
 			err = hystrix.DoC(ctx, "{{$clientID}}", func(ctx context.Context) error {
-				success, respHeaders, err = c.client.Call(
+				success, respHeaders, clientErr = c.client.Call(
 					ctx, "{{$svc.Name}}", "{{.Name}}", reqHeaders, args, &result,
 				)
-				return err
+				if _, isSysErr := clientErr.(tchannel.SystemError); !isSysErr {
+					// Declare ok if it is not a system-error
+					return nil
+				}
+				return clientErr
 			}, nil)
+			if err == nil {
+				// ckt-breaker was ok, bubble up client error if set
+				err = clientErr
+			}
 		}
 
-
 		if err == nil && !success {
 			switch {
 				{{range .Exceptions -}}