Skip to content

Commit

Permalink
Add the ability for ribbon clients to specify response status codes t…
Browse files Browse the repository at this point in the history
…hey would like to retry. Fixes #1540.
  • Loading branch information
ryanjbaxter committed Apr 4, 2017
1 parent d331444 commit 6290859
Show file tree
Hide file tree
Showing 15 changed files with 447 additions and 75 deletions.
16 changes: 16 additions & 0 deletions docs/src/main/asciidoc/spring-cloud-netflix.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -2448,6 +2448,22 @@ certain Ribbon properties. The properties you can use are
`client.ribbon.OkToRetryOnAllOperations`. See the https://github.com/Netflix/ribbon/wiki/Getting-Started#the-properties-file-sample-clientproperties[Ribbon documentation]
for a description of what there properties do.

In addition you may want to retry requests when certain status codes are returned in the
response. You can list the response codes you would like the Ribbon client to retry using the
property `clientName.ribbon.retryableStatusCodes`. For example

[source,yaml]
----
clientName:
ribbon:
retryableStatusCodes: 404,502
----

You can also create a bean of type `LoadBalancedRetryPolicy` and implement the `retryableStatusCode`
method to determine whether you want to retry a request given the status code.



==== Zuul

You can turn off Zuul's retry functionality by setting `zuul.retryable` to `false`. You
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.springframework.cloud.client.loadbalancer.ServiceInstanceChooser;
import org.springframework.cloud.netflix.ribbon.RibbonLoadBalancerClient;
import org.springframework.cloud.netflix.ribbon.ServerIntrospector;
import org.springframework.cloud.netflix.ribbon.support.RetryableStatusCodeException;
import org.springframework.retry.RetryCallback;
import org.springframework.retry.RetryContext;
import org.springframework.retry.policy.NeverRetryPolicy;
Expand Down Expand Up @@ -69,7 +70,7 @@ public RibbonResponse execute(final RibbonRequest request, IClientConfig configO
else {
options = new Request.Options(this.connectTimeout, this.readTimeout);
}
LoadBalancedRetryPolicy retryPolicy = loadBalancedRetryPolicyFactory.create(this.getClientName(), this);
final LoadBalancedRetryPolicy retryPolicy = loadBalancedRetryPolicyFactory.create(this.getClientName(), this);
RetryTemplate retryTemplate = new RetryTemplate();
retryTemplate.setRetryPolicy(retryPolicy == null ? new NeverRetryPolicy()
: new FeignRetryPolicy(request.toHttpRequest(), retryPolicy, this, this.getClientName()));
Expand All @@ -89,6 +90,9 @@ public RibbonResponse doWithRetry(RetryContext retryContext) throws IOException
feignRequest = request.toRequest();
}
Response response = request.client().execute(feignRequest, options);
if(retryPolicy.retryableStatusCode(response.status())) {
throw new RetryableStatusCodeException(RetryableFeignLoadBalancer.this.getClientName(), response.status());
}
return new RibbonResponse(request.getUri(), response);
}
});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -90,12 +90,14 @@ public LoadBalancerClient loadBalancerClient() {

@Bean
@ConditionalOnClass(name = "org.springframework.retry.support.RetryTemplate")
@ConditionalOnMissingBean
public LoadBalancedRetryPolicyFactory loadBalancedRetryPolicyFactory(SpringClientFactory clientFactory) {
return new RibbonLoadBalancedRetryPolicyFactory(clientFactory);
}

@Bean
@ConditionalOnMissingClass(value = "org.springframework.retry.support.RetryTemplate")
@ConditionalOnMissingBean
public LoadBalancedRetryPolicyFactory neverRetryPolicyFactory() {
return new LoadBalancedRetryPolicyFactory.NeverRetryFactory();
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
/*
*
* * Copyright 2013-2016 the original author or authors.
* *
* * Licensed under the Apache License, Version 2.0 (the "License");
* * you may not use this file except in compliance with the License.
* * You may obtain a copy of the License at
* *
* * http://www.apache.org/licenses/LICENSE-2.0
* *
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS,
* * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* * See the License for the specific language governing permissions and
* * limitations under the License.
*
*/

package org.springframework.cloud.netflix.ribbon;

import java.util.ArrayList;
import java.util.List;
import org.springframework.cloud.client.loadbalancer.LoadBalancedRetryContext;
import org.springframework.cloud.client.loadbalancer.LoadBalancedRetryPolicy;
import org.springframework.cloud.client.loadbalancer.ServiceInstanceChooser;
import org.springframework.core.env.Environment;
import org.springframework.http.HttpMethod;
import org.springframework.util.StringUtils;
import com.netflix.client.config.CommonClientConfigKey;
import com.netflix.client.config.IClientConfig;
import com.netflix.client.config.IClientConfigKey;

/**
* {@link LoadBalancedRetryPolicy} for Ribbon clients.
* @author Ryan Baxter
*/
public class RibbonLoadBalancedRetryPolicy implements LoadBalancedRetryPolicy {

public static final IClientConfigKey<String> RETRYABLE_STATUS_CODES = new CommonClientConfigKey<String>("retryableStatusCodes") {};
private int sameServerCount = 0;
private int nextServerCount = 0;
private String serviceId;
private RibbonLoadBalancerContext lbContext;
private ServiceInstanceChooser loadBalanceChooser;
List<Integer> retryableStatusCodes = new ArrayList<>();

public RibbonLoadBalancedRetryPolicy(String serviceId, RibbonLoadBalancerContext context, ServiceInstanceChooser loadBalanceChooser) {
this.serviceId = serviceId;
this.lbContext = context;
this.loadBalanceChooser = loadBalanceChooser;
}

public RibbonLoadBalancedRetryPolicy(String serviceId, RibbonLoadBalancerContext context, ServiceInstanceChooser loadBalanceChooser,
IClientConfig clientConfig) {
this.serviceId = serviceId;
this.lbContext = context;
this.loadBalanceChooser = loadBalanceChooser;
String retryableStatusCodesProp = clientConfig.getPropertyAsString(RETRYABLE_STATUS_CODES, "");
String[] retryableStatusCodesArray = retryableStatusCodesProp.split(",");
for(String code : retryableStatusCodesArray) {
if(!StringUtils.isEmpty(code)) {
try {
retryableStatusCodes.add(Integer.valueOf(code));
} catch (NumberFormatException e) {
//TODO log
}
}
}
}

public boolean canRetry(LoadBalancedRetryContext context) {
HttpMethod method = context.getRequest().getMethod();
return HttpMethod.GET == method || lbContext.isOkToRetryOnAllOperations();
}

@Override
public boolean canRetrySameServer(LoadBalancedRetryContext context) {
return sameServerCount < lbContext.getRetryHandler().getMaxRetriesOnSameServer() && canRetry(context);
}

@Override
public boolean canRetryNextServer(LoadBalancedRetryContext context) {
//this will be called after a failure occurs and we increment the counter
//so we check that the count is less than or equals to too make sure
//we try the next server the right number of times
return nextServerCount <= lbContext.getRetryHandler().getMaxRetriesOnNextServer() && canRetry(context);
}

@Override
public void close(LoadBalancedRetryContext context) {

}

@Override
public void registerThrowable(LoadBalancedRetryContext context, Throwable throwable) {
//Check if we need to ask the load balancer for a new server.
//Do this before we increment the counters because the first call to this method
//is not a retry it is just an initial failure.
if(!canRetrySameServer(context) && canRetryNextServer(context)) {
context.setServiceInstance(loadBalanceChooser.choose(serviceId));
}
//This method is called regardless of whether we are retrying or making the first request.
//Since we do not count the initial request in the retry count we don't reset the counter
//until we actually equal the same server count limit. This will allow us to make the initial
//request plus the right number of retries.
if(sameServerCount >= lbContext.getRetryHandler().getMaxRetriesOnSameServer() && canRetry(context)) {
//reset same server since we are moving to a new server
sameServerCount = 0;
nextServerCount++;
if(!canRetryNextServer(context)) {
context.setExhaustedOnly();
}
} else {
sameServerCount++;
}

}

@Override
public boolean retryableStatusCode(int statusCode) {
return retryableStatusCodes.contains(statusCode);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,9 @@
*/
package org.springframework.cloud.netflix.ribbon;

import org.springframework.cloud.client.ServiceInstance;
import org.springframework.cloud.client.loadbalancer.LoadBalancedRetryContext;
import org.springframework.cloud.client.loadbalancer.LoadBalancedRetryPolicy;
import org.springframework.cloud.client.loadbalancer.LoadBalancedRetryPolicyFactory;
import org.springframework.cloud.client.loadbalancer.ServiceInstanceChooser;
import org.springframework.http.HttpMethod;

/**
* @author Ryan Baxter
Expand All @@ -34,60 +31,10 @@ public RibbonLoadBalancedRetryPolicyFactory(SpringClientFactory clientFactory) {
}

@Override
public LoadBalancedRetryPolicy create(final String serviceId, final ServiceInstanceChooser loadBalanceChooser) {
final RibbonLoadBalancerContext lbContext = this.clientFactory
public LoadBalancedRetryPolicy create(String serviceId, ServiceInstanceChooser loadBalanceChooser) {
RibbonLoadBalancerContext lbContext = this.clientFactory
.getLoadBalancerContext(serviceId);
return new LoadBalancedRetryPolicy() {
private int sameServerCount = 0;
private int nextServerCount = 0;
private ServiceInstance lastServiceInstance = null;
public boolean canRetry(LoadBalancedRetryContext context) {
HttpMethod method = context.getRequest().getMethod();
return HttpMethod.GET == method || lbContext.isOkToRetryOnAllOperations();
}

@Override
public boolean canRetrySameServer(LoadBalancedRetryContext context) {
return sameServerCount < lbContext.getRetryHandler().getMaxRetriesOnSameServer() && canRetry(context);
}

@Override
public boolean canRetryNextServer(LoadBalancedRetryContext context) {
//this will be called after a failure occurs and we increment the counter
//so we check that the count is less than or equals to too make sure
//we try the next server the right number of times
return nextServerCount <= lbContext.getRetryHandler().getMaxRetriesOnNextServer() && canRetry(context);
}

@Override
public void close(LoadBalancedRetryContext context) {

}

@Override
public void registerThrowable(LoadBalancedRetryContext context, Throwable throwable) {
//Check if we need to ask the load balancer for a new server.
//Do this before we increment the counters because the first call to this method
//is not a retry it is just an initial failure.
if(!canRetrySameServer(context) && canRetryNextServer(context)) {
context.setServiceInstance(loadBalanceChooser.choose(serviceId));
}
//This method is called regardless of whether we are retrying or making the first request.
//Since we do not count the initial request in the retry count we don't reset the counter
//until we actually equal the same server count limit. This will allow us to make the initial
//request plus the right number of retries.
if(sameServerCount >= lbContext.getRetryHandler().getMaxRetriesOnSameServer() && canRetry(context)) {
//reset same server since we are moving to a new server
sameServerCount = 0;
nextServerCount++;
if(!canRetryNextServer(context)) {
context.setExhaustedOnly();
}
} else {
sameServerCount++;
}

}
};
return new RibbonLoadBalancedRetryPolicy(serviceId, lbContext, loadBalanceChooser, clientFactory.getClientConfig(serviceId));
}
}

Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.springframework.cloud.netflix.feign.ribbon.FeignRetryPolicy;
import org.springframework.cloud.netflix.ribbon.RibbonLoadBalancerClient;
import org.springframework.cloud.netflix.ribbon.ServerIntrospector;
import org.springframework.cloud.netflix.ribbon.support.RetryableStatusCodeException;
import org.springframework.http.HttpRequest;
import org.springframework.retry.RetryCallback;
import org.springframework.retry.RetryContext;
Expand Down Expand Up @@ -65,7 +66,8 @@ public RibbonApacheHttpResponse execute(final RibbonApacheHttpRequest request, f
CommonClientConfigKey.FollowRedirects, this.followRedirects));

final RequestConfig requestConfig = builder.build();
return this.executeWithRetry(request, new RetryCallback() {
final LoadBalancedRetryPolicy retryPolicy = loadBalancedRetryPolicyFactory.create(this.getClientName(), this);
RetryCallback retryCallback = new RetryCallback() {
@Override
public RibbonApacheHttpResponse doWithRetry(RetryContext context) throws Exception {
//on retries the policy will choose the server and set it in the context
Expand All @@ -88,13 +90,17 @@ public RibbonApacheHttpResponse doWithRetry(RetryContext context) throws Excepti
}
HttpUriRequest httpUriRequest = newRequest.toRequest(requestConfig);
final HttpResponse httpResponse = RetryableRibbonLoadBalancingHttpClient.this.delegate.execute(httpUriRequest);
if(retryPolicy.retryableStatusCode(httpResponse.getStatusLine().getStatusCode())) {
throw new RetryableStatusCodeException(RetryableRibbonLoadBalancingHttpClient.this.clientName,
httpResponse.getStatusLine().getStatusCode());
}
return new RibbonApacheHttpResponse(httpResponse, httpUriRequest.getURI());
}
});
};
return this.executeWithRetry(request, retryPolicy, retryCallback);
}

private RibbonApacheHttpResponse executeWithRetry(RibbonApacheHttpRequest request, RetryCallback<RibbonApacheHttpResponse, IOException> callback) throws Exception {
LoadBalancedRetryPolicy retryPolicy = loadBalancedRetryPolicyFactory.create(this.getClientName(), this);
private RibbonApacheHttpResponse executeWithRetry(RibbonApacheHttpRequest request, LoadBalancedRetryPolicy retryPolicy, RetryCallback<RibbonApacheHttpResponse, IOException> callback) throws Exception {
RetryTemplate retryTemplate = new RetryTemplate();
boolean retryable = request.getContext() == null ? true :
BooleanUtils.toBooleanDefaultIfNull(request.getContext().getRetryable(), true);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
import okhttp3.Request;
import okhttp3.Response;

import java.io.IOException;
import java.net.URI;
import org.apache.commons.lang.BooleanUtils;
import org.springframework.cloud.client.ServiceInstance;
Expand All @@ -30,6 +29,7 @@
import org.springframework.cloud.netflix.feign.ribbon.FeignRetryPolicy;
import org.springframework.cloud.netflix.ribbon.RibbonLoadBalancerClient;
import org.springframework.cloud.netflix.ribbon.ServerIntrospector;
import org.springframework.cloud.netflix.ribbon.support.RetryableStatusCodeException;
import org.springframework.http.HttpRequest;
import org.springframework.retry.RetryCallback;
import org.springframework.retry.RetryContext;
Expand All @@ -55,10 +55,9 @@ public RetryableOkHttpLoadBalancingClient(IClientConfig config, ServerIntrospect
this.loadBalancedRetryPolicyFactory = loadBalancedRetryPolicyFactory;
}

private OkHttpRibbonResponse executeWithRetry(OkHttpRibbonRequest request,
RetryCallback<OkHttpRibbonResponse, IOException> callback)
private OkHttpRibbonResponse executeWithRetry(OkHttpRibbonRequest request, LoadBalancedRetryPolicy retryPolicy,
RetryCallback<OkHttpRibbonResponse, Exception> callback)
throws Exception {
LoadBalancedRetryPolicy retryPolicy = loadBalancedRetryPolicyFactory.create(this.getClientName(), this);
RetryTemplate retryTemplate = new RetryTemplate();
boolean retryable = request.getContext() == null ? true :
BooleanUtils.toBooleanDefaultIfNull(request.getContext().getRetryable(), true);
Expand All @@ -70,7 +69,8 @@ private OkHttpRibbonResponse executeWithRetry(OkHttpRibbonRequest request,
@Override
public OkHttpRibbonResponse execute(final OkHttpRibbonRequest ribbonRequest,
final IClientConfig configOverride) throws Exception {
return this.executeWithRetry(ribbonRequest, new RetryCallback() {
final LoadBalancedRetryPolicy retryPolicy = loadBalancedRetryPolicyFactory.create(this.getClientName(), this);
RetryCallback<OkHttpRibbonResponse, Exception> retryCallback = new RetryCallback<OkHttpRibbonResponse, Exception>() {
@Override
public OkHttpRibbonResponse doWithRetry(RetryContext context) throws Exception {
//on retries the policy will choose the server and set it in the context
Expand All @@ -95,9 +95,13 @@ public OkHttpRibbonResponse doWithRetry(RetryContext context) throws Exception {

final Request request = newRequest.toRequest();
Response response = httpClient.newCall(request).execute();
if(retryPolicy.retryableStatusCode(response.code())) {
throw new RetryableStatusCodeException(RetryableOkHttpLoadBalancingClient.this.clientName, response.code());
}
return new OkHttpRibbonResponse(response, newRequest.getUri());
}
});
};
return this.executeWithRetry(ribbonRequest, retryPolicy, retryCallback);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
/*
*
* * Copyright 2013-2016 the original author or authors.
* *
* * Licensed under the Apache License, Version 2.0 (the "License");
* * you may not use this file except in compliance with the License.
* * You may obtain a copy of the License at
* *
* * http://www.apache.org/licenses/LICENSE-2.0
* *
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS,
* * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* * See the License for the specific language governing permissions and
* * limitations under the License.
*
*/

package org.springframework.cloud.netflix.ribbon.support;

import java.io.IOException;

/**
* Exception to be thrown when the status code is deemed to be retryable.
* @author Ryan Baxter
*/
public class RetryableStatusCodeException extends IOException {

private static final String MESSAGE = "Service %s returned a status code of %d";

public RetryableStatusCodeException(String serviceId, int statusCode) {
super(String.format(MESSAGE, serviceId, statusCode));
}
}
Loading

0 comments on commit 6290859

Please sign in to comment.