[JAV-127]retry的场景在自己的线程池执行

demo/demo-springmvc/springmvc-client/src/main/java/io/servicecomb/demo/springmvc/client/SpringmvcClient.java

@@ -49,6 +49,16 @@ public static void run() throws Exception {
         restTemplate = RestTemplateBuilder.create();
         controller = BeanUtils.getBean("controller");
 
+        String prefix = "cse://springmvc";
+
+        try {
+            // this test class is intended for rery hang issue JAV-27
+            restTemplate.getForObject(prefix + "/controller/sayhi?name=throwexception", String.class);
+            TestMgr.check("true", "false");
+        } catch (Exception e) {
+            TestMgr.check("true", "true");
+        }
+
         CodeFirstRestTemplateSpringmvc codeFirstClient =
             BeanUtils.getContext().getBean(CodeFirstRestTemplateSpringmvc.class);
         codeFirstClient.testCodeFirst(restTemplate, "springmvc", "/codeFirstSpringmvc/");

demo/demo-springmvc/springmvc-client/src/main/resources/microservice.yaml

@@ -13,6 +13,10 @@ cse:
   tracing:
     enabled: true
     samplingRate: 0.5
+  loadbalance:
+    retryEnabled: true
+    retryOnSame: 1
+    retryOnNext: 1
 
 #########SSL options
 ssl.protocols: TLSv1.2

demo/demo-springmvc/springmvc-server/src/main/java/io/servicecomb/demo/springmvc/server/ControllerImpl.java

@@ -52,6 +52,9 @@ public String saySomething(String prefix, @RequestBody Person user) {
     @RequestMapping(path = "/sayhi", method = RequestMethod.GET)
     public String sayHi(HttpServletRequest request) {
         String[] values = request.getParameterValues("name");
+        if (values != null && values.length > 0 && values[0].equals("throwexception")) {
+            throw new RuntimeException();
+        }
         return "hi " + request.getParameter("name") + " " + Arrays.toString(values);
     }
 

foundation-vertx/src/main/java/io/servicecomb/foundation/vertx/VertxUtils.java

@@ -79,6 +79,7 @@ public static <CLIENT_POOL, CLIENT_OPTIONS> DeploymentOptions createClientDeploy
         return options;
     }
 
+    // deploy Verticle and wait for its success. do not call this method in event-loop thread
     public static <VERTICLE extends AbstractVerticle> boolean blockDeploy(Vertx vertx,
             Class<VERTICLE> cls,
             DeploymentOptions options) throws InterruptedException {

handler-loadbalance/src/main/java/io/servicecomb/loadbalance/LoadbalanceHandler.java

@@ -21,6 +21,10 @@
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -53,6 +57,15 @@
 public class LoadbalanceHandler extends AbstractHandler {
     private static final Logger LOGGER = LoggerFactory.getLogger(LoadbalanceHandler.class);
 
+    private static final ExecutorService RETRY_POOL = Executors.newCachedThreadPool(new ThreadFactory() {
+        private AtomicInteger count = new AtomicInteger(0);
+
+        @Override
+        public Thread newThread(Runnable r) {
+            return new Thread(r, "retry-pool-thread-" + count.getAndIncrement());
+        }
+    });
+
     // 会给每个Microservice创建一个handler实例，因此这里的key为transportName，保证每个通道使用一个负载均衡策略
     private volatile Map<String, LoadBalancer> loadBalancerMap = new ConcurrentHashMap<>();
 
@@ -181,7 +194,9 @@ private void sendWithRetry(Invocation invocation, AsyncResponse asyncResp,
             newExecutor = new Executor() {
                 @Override
                 public void execute(Runnable command) {
-                    command.run(); // retry的场景，对于同步调用， 需要在网络线程中进行。同步调用的主线程已经被挂起，无法再主线程中进行重试。
+                    // retry的场景，对于同步调用, 同步调用的主线程已经被挂起，无法再主线程中进行重试;
+                    // 重试也不能在网络线程（event-loop）中进行，未被保护的阻塞操作会导致网络线程挂起
+                    RETRY_POOL.submit(command);
                 }
             };
             invocation.setResponseExecutor(newExecutor);