feat: 多线程百万数据测试

livk-cloud · Mar 4, 2023 · 7f7542c · 7f7542c
1 parent 7d2c59e
commit 7f7542c
Show file tree

Hide file tree

Showing 3 changed files with 60 additions and 2 deletions.
diff --git a/...-webmvc-easyexcel-example/src/main/java/com/livk/excel/mvc/controller/InfoController.java b/...-webmvc-easyexcel-example/src/main/java/com/livk/excel/mvc/controller/InfoController.java
@@ -1,16 +1,19 @@
 package com.livk.excel.mvc.controller;
 
+import com.google.common.collect.Lists;
 import com.livk.autoconfigure.easyexcel.annotation.ExcelImport;
 import com.livk.autoconfigure.easyexcel.annotation.ExcelParam;
 import com.livk.autoconfigure.easyexcel.annotation.ExcelReturn;
 import com.livk.excel.mvc.entity.Info;
 import com.livk.excel.mvc.listener.InfoExcelListener;
 import com.livk.excel.mvc.service.InfoService;
 import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.http.HttpEntity;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
 import java.util.List;
@@ -24,6 +27,7 @@
  *
  * @author livk
  */
+@Slf4j
 @RestController
 @RequiredArgsConstructor
 public class InfoController {
@@ -32,11 +36,27 @@ public class InfoController {
 
     @ExcelImport(parse = InfoExcelListener.class)
     @PostMapping("upload")
-    public HttpEntity<Boolean> upload(@ExcelParam List<Info> dataExcels) {
-        infoService.insertBatch(dataExcels);
+    public HttpEntity<Boolean> upload(@ExcelParam List<Info> dataExcels,
+                                      @RequestParam(defaultValue = "false") Boolean async,
+                                      @RequestParam(defaultValue = "1") Integer multiple) {
+        List<Info> infos = this.multiple(dataExcels, multiple);
+        log.info("size:{},async:{},multiple:{}", infos.size(), async, multiple);
+        if (async) {
+            infoService.insertBatchMultithreading(infos);
+        } else {
+            infoService.insertBatch(infos);
+        }
         return ResponseEntity.ok(Boolean.TRUE);
     }
 
+    private <T> List<T> multiple(List<T> list, int multiple) {
+        List<T> ts = Lists.newArrayList();
+        for (int i = 0; i < multiple; i++) {
+            ts.addAll(Lists.newArrayList(list));
+        }
+        return ts;
+    }
+
     @ExcelImport(parse = InfoExcelListener.class)
     @PostMapping("uploadList")
     public HttpEntity<List<Info>> uploadList(@ExcelParam List<Info> dataExcels) {

diff --git a/...spring-webmvc-easyexcel-example/src/main/java/com/livk/excel/mvc/service/InfoService.java b/...spring-webmvc-easyexcel-example/src/main/java/com/livk/excel/mvc/service/InfoService.java
@@ -15,4 +15,6 @@ public interface InfoService {
 
     void insertBatch(List<Info> records);
 
+    void insertBatchMultithreading(List<Info> records);
+
 }
diff --git a/...bmvc-easyexcel-example/src/main/java/com/livk/excel/mvc/service/impl/InfoServiceImpl.java b/...bmvc-easyexcel-example/src/main/java/com/livk/excel/mvc/service/impl/InfoServiceImpl.java
@@ -11,6 +11,8 @@
 import org.springframework.stereotype.Service;
 
 import java.util.List;
+import java.util.concurrent.*;
+import java.util.function.Function;
 
 /**
  * <p>
@@ -50,4 +52,38 @@ public void insertBatch(List<Info> records) {
         sqlSession.close();
     }
 
+    public void insertBatchMultithreading(List<Info> records) {
+        ThreadPoolExecutor executor = new ThreadPoolExecutor(10, 20,
+                0L, TimeUnit.MILLISECONDS,
+                new LinkedBlockingQueue<>());
+        SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH, false);
+        InfoMapper infoMapper = sqlSession.getMapper(InfoMapper.class);
+        List<List<Info>> partition = Lists.partition(records, (int) Math.pow(1000, 2));
+        List<Callable<Integer>> callables = partition.stream()
+                .map((Function<List<Info>, Callable<Integer>>) infos -> {
+                    List<List<Info>> lists = Lists.partition(infos, 1000);
+                    for (List<Info> list : lists) {
+                        try {
+                            infoMapper.insertBatch(list);
+                        } catch (Exception e) {
+                            return () -> 0;
+                        }
+                    }
+                    return () -> 1;
+                }).toList();
+        try {
+            List<Future<Integer>> futures = executor.invokeAll(callables);
+            for (Future<Integer> future : futures) {
+                if (future.get() <= 0) {
+                    sqlSession.rollback();
+                    break;
+                }
+            }
+        } catch (InterruptedException | ExecutionException e) {
+            throw new RuntimeException(e);
+        }
+        sqlSession.commit();
+        sqlSession.clearCache();
+        sqlSession.close();
+    }
 }