diff --git a/src/main/java/asia/asoulcnki/api/common/BizException.java b/src/main/java/asia/asoulcnki/api/common/BizException.java index 12e0b02..9ba4449 100644 --- a/src/main/java/asia/asoulcnki/api/common/BizException.java +++ b/src/main/java/asia/asoulcnki/api/common/BizException.java @@ -5,73 +5,74 @@ public class BizException extends RuntimeException { - private static final long serialVersionUID = 1L; - - /** - * 错误码 - */ - protected int errorCode; - /** - * 错误信息 - */ - protected String errorMsg; - - public BizException() { - super(); - } - - public BizException(BaseErrorInfoInterface errorInfoInterface) { - super(String.format("%d", errorInfoInterface.getResultCode())); - this.errorCode = errorInfoInterface.getResultCode(); - this.errorMsg = errorInfoInterface.getResultMsg(); - } - - public BizException(BaseErrorInfoInterface errorInfoInterface, Throwable cause) { - super(String.format("%d", errorInfoInterface.getResultCode())); - this.errorCode = errorInfoInterface.getResultCode(); - this.errorMsg = errorInfoInterface.getResultMsg(); - } - - public BizException(String errorMsg) { - super(errorMsg); - this.errorMsg = errorMsg; - } - - public BizException(int errorCode, String errorMsg) { - super(String.format("%d", errorCode)); - this.errorCode = errorCode; - this.errorMsg = errorMsg; - } - - public BizException(int errorCode, String errorMsg, Throwable cause) { - super(String.format("%d", errorCode), cause); - this.errorCode = errorCode; - this.errorMsg = errorMsg; - } - - public int getErrorCode() { - return errorCode; - } - - public void setErrorCode(int errorCode) { - this.errorCode = errorCode; - } - - public String getErrorMsg() { - return errorMsg; - } - - public void setErrorMsg(String errorMsg) { - this.errorMsg = errorMsg; - } - - public String getMessage() { - return errorMsg; - } - - @Override - public Throwable fillInStackTrace() { - return this; - } - -} \ No newline at end of file + private static final long serialVersionUID = 1L; + + /** + * 错误码 + */ + protected int errorCode; + /** + * 错误信息 + */ + protected String errorMsg; + + public BizException() { + super(); + } + + public BizException(BaseErrorInfoInterface errorInfoInterface) { + super(String.format("%d", errorInfoInterface.getResultCode())); + this.errorCode = errorInfoInterface.getResultCode(); + this.errorMsg = errorInfoInterface.getResultMsg(); + } + + public BizException(BaseErrorInfoInterface errorInfoInterface, Throwable cause) { + super(String.format("%d", errorInfoInterface.getResultCode())); + this.errorCode = errorInfoInterface.getResultCode(); + this.errorMsg = errorInfoInterface.getResultMsg(); + } + + public BizException(String errorMsg) { + super(errorMsg); + this.errorMsg = errorMsg; + } + + public BizException(int errorCode, String errorMsg) { + super(String.format("%d", errorCode)); + this.errorCode = errorCode; + this.errorMsg = errorMsg; + } + + public BizException(int errorCode, String errorMsg, Throwable cause) { + super(String.format("%d", errorCode), cause); + this.errorCode = errorCode; + this.errorMsg = errorMsg; + } + + public int getErrorCode() { + return errorCode; + } + + public void setErrorCode(int errorCode) { + this.errorCode = errorCode; + } + + public String getErrorMsg() { + return errorMsg; + } + + public void setErrorMsg(String errorMsg) { + this.errorMsg = errorMsg; + } + + @Override + public String getMessage() { + return errorMsg; + } + + @Override + public Throwable fillInStackTrace() { + return this; + } + +} diff --git a/src/main/java/asia/asoulcnki/api/common/GlobalExceptionHandler.java b/src/main/java/asia/asoulcnki/api/common/GlobalExceptionHandler.java index 89a0759..5cf8df4 100644 --- a/src/main/java/asia/asoulcnki/api/common/GlobalExceptionHandler.java +++ b/src/main/java/asia/asoulcnki/api/common/GlobalExceptionHandler.java @@ -14,43 +14,43 @@ @ControllerAdvice public class GlobalExceptionHandler { - private static final Logger logger = LoggerFactory.getLogger(GlobalExceptionHandler.class); + private static final Logger logger = LoggerFactory.getLogger(GlobalExceptionHandler.class); - /** - * 处理自定义的业务异常 - * - * @param req - * @param e - * @return - */ - @ExceptionHandler(value = BizException.class) - @ResponseBody - public ApiResult bizExceptionHandler(HttpServletRequest req, BizException e) { - logger.error("Business exception, caused by {} ", e.getErrorMsg()); - e.printStackTrace(); - return ApiResult.error(e.getErrorCode(), e.getErrorMsg()); - } + /** + * 处理自定义的业务异常 + * + * @param req HttpServletRequest 请求 + * @param e BizException 异常 + * @return ApiResult + */ + @ExceptionHandler(value = BizException.class) + @ResponseBody + public ApiResult bizExceptionHandler(HttpServletRequest req, BizException e) { + logger.error("Business exception, caused by {} ", e.getErrorMsg()); + e.printStackTrace(); + return ApiResult.error(e.getErrorCode(), e.getErrorMsg()); + } - /** - * 处理其他异常 - * - * @param req - * @param e - * @return - */ - @ExceptionHandler(value = Exception.class) - @ResponseBody - public ApiResult exceptionHandler(HttpServletRequest req, Exception e) { - int errorCode; - String errorMsg; - if (e instanceof ConstraintViolationException) { - errorCode = CnkiCommonEnum.INVALID_REQUEST.getResultCode(); - errorMsg = "invalid input param: " + e.getMessage(); - } else { - e.printStackTrace(); - errorCode = CnkiCommonEnum.INTERNAL_SERVER_ERROR.getResultCode(); - errorMsg = "internal server error : " + e.toString(); - } - return ApiResult.error(errorCode, errorMsg); - } -} \ No newline at end of file + /** + * 处理其他异常 + * + * @param req HttpServletRequest 请求 + * @param e BizException 异常 + * @return ApiResult + */ + @ExceptionHandler(value = Exception.class) + @ResponseBody + public ApiResult exceptionHandler(HttpServletRequest req, Exception e) { + int errorCode; + String errorMsg; + if (e instanceof ConstraintViolationException) { + errorCode = CnkiCommonEnum.INVALID_REQUEST.getResultCode(); + errorMsg = "invalid input param: " + e.getMessage(); + } else { + e.printStackTrace(); + errorCode = CnkiCommonEnum.INTERNAL_SERVER_ERROR.getResultCode(); + errorMsg = "internal server error : " + e; + } + return ApiResult.error(errorCode, errorMsg); + } +} diff --git a/src/main/java/asia/asoulcnki/api/common/config/CacheConfig.java b/src/main/java/asia/asoulcnki/api/common/config/CacheConfig.java index 86dee58..865b14b 100644 --- a/src/main/java/asia/asoulcnki/api/common/config/CacheConfig.java +++ b/src/main/java/asia/asoulcnki/api/common/config/CacheConfig.java @@ -10,19 +10,19 @@ @Configuration public class CacheConfig { - @Bean("checkCache") - public CacheManager cacheManager() { - CaffeineCacheManager cacheManager = new CaffeineCacheManager(); - cacheManager.setCaffeine(Caffeine.newBuilder() - // 设置最后一次写入或访问后经过固定时间过期 - .expireAfterAccess(60, TimeUnit.SECONDS) - // 初始的缓存空间大小 - .initialCapacity(1000) - // 缓存的最大条数 - .maximumSize(2000) - // 减少gc压力 - .weakValues()); - return cacheManager; - } + @Bean("checkCache") + public CacheManager cacheManager() { + CaffeineCacheManager cacheManager = new CaffeineCacheManager(); + cacheManager.setCaffeine(Caffeine.newBuilder() + // 设置最后一次写入或访问后经过固定时间过期 + .expireAfterAccess(60, TimeUnit.SECONDS) + // 初始的缓存空间大小 + .initialCapacity(1000) + // 缓存的最大条数 + .maximumSize(2000) + // 减少gc压力 + .weakValues()); + return cacheManager; + } -} \ No newline at end of file +} diff --git a/src/main/java/asia/asoulcnki/api/common/config/MybatisPlusConfig.java b/src/main/java/asia/asoulcnki/api/common/config/MybatisPlusConfig.java index 3ea15a9..320693e 100644 --- a/src/main/java/asia/asoulcnki/api/common/config/MybatisPlusConfig.java +++ b/src/main/java/asia/asoulcnki/api/common/config/MybatisPlusConfig.java @@ -8,19 +8,14 @@ import org.springframework.transaction.annotation.EnableTransactionManagement; @Configuration -//@MapperScan("asia.asoulcnki.api.persistence.mapper") @EnableTransactionManagement public class MybatisPlusConfig { - @Bean - public MybatisPlusInterceptor mybatisPlusInterceptor() { - MybatisPlusInterceptor mybatisPlusInterceptor = new MybatisPlusInterceptor(); - // 设置请求的页面大于最大页后操作， true调回到首页，false 继续请求 默认false - // paginationInterceptor.setOverflow(false); - // 设置最大单页限制数量，默认 500 条，-1 不受限制 - // paginationInterceptor.setLimit(500); - // 开启 count 的 join 优化,只针对部分 left join - mybatisPlusInterceptor.addInnerInterceptor(new PaginationInnerInterceptor(DbType.MYSQL)); - return mybatisPlusInterceptor; - } + @Bean + public MybatisPlusInterceptor mybatisPlusInterceptor() { + MybatisPlusInterceptor mybatisPlusInterceptor = new MybatisPlusInterceptor(); + // 开启 count 的 join 优化,只针对部分 left join + mybatisPlusInterceptor.addInnerInterceptor(new PaginationInnerInterceptor(DbType.MYSQL)); + return mybatisPlusInterceptor; + } } diff --git a/src/main/java/asia/asoulcnki/api/common/config/Swagger3Configuration.java b/src/main/java/asia/asoulcnki/api/common/config/Swagger3Configuration.java index 7c1dfc9..2da84ae 100644 --- a/src/main/java/asia/asoulcnki/api/common/config/Swagger3Configuration.java +++ b/src/main/java/asia/asoulcnki/api/common/config/Swagger3Configuration.java @@ -13,9 +13,7 @@ import springfox.documentation.spring.web.plugins.Docket; /** - * - Springfox-swagger 3.0.0配置 - + * Springfox-swagger 3.0.0配置 * * @author rmym * @version V1.0 @@ -47,30 +45,28 @@ public class Swagger3Configuration { private String applicationDescription; @Bean - public Docket createRestApi() - { + public Docket createRestApi() { return new Docket(DocumentationType.OAS_30).pathMapping("/") - // 定义是否开启swagger，false为关闭，可以通过变量控制 - .enable(enable) - // 将api的元信息设置为包含在json ResourceListing响应中。 - .apiInfo(apiInfo()) - // 选择哪些接口作为swagger的doc发布 - .select() - .apis(RequestHandlerSelectors.any()) - .paths(PathSelectors.any()) - .build(); + // 定义是否开启swagger，false为关闭，可以通过变量控制 + .enable(enable) + // 将api的元信息设置为包含在json ResourceListing响应中。 + .apiInfo(apiInfo()) + // 选择哪些接口作为swagger的doc发布 + .select() + .apis(RequestHandlerSelectors.any()) + .paths(PathSelectors.any()) + .build(); } /** * API 页面上半部分展示信息 */ - private ApiInfo apiInfo() - { + private ApiInfo apiInfo() { return new ApiInfoBuilder().title(applicationName + " Api文档") - .description(applicationDescription) - .contact(new Contact("一个魂捏", null, "")) - .version("应用版本号: " + applicationVersion + ", Spring Boot 版本号: " + SpringBootVersion.getVersion()) - .build(); + .description(applicationDescription) + .contact(new Contact("一个魂捏", null, "")) + .version("应用版本号: " + applicationVersion + ", Spring Boot 版本号: " + SpringBootVersion.getVersion()) + .build(); } diff --git a/src/main/java/asia/asoulcnki/api/common/config/WebMvcConfig.java b/src/main/java/asia/asoulcnki/api/common/config/WebMvcConfig.java index cdb223b..0b640db 100644 --- a/src/main/java/asia/asoulcnki/api/common/config/WebMvcConfig.java +++ b/src/main/java/asia/asoulcnki/api/common/config/WebMvcConfig.java @@ -9,30 +9,33 @@ @Configuration public class WebMvcConfig implements WebMvcConfigurer { - //实例化对象 - @Bean - public HandlerInterceptor getInterceptor() { - return new DefaultInterceptor(); - } + /** + * 实例化对象 + */ + @Bean + public HandlerInterceptor getInterceptor() { + return new DefaultInterceptor(); + } - @Override - public void addInterceptors(InterceptorRegistry registry) { - registry.addInterceptor(getInterceptor()) - .excludePathPatterns("/swagger**/**") - .excludePathPatterns("/webjars/**") - .excludePathPatterns("/v3/**") - .excludePathPatterns("/doc.html"); + @Override + public void addInterceptors(InterceptorRegistry registry) { + registry.addInterceptor(getInterceptor()) + .excludePathPatterns("/swagger**/**") + .excludePathPatterns("/webjars/**") + .excludePathPatterns("/v3/**") + .excludePathPatterns("/doc.html"); - } + } - @Override - public void addCorsMappings(CorsRegistry registry) { - registry.addMapping("/**"). - allowedOrigins("*"). - allowedMethods("GET", "POST", "DELETE", "PUT", "OPTIONS"). - allowCredentials(false).maxAge(3600); - } + @Override + public void addCorsMappings(CorsRegistry registry) { + registry.addMapping("/**"). + allowedOrigins("*"). + allowedMethods("GET", "POST", "DELETE", "PUT", "OPTIONS"). + allowCredentials(false).maxAge(3600); + } } + class DefaultInterceptor implements HandlerInterceptor { -} \ No newline at end of file +} diff --git a/src/main/java/asia/asoulcnki/api/common/duplicationcheck/ArticleCompareUtil.java b/src/main/java/asia/asoulcnki/api/common/duplicationcheck/ArticleCompareUtil.java index f5132cc..3acb095 100644 --- a/src/main/java/asia/asoulcnki/api/common/duplicationcheck/ArticleCompareUtil.java +++ b/src/main/java/asia/asoulcnki/api/common/duplicationcheck/ArticleCompareUtil.java @@ -8,77 +8,109 @@ import java.util.List; import java.util.Set; +/** + * 小作文比对工具类 + */ public class ArticleCompareUtil { - public static boolean isHighSimilarity(int textLength, float similarity) { - if (textLength > 250) { - return similarity > 0.5; - } else if (textLength > 150) { - return similarity > 0.6; - } else { - return similarity > 0.7; - } - } + /** + * 判断小作文是否高度相似 + * + * @param textLength 小作文长度 + * @param similarity 相似度 + * @return + */ + public static boolean isHighSimilarity(int textLength, float similarity) { + if (textLength > 250) { + return similarity > 0.5; + } else if (textLength > 150) { + return similarity > 0.6; + } else { + return similarity > 0.7; + } + } - public static String trim(String s) { - String stopWord = "[\\pP\\p{Punct}]"; - s = s.replaceAll("\\s*", ""); - s = s.replaceAll("\\p{Cf}",""); //去除控制字符 - s = s.replaceAll("/[\\u200b-\\u200f\\ufeff\\u202a-\\u202e]/g",""); //去除零宽空格等 - s = s.replaceAll(stopWord, ""); - return s; - } + /** + * 过滤空白字符、标点符号、零宽空格等特殊字符 + * + * @param s 小作文 + * @return 经过过滤后的小作文 + */ + public static String trim(String s) { + String stopWord = "[\\pP\\p{Punct}]"; + s = s.replaceAll("\\s*", ""); + //去除控制字符 + s = s.replaceAll("\\p{Cf}", ""); + //去除零宽空格等 + s = s.replaceAll("/[\\u200b-\\u200f\\ufeff\\u202a-\\u202e]/g", ""); + s = s.replaceAll(stopWord, ""); + return s; + } - static List getStringSegs(String s) { - int codePointCount = s.codePointCount(0, s.length()); - if (codePointCount <= SummaryHash.DEFAULT_K) { - return Lists.newArrayList(s); - } - int startOffset = 0; - List stringSegs = new ArrayList<>(codePointCount - SummaryHash.DEFAULT_K + 1); - for (int i = 0; i < codePointCount - SummaryHash.DEFAULT_K + 1; i++) { - String subString = unicodeSubString(s, startOffset, SummaryHash.DEFAULT_K); - startOffset = s.offsetByCodePoints(startOffset, 1); - stringSegs.add(subString); - } - return stringSegs; - } + /** + * 使用滑动窗口将字符串分段，以便 compareArticle 方法比较 + * + * @param s 小作文 + * @return + */ + static List getStringSegs(String s) { + int codePointCount = s.codePointCount(0, s.length()); + if (codePointCount <= SummaryHash.DEFAULT_K) { + return Lists.newArrayList(s); + } + int startOffset = 0; + List stringSegs = new ArrayList<>(codePointCount - SummaryHash.DEFAULT_K + 1); + for (int i = 0; i < codePointCount - SummaryHash.DEFAULT_K + 1; i++) { + String subString = unicodeSubString(s, startOffset, SummaryHash.DEFAULT_K); + startOffset = s.offsetByCodePoints(startOffset, 1); + stringSegs.add(subString); + } + return stringSegs; + } - static String unicodeSubString(String str, int idx, int len) { - return str.substring(idx, str.offsetByCodePoints(idx, len)); - } + static String unicodeSubString(String str, int idx, int len) { + return str.substring(idx, str.offsetByCodePoints(idx, len)); + } - public static float compareArticle(String article1, String article2) { - int codePointsCount = article1.codePointCount(0, article1.length()); - List article1Segs = getStringSegs(article1); - Set article2Segs = new HashSet<>(getStringSegs(article2)); - float count = 0; - Set redList = new HashSet<>(); - for (int i = 0; i < codePointsCount - SummaryHash.DEFAULT_K + 1; i++) { - String seg = article1Segs.get(i); - if (article2Segs.contains(seg)) { - for (int j = 0; j < SummaryHash.DEFAULT_K; j++) { - redList.add(i + j); - } - } - } - for (int i = 0; i < codePointsCount; i++) { - if (redList.contains(i)) { - count += 1; - } - } + /** + * 获取两篇小作文的文本重复度 + * + * @param article1 + * @param article2 + * @return 文本重复率 + */ + public static float compareArticle(String article1, String article2) { + int codePointsCount = article1.codePointCount(0, article1.length()); + List article1Segs = getStringSegs(article1); + Set article2Segs = new HashSet<>(getStringSegs(article2)); + float count = 0; + Set redList = new HashSet<>(); + for (int i = 0; i < codePointsCount - SummaryHash.DEFAULT_K + 1; i++) { + String seg = article1Segs.get(i); + if (article2Segs.contains(seg)) { + for (int j = 0; j < SummaryHash.DEFAULT_K; j++) { + redList.add(i + j); + } + } + } + for (int i = 0; i < codePointsCount; i++) { + if (redList.contains(i)) { + count += 1; + } + } - return count / (float) codePointsCount; - } + return count / (float) codePointsCount; + } - public static int textLength(String s) { - if (StringUtils.isBlank(s)) { - return 0; - } - return s.codePointCount(0, s.length()); - } - - public static void main(String[] args) { - String s = "7月9日，认识了一个男人，他跟我聊叔本华，聊弗洛伊，聊庄子妻死，聊伽罗瓦和近世代数，聊彭罗斯和宇宙督察假说\n" + "\n" + "7月10日，我让他b站关注嘉然今天吃什么"; - System.out.println(textLength(trim(s))); - } + /** + * 获取小作文字数 + * + * @param s 小作文字符串 + * @return 作文字数 + */ + public static int textLength(String s) { + if (StringUtils.isBlank(s)) { + return 0; + } + return s.codePointCount(0, s.length()); + } } diff --git a/src/main/java/asia/asoulcnki/api/common/duplicationcheck/LeaderBoard.java b/src/main/java/asia/asoulcnki/api/common/duplicationcheck/LeaderBoard.java index 287b5d9..c9e801f 100644 --- a/src/main/java/asia/asoulcnki/api/common/duplicationcheck/LeaderBoard.java +++ b/src/main/java/asia/asoulcnki/api/common/duplicationcheck/LeaderBoard.java @@ -10,7 +10,6 @@ import java.util.Comparator; import java.util.Date; import java.util.List; -import java.util.TimeZone; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Predicate; @@ -18,187 +17,223 @@ public class LeaderBoard { - private final static Logger log = LoggerFactory.getLogger(LeaderBoard.class); - - private static LeaderBoard instance; - // 累积赞数排序，至少为50，且引用次数至少为1 - private final LeaderBoardEntry similarLikeSumLeaderboard; - // 单评论赞数排序，至少为50 - private final LeaderBoardEntry likeLeaderBoard; - // 引用次数排序，至少为5 - private final LeaderBoardEntry similarCountLeaderBoard; - - private LeaderBoard() { - - similarLikeSumLeaderboard = new LeaderBoardEntry(Comparator.comparing(Reply::getSimilarLikeSum).reversed()); - similarLikeSumLeaderboard.setAllRepliesFilter(CommonFilterRules.similarLikeSumGreaterThan(50).and(CommonFilterRules.similarLikeCountGreaterThan(1))); - similarLikeSumLeaderboard.setRepliesInOneWeekFilter(CommonFilterRules.similarLikeSumGreaterThan(30).and(CommonFilterRules.similarLikeCountGreaterThan(0))); - similarLikeSumLeaderboard.setRepliesInThreeDaysFilter(CommonFilterRules.similarLikeSumGreaterThan(10).and(CommonFilterRules.similarLikeCountGreaterThan(0))); - - likeLeaderBoard = new LeaderBoardEntry(Comparator.comparing(Reply::getLikeNum).reversed()); - likeLeaderBoard.setAllRepliesFilter(CommonFilterRules.likeNumGreaterThan(100)); - likeLeaderBoard.setRepliesInOneWeekFilter(CommonFilterRules.likeNumGreaterThan(80)); - likeLeaderBoard.setRepliesInThreeDaysFilter(CommonFilterRules.likeNumGreaterThan(50)); - - similarCountLeaderBoard = new LeaderBoardEntry(Comparator.comparing(Reply::getSimilarCount).reversed()); - similarCountLeaderBoard.setAllRepliesFilter(CommonFilterRules.similarLikeCountGreaterThan(5)); - similarCountLeaderBoard.setRepliesInOneWeekFilter(CommonFilterRules.similarLikeCountGreaterThan(1)); - similarCountLeaderBoard.setRepliesInThreeDaysFilter(CommonFilterRules.similarLikeCountGreaterThan(0)); - - refresh(); - } - - public static synchronized LeaderBoard getInstance() { - if (instance == null) { - synchronized (LeaderBoard.class) { - if (instance == null) { - instance = new LeaderBoard(); - } - } - } - return instance; - } - - public static List page(List list, Integer pageSize, Integer pageNum) { - // now we only support page size as 10 - if (list == null || list.isEmpty() || pageSize != 10) { - return null; - } - - int recordCount = list.size(); - int pageCount = recordCount % pageSize == 0 ? recordCount / pageSize : recordCount / pageSize + 1; - - int fromIndex; //开始索引 - int toIndex; //结束索引 - - if (pageNum > pageCount) { - pageNum = pageCount; - } - - if (!pageNum.equals(pageCount)) { - fromIndex = (pageNum - 1) * pageSize; - toIndex = fromIndex + pageSize; - } else { - fromIndex = (pageNum - 1) * pageSize; - toIndex = recordCount; - } - - return list.subList(fromIndex, toIndex); - } - - public static void main(String[] args) { - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("GMT+8")); - calendar.set(Calendar.HOUR_OF_DAY, 0); - calendar.set(Calendar.MINUTE, 0); - calendar.set(Calendar.SECOND, 0); - long result = calendar.getTimeInMillis(); - System.out.println(result / 1000); - } - - public LeaderBoardEntry getSimilarLikeSumLeaderboard() { - return similarLikeSumLeaderboard; - } - - public LeaderBoardEntry getLikeLeaderBoard() { - return likeLeaderBoard; - } - - public LeaderBoardEntry getSimilarCountLeaderBoard() { - return similarCountLeaderBoard; - } - - public void refresh() { - similarCountLeaderBoard.refresh(); - likeLeaderBoard.refresh(); - similarLikeSumLeaderboard.refresh(); - } - - public static class LeaderBoardEntry { - private final Comparator comparator; - private final ReadWriteLock rwLock = new ReentrantReadWriteLock(); - List allReplies; - List repliesInOneWeek; - List repliesInThreeDays; - private Predicate allRepliesFilter; - private Predicate repliesInOneWeekFilter; - private Predicate repliesInThreeDaysFilter; - - private LeaderBoardEntry(final Comparator comparator) { - this.comparator = comparator; - } - - - public void setAllRepliesFilter(final Predicate allRepliesFilter) { - this.allRepliesFilter = allRepliesFilter; - } - - public void setRepliesInThreeDaysFilter(final Predicate repliesInThreeDaysFilter) { - this.repliesInThreeDaysFilter = repliesInThreeDaysFilter; - } - - public void setRepliesInOneWeekFilter(final Predicate repliesInOneWeekFilter) { - this.repliesInOneWeekFilter = repliesInOneWeekFilter; - } - - void refresh() { - ComparisonDatabase.getInstance().readLock(); - rwLock.writeLock().lock(); - try { - - allReplies = ComparisonDatabase.getInstance().getReplyMap().values().stream(). // - filter(allRepliesFilter).sorted(comparator).collect(Collectors.toList()); - - Calendar c = Calendar.getInstance(); - Date maxTime = new Date(ComparisonDatabase.getInstance().getMaxTime() * 1000L); - c.setTime(maxTime); - c.add(Calendar.DATE, -7); - Predicate timePredicate = r -> r.getCtime() * 1000L >= c.getTime().getTime(); - repliesInOneWeek = ComparisonDatabase.getInstance().getReplyMap().values().stream(). // - filter(repliesInOneWeekFilter.and(timePredicate)).sorted(comparator).collect(Collectors.toList()); - - c.setTime(maxTime); - c.add(Calendar.DATE, -3); - timePredicate = r -> r.getCtime() * 1000L >= c.getTime().getTime(); - repliesInThreeDays = ComparisonDatabase.getInstance().getReplyMap().values().stream(). // - filter(repliesInThreeDaysFilter.and(timePredicate)).sorted(comparator).collect(Collectors.toList()); - } finally { - ComparisonDatabase.getInstance().readUnLock(); - rwLock.writeLock().unlock(); - } - } - - public RankingResultVo query(Predicate filter, TimeRangeEnum timeRange, int pageSize, int pageNum) { - ComparisonDatabase.getInstance().readLock(); - rwLock.readLock().lock(); - try { - if (pageSize == 0 || pageNum < 1) { - return new RankingResultVo(); - } - - // set time filter - List targetReplySource = allReplies; - switch (timeRange) { - case ONE_WEEK: - targetReplySource = repliesInOneWeek; - break; - case THREE_DAYS: - targetReplySource = repliesInThreeDays; - break; - default: - break; - } - - List result = targetReplySource.stream().filter(filter).collect(Collectors.toList()); - - int minTime = ComparisonDatabase.getInstance().getMinTime(); - int maxTime = ComparisonDatabase.getInstance().getMaxTime(); - - return new RankingResultVo(page(result, pageSize, pageNum), result.size(), minTime, - maxTime); - } finally { - ComparisonDatabase.getInstance().readUnLock(); - rwLock.readLock().unlock(); - } - } - } + private final static Logger log = LoggerFactory.getLogger(LeaderBoard.class); + + private static volatile LeaderBoard instance; + /** + * 按累积赞数降序排序 + */ + private final LeaderBoardEntry similarLikeSumLeaderboard; + /** + * 按单评论赞数降序排序 + */ + private final LeaderBoardEntry likeLeaderBoard; + /** + * 按引用次数降序排序 + */ + private final LeaderBoardEntry similarCountLeaderBoard; + + /** + * 单页最大元素数 + */ + private final static int MAX_PAGE_SIZE = 20; + + private LeaderBoard() { + /* + * 累积赞数要求 + * 1. 在所有的评论中，只筛选累积赞数大于50，且引用次数大于1的 + * 2. 在近7天的评论中，只筛选累积赞数大于30，且引用次数大于0的 + * 3. 在近3天的评论中，只筛选累积赞数大于10，且引用次数大于0的 + */ + similarLikeSumLeaderboard = new LeaderBoardEntry(Comparator.comparing(Reply::getSimilarLikeSum).reversed()); + similarLikeSumLeaderboard.setAllRepliesFilter(CommonFilterRules.similarLikeSumGreaterThan(50).and(CommonFilterRules.similarLikeCountGreaterThan(1))); + similarLikeSumLeaderboard.setRepliesInOneWeekFilter(CommonFilterRules.similarLikeSumGreaterThan(30).and(CommonFilterRules.similarLikeCountGreaterThan(0))); + similarLikeSumLeaderboard.setRepliesInThreeDaysFilter(CommonFilterRules.similarLikeSumGreaterThan(10).and(CommonFilterRules.similarLikeCountGreaterThan(0))); + + /* + * 单点赞数要求 + * 1. 在所有的评论中，只筛选单点赞数大于100的 + * 2. 在近7天的评论中，只筛选单点赞数大于80的 + * 3. 在近3天的评论中，只筛选单点赞数大于50的 + */ + likeLeaderBoard = new LeaderBoardEntry(Comparator.comparing(Reply::getLikeNum).reversed()); + likeLeaderBoard.setAllRepliesFilter(CommonFilterRules.likeNumGreaterThan(100)); + likeLeaderBoard.setRepliesInOneWeekFilter(CommonFilterRules.likeNumGreaterThan(80)); + likeLeaderBoard.setRepliesInThreeDaysFilter(CommonFilterRules.likeNumGreaterThan(50)); + + /* + * 引用次数要求 + * 1. 在所有的评论中，只筛选引用次数大于5的 + * 2. 在近7天的评论中，只筛选引用次数大于1的 + * 3. 在近3天的评论中，只筛选应用次数大于0的 + */ + similarCountLeaderBoard = new LeaderBoardEntry(Comparator.comparing(Reply::getSimilarCount).reversed()); + similarCountLeaderBoard.setAllRepliesFilter(CommonFilterRules.similarLikeCountGreaterThan(5)); + similarCountLeaderBoard.setRepliesInOneWeekFilter(CommonFilterRules.similarLikeCountGreaterThan(1)); + similarCountLeaderBoard.setRepliesInThreeDaysFilter(CommonFilterRules.similarLikeCountGreaterThan(0)); + + // 刷新以获取最新 + refresh(); + } + + public static synchronized LeaderBoard getInstance() { + if (instance == null) { + synchronized (LeaderBoard.class) { + if (instance == null) { + instance = new LeaderBoard(); + } + } + } + return instance; + } + + /** + * 分页 + * + * @param list 列表 + * @param pageSize 单页大小 + * @param pageNum 页数 + * @param 列表泛型类型 + * @return + */ + public static List page(List list, Integer pageSize, Integer pageNum) { + if (list == null || list.isEmpty() || pageSize <= 0) { + return null; + } + + if (pageSize > MAX_PAGE_SIZE) { + pageSize = MAX_PAGE_SIZE; + } + + int recordCount = list.size(); + int pageCount = recordCount % pageSize == 0 ? recordCount / pageSize : recordCount / pageSize + 1; + //开始索引 + int fromIndex; + //结束索引 + int toIndex; + + if (pageNum > pageCount) { + pageNum = pageCount; + } + + if (!pageNum.equals(pageCount)) { + fromIndex = (pageNum - 1) * pageSize; + toIndex = fromIndex + pageSize; + } else { + fromIndex = (pageNum - 1) * pageSize; + toIndex = recordCount; + } + + return list.subList(fromIndex, toIndex); + } + + public LeaderBoardEntry getSimilarLikeSumLeaderboard() { + return similarLikeSumLeaderboard; + } + + public LeaderBoardEntry getLikeLeaderBoard() { + return likeLeaderBoard; + } + + public LeaderBoardEntry getSimilarCountLeaderBoard() { + return similarCountLeaderBoard; + } + + public void refresh() { + similarCountLeaderBoard.refresh(); + likeLeaderBoard.refresh(); + similarLikeSumLeaderboard.refresh(); + } + + public static class LeaderBoardEntry { + private final Comparator comparator; + private final ReadWriteLock rwLock = new ReentrantReadWriteLock(); + List allReplies; + List repliesInOneWeek; + List repliesInThreeDays; + private Predicate allRepliesFilter; + private Predicate repliesInOneWeekFilter; + private Predicate repliesInThreeDaysFilter; + + private LeaderBoardEntry(final Comparator comparator) { + this.comparator = comparator; + } + + + public void setAllRepliesFilter(final Predicate allRepliesFilter) { + this.allRepliesFilter = allRepliesFilter; + } + + public void setRepliesInThreeDaysFilter(final Predicate repliesInThreeDaysFilter) { + this.repliesInThreeDaysFilter = repliesInThreeDaysFilter; + } + + public void setRepliesInOneWeekFilter(final Predicate repliesInOneWeekFilter) { + this.repliesInOneWeekFilter = repliesInOneWeekFilter; + } + + void refresh() { + ComparisonDatabase.getInstance().readLock(); + rwLock.writeLock().lock(); + try { + // 刷新全部 + allReplies = ComparisonDatabase.getInstance().getReplyMap().values().stream(). // + filter(allRepliesFilter).sorted(comparator).collect(Collectors.toList()); + + // 刷新7天内 + Calendar c = Calendar.getInstance(); + Date maxTime = new Date(ComparisonDatabase.getInstance().getMaxTime() * 1000L); + c.setTime(maxTime); + c.add(Calendar.DATE, -7); + Predicate timePredicate = r -> r.getCtime() * 1000L >= c.getTime().getTime(); + repliesInOneWeek = ComparisonDatabase.getInstance().getReplyMap().values().stream(). // + filter(repliesInOneWeekFilter.and(timePredicate)).sorted(comparator).collect(Collectors.toList()); + + // 刷新3天内 + c.setTime(maxTime); + c.add(Calendar.DATE, -3); + timePredicate = r -> r.getCtime() * 1000L >= c.getTime().getTime(); + repliesInThreeDays = ComparisonDatabase.getInstance().getReplyMap().values().stream(). // + filter(repliesInThreeDaysFilter.and(timePredicate)).sorted(comparator).collect(Collectors.toList()); + } finally { + ComparisonDatabase.getInstance().readUnLock(); + rwLock.writeLock().unlock(); + } + } + + // 返回符合过滤条件的特定页小作文VO + public RankingResultVo query(Predicate filter, TimeRangeEnum timeRange, int pageSize, int pageNum) { + ComparisonDatabase.getInstance().readLock(); + rwLock.readLock().lock(); + try { + if (pageSize == 0 || pageNum < 1) { + return new RankingResultVo(); + } + + // set time filter + List targetReplySource = allReplies; + switch (timeRange) { + case ONE_WEEK: + targetReplySource = repliesInOneWeek; + break; + case THREE_DAYS: + targetReplySource = repliesInThreeDays; + break; + default: + break; + } + + List result = targetReplySource.stream().filter(filter).collect(Collectors.toList()); + + int minTime = ComparisonDatabase.getInstance().getMinTime(); + int maxTime = ComparisonDatabase.getInstance().getMaxTime(); + + return new RankingResultVo(page(result, pageSize, pageNum), result.size(), minTime, + maxTime); + } finally { + ComparisonDatabase.getInstance().readUnLock(); + rwLock.readLock().unlock(); + } + } + } } diff --git a/src/main/java/asia/asoulcnki/api/common/response/ApiResult.java b/src/main/java/asia/asoulcnki/api/common/response/ApiResult.java index 16a2129..d89a540 100644 --- a/src/main/java/asia/asoulcnki/api/common/response/ApiResult.java +++ b/src/main/java/asia/asoulcnki/api/common/response/ApiResult.java @@ -4,119 +4,119 @@ public class ApiResult { - /** - * 响应代码 - */ - private int code; - - /** - * 响应消息 - */ - private String message; - - /** - * 响应结果 - */ - private T data; - - public ApiResult() { - } - - public ApiResult(BaseErrorInfoInterface errorInfo) { - this.code = errorInfo.getResultCode(); - this.message = errorInfo.getResultMsg(); - } - - /** - * 成功 - * - * @return - */ - public static ApiResult ok() { - return ok(null); - } - - /** - * 成功 - * - * @param data - * @return - */ - public static ApiResult ok(T data) { - ApiResult rb = new ApiResult<>(); - rb.setCode(CnkiCommonEnum.SUCCESS.getResultCode()); - rb.setMessage(CnkiCommonEnum.SUCCESS.getResultMsg()); - rb.setData(data); - return rb; - } - - /** - * 失败 - */ - public static ApiResult error(BaseErrorInfoInterface errorInfo) { - ApiResult rb = new ApiResult<>(); - rb.setCode(errorInfo.getResultCode()); - rb.setMessage(errorInfo.getResultMsg()); - rb.setData(null); - return rb; - } - - /** - * 失败 - */ - public static ApiResult error(int code, String message) { - ApiResult rb = new ApiResult<>(); - rb.setCode(code); - rb.setMessage(message); - rb.setData(null); - return rb; - } - - public static ApiResult error(int code) { - ApiResult rb = new ApiResult<>(); - rb.setCode(code); - rb.setData(null); - return rb; - } - - /** - * 失败 - */ - public static ApiResult error(String message) { - ApiResult rb = new ApiResult<>(); - rb.setCode(-1); - rb.setMessage(message); - rb.setData(null); - return rb; - } - - public int getCode() { - return code; - } - - public void setCode(int code) { - this.code = code; - } - - public String getMessage() { - return message; - } - - public void setMessage(String message) { - this.message = message; - } - - public T getData() { - return data; - } - - public void setData(T data) { - this.data = data; - } - - @Override - public String toString() { - return JSONObject.toJSONString(this); - } - -} \ No newline at end of file + /** + * 响应代码 + */ + private int code; + + /** + * 响应消息 + */ + private String message; + + /** + * 响应结果 + */ + private T data; + + public ApiResult() { + } + + public ApiResult(BaseErrorInfoInterface errorInfo) { + this.code = errorInfo.getResultCode(); + this.message = errorInfo.getResultMsg(); + } + + /** + * 成功 + * + * @return + */ + public static ApiResult ok() { + return ok(null); + } + + /** + * 成功 + * + * @param data + * @return + */ + public static ApiResult ok(T data) { + ApiResult rb = new ApiResult<>(); + rb.setCode(CnkiCommonEnum.SUCCESS.getResultCode()); + rb.setMessage(CnkiCommonEnum.SUCCESS.getResultMsg()); + rb.setData(data); + return rb; + } + + /** + * 失败 + */ + public static ApiResult error(BaseErrorInfoInterface errorInfo) { + ApiResult rb = new ApiResult<>(); + rb.setCode(errorInfo.getResultCode()); + rb.setMessage(errorInfo.getResultMsg()); + rb.setData(null); + return rb; + } + + /** + * 失败 + */ + public static ApiResult error(int code, String message) { + ApiResult rb = new ApiResult<>(); + rb.setCode(code); + rb.setMessage(message); + rb.setData(null); + return rb; + } + + public static ApiResult error(int code) { + ApiResult rb = new ApiResult<>(); + rb.setCode(code); + rb.setData(null); + return rb; + } + + /** + * 失败 + */ + public static ApiResult error(String message) { + ApiResult rb = new ApiResult<>(); + rb.setCode(-1); + rb.setMessage(message); + rb.setData(null); + return rb; + } + + public int getCode() { + return code; + } + + public void setCode(int code) { + this.code = code; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public T getData() { + return data; + } + + public void setData(T data) { + this.data = data; + } + + @Override + public String toString() { + return JSONObject.toJSONString(this); + } + +} diff --git a/src/main/java/asia/asoulcnki/api/common/response/BaseErrorInfoInterface.java b/src/main/java/asia/asoulcnki/api/common/response/BaseErrorInfoInterface.java index 3a4b528..a3f8eb0 100644 --- a/src/main/java/asia/asoulcnki/api/common/response/BaseErrorInfoInterface.java +++ b/src/main/java/asia/asoulcnki/api/common/response/BaseErrorInfoInterface.java @@ -2,8 +2,10 @@ public interface BaseErrorInfoInterface extends BaseErrorInterface { - /** - * 错误描述 - */ - String getResultMsg(); + /** + * 获取报错信息 + * + * @return String 报错信息 + */ + String getResultMsg(); } diff --git a/src/main/java/asia/asoulcnki/api/common/response/BaseErrorInterface.java b/src/main/java/asia/asoulcnki/api/common/response/BaseErrorInterface.java index 6d901c8..f32cfcc 100644 --- a/src/main/java/asia/asoulcnki/api/common/response/BaseErrorInterface.java +++ b/src/main/java/asia/asoulcnki/api/common/response/BaseErrorInterface.java @@ -2,8 +2,10 @@ public interface BaseErrorInterface { - /** - * 错误码 - */ - int getResultCode(); + /** + * 获取错误码 + * + * @return int 错误码 + */ + int getResultCode(); } diff --git a/src/main/java/asia/asoulcnki/api/common/response/CnkiCommonEnum.java b/src/main/java/asia/asoulcnki/api/common/response/CnkiCommonEnum.java index 39a7d78..445007d 100644 --- a/src/main/java/asia/asoulcnki/api/common/response/CnkiCommonEnum.java +++ b/src/main/java/asia/asoulcnki/api/common/response/CnkiCommonEnum.java @@ -1,41 +1,43 @@ package asia.asoulcnki.api.common.response; public enum CnkiCommonEnum implements BaseErrorInfoInterface { - // 数据操作错误定义 - SUCCESS(0, "success"), INVALID_REQUEST(400, "bad request"), BODY_NOT_MATCH(400, "invalid request format"), - SIGNATURE_NOT_MATCH(401, "signature not match"), NOT_FOUND(404, "resource not found"), INTERNAL_SERVER_ERROR(500, - "internal server error!"), SERVER_BUSY(503, "server is under strong traffic, please retry later"), - NO_TOKEN(401, "No jwt token presents"), AUTH(401, "JWT AUTH ERROR"), AUTH_USER_NOT_FOUND(401, "JWT AUTH ERROR"), - - TEXT_TO_CHECK_TOO_LONG(20001, "text to check too long"), - TEXT_TO_CHECK_TOO_SHORT(20002, "text to check too short"), - - // END - ; - - /** - * 错误码 - */ - private int resultCode; - - /** - * 错误描述 - */ - private String resultMsg; - - CnkiCommonEnum(int resultCode, String resultMsg) { - this.resultCode = resultCode; - this.resultMsg = resultMsg; - } - - @Override - public int getResultCode() { - return resultCode; - } - - @Override - public String getResultMsg() { - return resultMsg; - } - -} \ No newline at end of file + // 数据操作错误定义 + SUCCESS(0, "success"), + INVALID_REQUEST(400, "bad request"), + BODY_NOT_MATCH(400, "invalid request format"), + SIGNATURE_NOT_MATCH(401, "signature not match"), + NOT_FOUND(404, "resource not found"), + INTERNAL_SERVER_ERROR(500, "internal server error!"), + SERVER_BUSY(503, "server is under strong traffic, please retry later"), + NO_TOKEN(401, "No JWT token presents"), + AUTH(401, "JWT auth error"), + AUTH_USER_NOT_FOUND(401, "JWT auth error"), + TEXT_TO_CHECK_TOO_LONG(20001, "text to check too long"), + TEXT_TO_CHECK_TOO_SHORT(20002, "text to check too short"); + + /** + * 错误码 + */ + private final int resultCode; + + /** + * 错误描述 + */ + private final String resultMsg; + + CnkiCommonEnum(int resultCode, String resultMsg) { + this.resultCode = resultCode; + this.resultMsg = resultMsg; + } + + @Override + public int getResultCode() { + return resultCode; + } + + @Override + public String getResultMsg() { + return resultMsg; + } + +} diff --git a/src/main/java/asia/asoulcnki/api/common/util/EsUtil.java b/src/main/java/asia/asoulcnki/api/common/util/EsUtil.java index d630fab..780124e 100644 --- a/src/main/java/asia/asoulcnki/api/common/util/EsUtil.java +++ b/src/main/java/asia/asoulcnki/api/common/util/EsUtil.java @@ -23,14 +23,11 @@ import org.elasticsearch.client.indices.CreateIndexResponse; import org.elasticsearch.client.indices.GetIndexRequest; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.core.TimeValue; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; -import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; -import org.elasticsearch.search.fetch.subphase.highlight.HighlightField; import org.elasticsearch.search.sort.SortOrder; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -65,10 +62,10 @@ public class EsUtil { * 创建索引 * * @param index 索引 - * @return + * @return bool 是否创建成功 */ public boolean createIndex(String index) throws IOException { - if(isIndexExist(index)){ + if (isIndexExist(index)) { log.error("Index is exits!"); return false; } @@ -77,7 +74,7 @@ public boolean createIndex(String index) throws IOException { //2.执行客户端请求 CreateIndexResponse response = restHighLevelClient.indices().create(request, RequestOptions.DEFAULT); - log.info("创建索引{}成功",index); + log.info("创建索引{}成功", index); return response.isAcknowledged(); } @@ -85,11 +82,11 @@ public boolean createIndex(String index) throws IOException { /** * 删除索引 * - * @param index - * @return + * @param index 索引 + * @return bool 是否删除成功 */ public boolean deleteIndex(String index) throws IOException { - if(!isIndexExist(index)) { + if (!isIndexExist(index)) { log.error("Index is not exits!"); return false; } @@ -98,30 +95,26 @@ public boolean deleteIndex(String index) throws IOException { //执行客户端请求 AcknowledgedResponse delete = restHighLevelClient.indices().delete(request, RequestOptions.DEFAULT); - log.info("删除索引{}成功",index); + log.info("删除索引{}成功", index); return delete.isAcknowledged(); } - /** * 判断索引是否存在 * - * @param index + * @param index 索引 * @return */ public boolean isIndexExist(String index) throws IOException { GetIndexRequest request = new GetIndexRequest(index); - boolean exists = restHighLevelClient.indices().exists(request, RequestOptions.DEFAULT); - - return exists; + return restHighLevelClient.indices().exists(request, RequestOptions.DEFAULT); } - /** * 数据添加，正定ID * @@ -140,12 +133,11 @@ public String addData(JSONObject jsonObject, String index, String id) throws IOE IndexRequest source = request.source(jsonObject, XContentType.JSON); //客户端发送请求 IndexResponse response = restHighLevelClient.index(request, RequestOptions.DEFAULT); - log.info("添加数据成功 索引为: {}, response 状态: {}, id为: {}",index,response.status().getStatus(), response.getId()); + log.info("添加数据成功 索引为: {}, response 状态: {}, id为: {}", index, response.status().getStatus(), response.getId()); return response.getId(); } - /** * 数据添加 随机id * @@ -168,40 +160,35 @@ public void deleteDataById(String index, String id) throws IOException { DeleteRequest request = new DeleteRequest(index, id); //执行客户端请求 DeleteResponse delete = restHighLevelClient.delete(request, RequestOptions.DEFAULT); - log.info("索引为: {}, id为: {}删除数据成功",index, id); + log.info("索引为: {}, id为: {}删除数据成功", index, id); } /** * 通过ID 更新数据 * - * @param object 要增加的数据 - * @param index 索引，类似数据库 - * @param id 数据ID - * @return + * @param object 要增加的数据 + * @param index 索引，类似数据库 + * @param id 数据ID */ public void updateDataById(Object object, String index, String id) throws IOException { //更新请求 UpdateRequest update = new UpdateRequest(index, id); - //保证数据实时更新 - //update.setRefreshPolicy("wait_for"); - update.timeout("1s"); update.doc(JSON.toJSONString(object), XContentType.JSON); //执行更新请求 UpdateResponse update1 = restHighLevelClient.update(update, RequestOptions.DEFAULT); - log.info("索引为: {}, id为: {}, 更新数据成功",index, id); + log.info("索引为: {}, id为: {}, 更新数据成功", index, id); } /** * 通过ID 更新数据,保证实时性 * - * @param object 要增加的数据 - * @param index 索引，类似数据库 - * @param id 数据ID - * @return + * @param object 要增加的数据 + * @param index 索引，类似数据库 + * @param id 数据ID */ public void updateDataByIdNoRealTime(Object object, String index, String id) throws IOException { //更新请求 @@ -214,7 +201,7 @@ public void updateDataByIdNoRealTime(Object object, String index, String id) thr update.doc(JSON.toJSONString(object), XContentType.JSON); //执行更新请求 UpdateResponse update1 = restHighLevelClient.update(update, RequestOptions.DEFAULT); - log.info("索引为: {}, id为: {}, 更新数据成功",index, id); + log.info("索引为: {}, id为: {}, 更新数据成功", index, id); } @@ -226,26 +213,27 @@ public void updateDataByIdNoRealTime(Object object, String index, String id) thr * @param fields 需要显示的字段，逗号分隔（缺省为全部字段） * @return */ - public Map searchDataById(String index, String id, String fields) throws IOException { + public Map searchDataById(String index, String id, String fields) throws IOException { GetRequest request = new GetRequest(index, id); - if (StringUtils.isNotEmpty(fields)){ + if (StringUtils.isNotEmpty(fields)) { //只查询特定字段。如果需要查询所有字段则不设置该项。 - request.fetchSourceContext(new FetchSourceContext(true,fields.split(","), Strings.EMPTY_ARRAY)); + request.fetchSourceContext(new FetchSourceContext(true, fields.split(","), Strings.EMPTY_ARRAY)); } GetResponse response = restHighLevelClient.get(request, RequestOptions.DEFAULT); Map map = response.getSource(); //为返回的数据添加id - map.put("id",response.getId()); + map.put("id", response.getId()); return map; } /** * 通过ID判断文档是否存在 - * @param index 索引，类似数据库 - * @param id 数据ID + * + * @param index 索引，类似数据库 + * @param id 数据ID * @return */ - public boolean existsById(String index,String id) throws IOException { + public boolean existsById(String index, String id) throws IOException { GetRequest request = new GetRequest(index, id); //不获取返回的_source的上下文 request.fetchSourceContext(new FetchSourceContext(false)); @@ -255,6 +243,7 @@ public boolean existsById(String index,String id) throws IOException { /** * 获取低水平客户端 + * * @return */ public RestClient getLowLevelClient() { @@ -265,29 +254,16 @@ public RestClient getLowLevelClient() { /** * 高亮结果集 特殊处理 * map转对象 JSONObject.parseObject(JSONObject.toJSONString(map), Content.class) + * * @param searchResponse * @param highlightField */ public List> setSearchResponse(SearchResponse searchResponse, String highlightField) { //解析结果 - ArrayList> list = new ArrayList<>(); + ArrayList> list = new ArrayList<>(); for (SearchHit hit : searchResponse.getHits().getHits()) { -// Map high = hit.getHighlightFields(); -// HighlightField title = high.get(highlightField); - -// hit.getSourceAsMap().put("id", hit.getId()); -// - Map sourceAsMap = hit.getSourceAsMap();//原来的结果 -// //解析高亮字段,将原来的字段换为高亮字段 -// if (title!=null){ -// Text[] texts = title.fragments(); -// String nTitle=""; -// for (Text text : texts) { -// nTitle+=text; -// } -// //替换 -// sourceAsMap.put(highlightField,nTitle); -// } + //原来的结果 + Map sourceAsMap = hit.getSourceAsMap(); list.add(sourceAsMap); } return list; @@ -296,6 +272,7 @@ public List> setSearchResponse(SearchResponse searchResponse /** * 查询并分页 + * * @param index 索引名称 * @param query 查询条件 * @param size 文档大小限制 @@ -313,36 +290,25 @@ public List> searchListData(String index, String sortField, String highlightField) throws IOException { SearchRequest request = new SearchRequest(index); - SearchSourceBuilder builder = query; - if (StringUtils.isNotEmpty(fields)){ + if (StringUtils.isNotEmpty(fields)) { //只查询特定字段。如果需要查询所有字段则不设置该项。 - builder.fetchSource(new FetchSourceContext(true,fields.split(","), Strings.EMPTY_ARRAY)); + query.fetchSource(new FetchSourceContext(true, fields.split(","), Strings.EMPTY_ARRAY)); } - from = from <= 0 ? 0 : from*size; + from = from <= 0 ? 0 : from * size; //设置确定结果要从哪个索引开始搜索的from选项，默认为0 - builder.from(from); - builder.size(size); - if (StringUtils.isNotEmpty(sortField)){ + query.from(from); + query.size(size); + if (StringUtils.isNotEmpty(sortField)) { //排序字段，注意如果proposal_no是text类型会默认带有keyword性质，需要拼接.keyword - builder.sort(sortField+".keyword", SortOrder.ASC); + query.sort(sortField + ".keyword", SortOrder.ASC); } - //高亮 -// HighlightBuilder highlight = new HighlightBuilder(); -// highlight.field(highlightField); - //关闭多个高亮 -// highlight.requireFieldMatch(false); -// highlight.preTags(""); -// highlight.postTags(""); -// builder.highlighter(highlight); - //不返回源数据。只有条数之类的数据。 - //builder.fetchSource(false); - request.source(builder); + request.source(query); SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT); - log.error("=="+response.getHits().getMaxScore()); + log.error("==" + response.getHits().getMaxScore()); if (response.status().getStatus() == 200) { // 解析对象 return setSearchResponse(response, highlightField); } return null; } -} \ No newline at end of file +} diff --git a/src/main/java/asia/asoulcnki/api/common/util/ObjectMapperFactory.java b/src/main/java/asia/asoulcnki/api/common/util/ObjectMapperFactory.java index ea15a81..403f653 100644 --- a/src/main/java/asia/asoulcnki/api/common/util/ObjectMapperFactory.java +++ b/src/main/java/asia/asoulcnki/api/common/util/ObjectMapperFactory.java @@ -6,7 +6,9 @@ public class ObjectMapperFactory { private static ObjectMapper INSTANCE; - // 私有构造方法，防止错误 new + /** + * 私有构造方法，防止错误 new + */ private ObjectMapperFactory() { } diff --git a/src/main/java/asia/asoulcnki/api/controller/RankingController.java b/src/main/java/asia/asoulcnki/api/controller/RankingController.java index 61d551e..4fe7c90 100644 --- a/src/main/java/asia/asoulcnki/api/controller/RankingController.java +++ b/src/main/java/asia/asoulcnki/api/controller/RankingController.java @@ -23,40 +23,45 @@ @Validated public class RankingController { - @Autowired - IRankingService rankingService; - - @GetMapping("/") - @ResponseBody - public ApiResult getRankingResult(@RequestParam int sortMode, @RequestParam int timeRangeMode, - @RequestParam(value = "ids", required = false) List ids, @RequestParam(value = "keywords", - required = false) List keywords, @RequestParam int pageSize, @RequestParam int pageNum) { - - SortMethodEnum sortMethod = SortMethodEnum.DEFAULT; - switch (sortMode) { - case 1: - sortMethod = SortMethodEnum.LIKE_NUM; - break; - case 2: - sortMethod = SortMethodEnum.SIMILAR_COUNT; - break; - } - - TimeRangeEnum timeRange = TimeRangeEnum.ALL; - switch (timeRangeMode) { - case 1: - timeRange = TimeRangeEnum.ONE_WEEK; - break; - case 2: - timeRange = TimeRangeEnum.THREE_DAYS; - break; - } - - FilterRulesContainer container = new FilterRulesContainer(); - container.addContainsKeywordsPredicate(keywords); - container.addUserIDInFilter(ids); - - return ApiResult.ok(rankingService.queryRankings(sortMethod, timeRange, container, pageSize, pageNum)); - } + @Autowired + IRankingService rankingService; + + @GetMapping("/") + @ResponseBody + public ApiResult getRankingResult(@RequestParam int sortMode, @RequestParam int timeRangeMode, + @RequestParam(value = "ids", required = false) List ids, @RequestParam(value = "keywords", + required = false) List keywords, @RequestParam int pageSize, @RequestParam int pageNum) { + // 选择排序方式和时间筛选范围 + SortMethodEnum sortMethod; + switch (sortMode) { + case 1: + sortMethod = SortMethodEnum.LIKE_NUM; + break; + case 2: + sortMethod = SortMethodEnum.SIMILAR_COUNT; + break; + default: + sortMethod = SortMethodEnum.DEFAULT; + } + + TimeRangeEnum timeRange; + switch (timeRangeMode) { + case 1: + timeRange = TimeRangeEnum.ONE_WEEK; + break; + case 2: + timeRange = TimeRangeEnum.THREE_DAYS; + break; + default: + timeRange = TimeRangeEnum.ALL; + } + + // 筛选关键词 + FilterRulesContainer container = new FilterRulesContainer(); + container.addContainsKeywordsPredicate(keywords); + container.addUserIDInFilter(ids); + + return ApiResult.ok(rankingService.queryRankings(sortMethod, timeRange, container, pageSize, pageNum)); + } } diff --git a/src/main/java/asia/asoulcnki/api/controller/SearchController.java b/src/main/java/asia/asoulcnki/api/controller/SearchController.java index 55d29e0..7b4e2f4 100644 --- a/src/main/java/asia/asoulcnki/api/controller/SearchController.java +++ b/src/main/java/asia/asoulcnki/api/controller/SearchController.java @@ -18,7 +18,6 @@ @RestController @Api(tags = "搜索相关") @Validated -//@Conditional(ConditionalPush.class) public class SearchController { @Autowired diff --git a/src/main/java/asia/asoulcnki/api/service/ICheckService.java b/src/main/java/asia/asoulcnki/api/service/ICheckService.java index 738e1f3..a4458cf 100644 --- a/src/main/java/asia/asoulcnki/api/service/ICheckService.java +++ b/src/main/java/asia/asoulcnki/api/service/ICheckService.java @@ -3,5 +3,11 @@ import asia.asoulcnki.api.persistence.vo.CheckResultVo; public interface ICheckService { - CheckResultVo check(String text); + /** + * 查重 + * + * @param text 小作文文本 + * @return 查重结果 ViewObject + */ + CheckResultVo check(String text); } diff --git a/src/main/java/asia/asoulcnki/api/service/IDataService.java b/src/main/java/asia/asoulcnki/api/service/IDataService.java index 2519e9a..97cad0f 100644 --- a/src/main/java/asia/asoulcnki/api/service/IDataService.java +++ b/src/main/java/asia/asoulcnki/api/service/IDataService.java @@ -3,11 +3,11 @@ import asia.asoulcnki.api.persistence.vo.ControlResultVo; public interface IDataService { - ControlResultVo pull(int startTime); + ControlResultVo pull(int startTime); - ControlResultVo checkpoint(); + ControlResultVo checkpoint(); - ControlResultVo reset(); + ControlResultVo reset(); - ControlResultVo train(); + ControlResultVo train(); } diff --git a/src/main/java/asia/asoulcnki/api/service/IRankingService.java b/src/main/java/asia/asoulcnki/api/service/IRankingService.java index 3981c20..20933e0 100644 --- a/src/main/java/asia/asoulcnki/api/service/IRankingService.java +++ b/src/main/java/asia/asoulcnki/api/service/IRankingService.java @@ -4,16 +4,39 @@ import asia.asoulcnki.api.persistence.vo.RankingResultVo; public interface IRankingService { - RankingResultVo queryRankings(SortMethodEnum sortMethod, final TimeRangeEnum timeRange, - final FilterRulesContainer container, final int pageSize, final int pageNum); + /** + * 按指定规则筛选排序展示小作文 + * + * @param sortMethod 排序方法 + * @param timeRange 筛选时间 + * @param container 过滤容器 + * @param pageSize 单页显示小作文数 + * @param pageNum 页数 + * @return + */ + RankingResultVo queryRankings(SortMethodEnum sortMethod, final TimeRangeEnum timeRange, + final FilterRulesContainer container, final int pageSize, final int pageNum); - void refresh(); + /** + * 刷新排行 + */ + void refresh(); - enum SortMethodEnum { - DEFAULT, LIKE_NUM, SIMILAR_COUNT - } + enum SortMethodEnum { + // 默认排序 + DEFAULT, + // 按照点赞数排序 + LIKE_NUM, + // 按照引用次数排序 + SIMILAR_COUNT + } - enum TimeRangeEnum { - ALL, ONE_WEEK, THREE_DAYS - } + enum TimeRangeEnum { + // 在所有时间内筛选 + ALL, + // 筛选一周内 + ONE_WEEK, + // 筛选三天内 + THREE_DAYS + } } diff --git a/src/main/java/asia/asoulcnki/api/service/impl/ICheckServiceImpl.java b/src/main/java/asia/asoulcnki/api/service/impl/ICheckServiceImpl.java index a13b4e6..283cb1d 100644 --- a/src/main/java/asia/asoulcnki/api/service/impl/ICheckServiceImpl.java +++ b/src/main/java/asia/asoulcnki/api/service/impl/ICheckServiceImpl.java @@ -2,7 +2,9 @@ import asia.asoulcnki.api.common.BizException; import asia.asoulcnki.api.common.duplicationcheck.ArticleCompareUtil; + import static asia.asoulcnki.api.common.duplicationcheck.ArticleCompareUtil.isHighSimilarity; + import asia.asoulcnki.api.common.duplicationcheck.ComparisonDatabase; import asia.asoulcnki.api.common.duplicationcheck.SummaryHash; import asia.asoulcnki.api.common.response.CnkiCommonEnum; @@ -24,102 +26,145 @@ @Service @CacheConfig(cacheNames = "checkCache") public class ICheckServiceImpl implements ICheckService { - private final static Logger log = LoggerFactory.getLogger(ICheckServiceImpl.class); - - @Override - public CheckResultVo check(String text) { - text = ArticleCompareUtil.trim(text); - int textLength = text.codePointCount(0, text.length()); - if (textLength < SummaryHash.DEFAULT_K) { - log.error("the text to check is too short, codepoint number {} ", textLength); - throw new BizException(CnkiCommonEnum.TEXT_TO_CHECK_TOO_SHORT); - } else if (textLength > 2000) { - log.error("the text to check is too long, codepoint number {} ", textLength); - throw new BizException(CnkiCommonEnum.TEXT_TO_CHECK_TOO_LONG); - } - return getDuplicationCheckResult(text); - } - - @Cacheable(key = "#text", value = "replyCache") - public CheckResultVo getDuplicationCheckResult(String text) { - ComparisonDatabase db = ComparisonDatabase.getInstance(); - - ArrayList textHashList = SummaryHash.defaultHash(text); - - int textLength = text.codePointCount(0, text.length()); - int minHit = 1; - if (textLength > 30) { - minHit = 2; - } - List> sortedRelatedReplyIdList = ComparisonDatabase.searchRelatedReplies(textHashList - , minHit); - List related = new ArrayList<>(textHashList.size() / 2); - - StringBuilder allContentBuilder = new StringBuilder(); - - for (Map.Entry entry : sortedRelatedReplyIdList) { - Reply reply = db.getReply(entry.getKey()); - String content = ArticleCompareUtil.trim(reply.getContent()); - float similarity = ArticleCompareUtil.compareArticle(text, content); - if (similarity < 0.2) { - continue; - } - String replyUrl = getReplyUrl(reply); - allContentBuilder.append(content); - related.add(new RelatedReply(similarity, reply, replyUrl)); - } - - // param -> left hand side and right hand side - related.sort((lhs, rhs) -> { - float lhsSimilarity = lhs.getRate(); - int lhsCTime = lhs.getReply().getCtime(); - - float rhsSimilarity = rhs.getRate(); - int rhsCTime = rhs.getReply().getCtime(); - - if (isHighSimilarity(textLength, lhsSimilarity) && isHighSimilarity(textLength, rhsSimilarity)) { - return Integer.compare(lhsCTime, rhsCTime); - } - - if (lhsSimilarity != rhsSimilarity) { - return -Float.compare(lhsSimilarity, rhsSimilarity); - } else { - return Integer.compare(lhsCTime, rhsCTime); - } - }); - - float allSimilarity = 0; - String allContent = allContentBuilder.toString(); - if (!StringUtils.isBlank(allContent)) { - allSimilarity = ArticleCompareUtil.compareArticle(text, allContentBuilder.toString()); - } - - - CheckResultVo vo = new CheckResultVo(); - vo.setStartTime(db.getMinTime()); - vo.setEndTime(db.getMaxTime()); - vo.setAllSimilarity(allSimilarity); - if (related.size() > 5) { - vo.setRelated(related.subList(0, 5)); - } else { - vo.setRelated(related); - } - return vo; - } - - private String getReplyUrl(Reply reply) { - String baseUrl = "https://www.bilibili.com"; - String dynamicBaseUrl = "https://t.bilibili.com"; - switch (reply.getTypeId()) { - case 1: - return String.format(" %s/video/av%d/#reply%d", baseUrl, reply.getOid(), reply.getRpid()); - case 11: - case 17: - return String.format(" %s/%d/#reply%d", dynamicBaseUrl, reply.getDynamicId(), reply.getRpid()); - case 12: - return String.format(" %s/read/cv%d/#reply%d", baseUrl, reply.getOid(), reply.getRpid()); - default: - return ""; - } - } + private final static Logger log = LoggerFactory.getLogger(ICheckServiceImpl.class); + /** + * 小作文最大长度限制 + */ + private final static int TEXT_MAX_LENGTH = 1000; + + /** + * 调高 minHit 的文本长度阈值 + */ + private final static int INCREASE_MIN_HIT_THRESHOLD = 30; + + /** + * 若相似小作文的个数超过 RELATED_ARTICLE_NUM，则只展示前 RELATED_ARTICLE_NUM 个 + */ + private final static int RELATED_ARTICLE_NUM = 5; + + /** + * 判断小作文相似的阈值 + */ + private static final double SIMILARITY_THRESHOLD = 0.2; + + @Override + public CheckResultVo check(String text) { + // 先过滤空白字符、换行符等 + text = ArticleCompareUtil.trim(text); + int textLength = text.codePointCount(0, text.length()); + // 小作文不能太短或者太长 + if (textLength < SummaryHash.DEFAULT_K) { + log.error("the text to check is too short, codepoint number {} ", textLength); + throw new BizException(CnkiCommonEnum.TEXT_TO_CHECK_TOO_SHORT); + } else if (textLength > TEXT_MAX_LENGTH) { + log.error("the text to check is too long, codepoint number {} ", textLength); + throw new BizException(CnkiCommonEnum.TEXT_TO_CHECK_TOO_LONG); + } + return getDuplicationCheckResult(text); + } + + /** + * 在对比库中查找与 text 相似的小作文 + * + * @param text 待查重文本 + * @return 查重结果 CheckResultVO + */ + @Cacheable(key = "#text", value = "replyCache") + public CheckResultVo getDuplicationCheckResult(String text) { + ComparisonDatabase db = ComparisonDatabase.getInstance(); + + ArrayList textHashList = SummaryHash.defaultHash(text); + + int textLength = text.codePointCount(0, text.length()); + // minHit 至少查询命中的 hash 次数 + int minHit = 1; + // 如果小作文长度长过阈值，则调高 minHit + if (textLength > INCREASE_MIN_HIT_THRESHOLD) { + minHit = 2; + } + // sortedRelatedReplayIdList 是所有包含 hash 值的评论的 键值对列表 + List> sortedRelatedReplyIdList = ComparisonDatabase.searchRelatedReplies(textHashList + , minHit); + // related 是相似小作文列表 + List related = new ArrayList<>(textHashList.size() / 2); + + StringBuilder allContentBuilder = new StringBuilder(); + + // 遍历列表，将输入与列表元素逐一比对，看是否有相似小作文 + for (Map.Entry entry : sortedRelatedReplyIdList) { + Reply reply = db.getReply(entry.getKey()); + String content = ArticleCompareUtil.trim(reply.getContent()); + // 预处理后如果相似度低于阈值，则认为不相似 + float similarity = ArticleCompareUtil.compareArticle(text, content); + if (similarity < SIMILARITY_THRESHOLD) { + continue; + } + String replyUrl = getReplyUrl(reply); + allContentBuilder.append(content); + related.add(new RelatedReply(similarity, reply, replyUrl)); + } + + // 对查到的相似小作文进行排序 + related.sort((leftHandSide, rightHandSide) -> { + float lhsSimilarity = leftHandSide.getRate(); + int lhsCTime = leftHandSide.getReply().getCtime(); + + float rhsSimilarity = rightHandSide.getRate(); + int rhsCTime = rightHandSide.getReply().getCtime(); + + // 如果都是高相似小作文，则优先根据评论时间先后排序 + if (isHighSimilarity(textLength, lhsSimilarity) && isHighSimilarity(textLength, rhsSimilarity)) { + return Integer.compare(lhsCTime, rhsCTime); + } + + // 如果相似的小作文都不是高度相似，则优先根据相似度高低排序，相似度相同再根据评论时间先后排序 + if (Float.compare(lhsSimilarity, rhsSimilarity) != 0) { + return -Float.compare(lhsSimilarity, rhsSimilarity); + } else { + return Integer.compare(lhsCTime, rhsCTime); + } + }); + + // 计算总复制比 + float allSimilarity = 0; + String allContent = allContentBuilder.toString(); + if (!StringUtils.isBlank(allContent)) { + allSimilarity = ArticleCompareUtil.compareArticle(text, allContentBuilder.toString()); + } + + // 构建查重结果VO + CheckResultVo vo = new CheckResultVo(); + vo.setStartTime(db.getMinTime()); + vo.setEndTime(db.getMaxTime()); + vo.setAllSimilarity(allSimilarity); + // 若相似小作文的个数超过 RELATED_ARTICLE_NUM，则只展示前 RELATED_ARTICLE_NUM 个 + if (related.size() > RELATED_ARTICLE_NUM) { + vo.setRelated(related.subList(0, RELATED_ARTICLE_NUM)); + } else { + vo.setRelated(related); + } + return vo; + } + + /** + * 获取原评论链接 + * + * @param reply Reply 对象 + * @return 原评论 URL 链接 + */ + private String getReplyUrl(Reply reply) { + String baseUrl = "https://www.bilibili.com"; + String dynamicBaseUrl = "https://t.bilibili.com"; + switch (reply.getTypeId()) { + case 1: + return String.format(" %s/video/av%d/#reply%d", baseUrl, reply.getOid(), reply.getRpid()); + case 11: + case 17: + return String.format(" %s/%d/#reply%d", dynamicBaseUrl, reply.getDynamicId(), reply.getRpid()); + case 12: + return String.format(" %s/read/cv%d/#reply%d", baseUrl, reply.getOid(), reply.getRpid()); + default: + return ""; + } + } } diff --git a/src/main/java/asia/asoulcnki/api/service/impl/IDataServiceImpl.java b/src/main/java/asia/asoulcnki/api/service/impl/IDataServiceImpl.java index 16aafd4..77be604 100644 --- a/src/main/java/asia/asoulcnki/api/service/impl/IDataServiceImpl.java +++ b/src/main/java/asia/asoulcnki/api/service/impl/IDataServiceImpl.java @@ -28,150 +28,150 @@ @Service @CacheConfig(cacheNames = "checkCache") public class IDataServiceImpl implements IDataService { - private final static Logger log = LoggerFactory.getLogger(IDataServiceImpl.class); - @Autowired - IReplyService replyService; - - @Autowired - IRankingService rankingService; - - private static final ObjectMapper objectMapper = ObjectMapperFactory.getInstance(); - - public static List getJsonFile(String path) { - try { - JavaType javaType = objectMapper.getTypeFactory().constructCollectionType(List.class, Reply.class); - return objectMapper.readValue(new File(path), javaType); - } catch (IOException e) { - e.printStackTrace(); - return null; - } - } - - @Cacheable(value = "defaultCache") - public long getStartRpid(int startTime) { - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.gt("ctime", startTime).select("min(rpid) as min_rpid"); - List r = replyService.getBaseMapper().selectObjs(queryWrapper); - if (r == null) { - return 0; - } else { - return (long) r.get(0); - } - } - - @Override - @CacheEvict(value = "replyCache", allEntries = true) - public ControlResultVo pull(int startTime) { - long queryStartRpidStart = System.currentTimeMillis(); - long startRpid = getStartRpid(startTime); - long queryStartRpidEnd = System.currentTimeMillis(); - log.info("query start rpid end, cost {} ms, start rpid : {}", queryStartRpidEnd - queryStartRpidStart, - startRpid); - - ComparisonDatabase db = ComparisonDatabase.getInstance(); - - int pageIndex = 1; - int pageSize = 10000; - - - int count = 0; - while (true) { - // construct query wrapper - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.gt("rpid", startRpid); - String lastSql = String.format("limit %d,%d", (pageIndex - 1) * pageSize, pageSize); - queryWrapper.last(lastSql); - - long start = System.currentTimeMillis(); - // query database - List replies = replyService.list(queryWrapper); - if (replies == null || replies.isEmpty()) { - break; - } - - // add to comparison database - db.writeLock(); - try { - addRepliesToDatabase(replies); - } finally { - db.writeUnLock(); - } - - count += replies.size(); - pageIndex++; - log.info("add {} records to database, cost {} ms", replies.size(), System.currentTimeMillis() - start); - } - - long pullDataEnd = System.currentTimeMillis(); - log.info("pull data cost {} ms, add {} records to comparison database in total", - pullDataEnd - queryStartRpidEnd, count); - return checkpoint(); - } - - private void addRepliesToDatabase(List replies) { - boolean addToHistoryDatabase = System.getProperty("history.enable") != null; - for (Reply reply : replies) { - ComparisonDatabase.getInstance().addReplyData(reply); - if (addToHistoryDatabase) { - UserSpeechHistoryList.getInstance().add(reply); - } - } - } - - @Override - public ControlResultVo checkpoint() { - long start = System.currentTimeMillis(); - ComparisonDatabase db = ComparisonDatabase.getInstance(); - db.readLock(); - try { - db.dumpToImage(ComparisonDatabase.DEFAULT_DATA_DIR, ComparisonDatabase.DEFAULT_IMAGE_FILE_NAME); - rankingService.refresh(); - } catch (Exception e) { - throw new BizException(CnkiCommonEnum.INTERNAL_SERVER_ERROR, e); - } finally { - db.readUnLock(); - } - log.info("checkpoint database finished, cost {} ms", System.currentTimeMillis() - start); - return new ControlResultVo(db.getMinTime(), db.getMaxTime()); - } - - @Override - @CacheEvict(value = "replyCache", allEntries = true) - public ControlResultVo reset() { - ComparisonDatabase db = ComparisonDatabase.getInstance(); - try { - db.reset(); - } catch (Exception e) { - throw new BizException(CnkiCommonEnum.INTERNAL_SERVER_ERROR, e); - } - return checkpoint(); - } - - @Override - @CacheEvict(value = "replyCache", allEntries = true) - public ControlResultVo train() { - ComparisonDatabase.getInstance().writeLock(); - try { - long start = System.currentTimeMillis(); - List node = getJsonFile("data/bilibili_cnki_reply.json"); - if (node == null) { - throw new BizException(CnkiCommonEnum.INTERNAL_SERVER_ERROR); - } - ComparisonDatabase db = ComparisonDatabase.getInstance(); - for (int i = 0; i < node.size(); i++) { - if (i % 10000 == 0) { - float percent = (float) i / node.size() * 100; - log.info(String.format("train process: %.2f%% detail: %d/%d", percent, i, node.size())); - } - db.addReplyData(node.get(i)); - } - log.info("train end cost {} ms", System.currentTimeMillis() - start); - } catch (Exception e) { - e.printStackTrace(); - throw new BizException(CnkiCommonEnum.INTERNAL_SERVER_ERROR, e); - } finally { - ComparisonDatabase.getInstance().writeUnLock(); - } - return checkpoint(); - } + private final static Logger log = LoggerFactory.getLogger(IDataServiceImpl.class); + @Autowired + IReplyService replyService; + + @Autowired + IRankingService rankingService; + + private static final ObjectMapper objectMapper = ObjectMapperFactory.getInstance(); + + public static List getJsonFile(String path) { + try { + JavaType javaType = objectMapper.getTypeFactory().constructCollectionType(List.class, Reply.class); + return objectMapper.readValue(new File(path), javaType); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + } + + @Cacheable(value = "defaultCache") + public long getStartRpid(int startTime) { + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.gt("ctime", startTime).select("min(rpid) as min_rpid"); + List r = replyService.getBaseMapper().selectObjs(queryWrapper); + if (r == null) { + return 0; + } else { + return (long) r.get(0); + } + } + + @Override + @CacheEvict(value = "replyCache", allEntries = true) + public ControlResultVo pull(int startTime) { + long queryStartRpidStart = System.currentTimeMillis(); + long startRpid = getStartRpid(startTime); + long queryStartRpidEnd = System.currentTimeMillis(); + log.info("query start rpid end, cost {} ms, start rpid : {}", queryStartRpidEnd - queryStartRpidStart, + startRpid); + + ComparisonDatabase db = ComparisonDatabase.getInstance(); + + int pageIndex = 1; + int pageSize = 10000; + + + int count = 0; + while (true) { + // construct query wrapper + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.gt("rpid", startRpid); + String lastSql = String.format("limit %d,%d", (pageIndex - 1) * pageSize, pageSize); + queryWrapper.last(lastSql); + + long start = System.currentTimeMillis(); + // query database + List replies = replyService.list(queryWrapper); + if (replies == null || replies.isEmpty()) { + break; + } + + // add to comparison database + db.writeLock(); + try { + addRepliesToDatabase(replies); + } finally { + db.writeUnLock(); + } + + count += replies.size(); + pageIndex++; + log.info("add {} records to database, cost {} ms", replies.size(), System.currentTimeMillis() - start); + } + + long pullDataEnd = System.currentTimeMillis(); + log.info("pull data cost {} ms, add {} records to comparison database in total", + pullDataEnd - queryStartRpidEnd, count); + return checkpoint(); + } + + private void addRepliesToDatabase(List replies) { + boolean addToHistoryDatabase = System.getProperty("history.enable") != null; + for (Reply reply : replies) { + ComparisonDatabase.getInstance().addReplyData(reply); + if (addToHistoryDatabase) { + UserSpeechHistoryList.getInstance().add(reply); + } + } + } + + @Override + public ControlResultVo checkpoint() { + long start = System.currentTimeMillis(); + ComparisonDatabase db = ComparisonDatabase.getInstance(); + db.readLock(); + try { + db.dumpToImage(ComparisonDatabase.DEFAULT_DATA_DIR, ComparisonDatabase.DEFAULT_IMAGE_FILE_NAME); + rankingService.refresh(); + } catch (Exception e) { + throw new BizException(CnkiCommonEnum.INTERNAL_SERVER_ERROR, e); + } finally { + db.readUnLock(); + } + log.info("checkpoint database finished, cost {} ms", System.currentTimeMillis() - start); + return new ControlResultVo(db.getMinTime(), db.getMaxTime()); + } + + @Override + @CacheEvict(value = "replyCache", allEntries = true) + public ControlResultVo reset() { + ComparisonDatabase db = ComparisonDatabase.getInstance(); + try { + db.reset(); + } catch (Exception e) { + throw new BizException(CnkiCommonEnum.INTERNAL_SERVER_ERROR, e); + } + return checkpoint(); + } + + @Override + @CacheEvict(value = "replyCache", allEntries = true) + public ControlResultVo train() { + ComparisonDatabase.getInstance().writeLock(); + try { + long start = System.currentTimeMillis(); + List node = getJsonFile("data/bilibili_cnki_reply.json"); + if (node == null) { + throw new BizException(CnkiCommonEnum.INTERNAL_SERVER_ERROR); + } + ComparisonDatabase db = ComparisonDatabase.getInstance(); + for (int i = 0; i < node.size(); i++) { + if (i % 10000 == 0) { + float percent = (float) i / node.size() * 100; + log.info(String.format("train process: %.2f%% detail: %d/%d", percent, i, node.size())); + } + db.addReplyData(node.get(i)); + } + log.info("train end cost {} ms", System.currentTimeMillis() - start); + } catch (Exception e) { + e.printStackTrace(); + throw new BizException(CnkiCommonEnum.INTERNAL_SERVER_ERROR, e); + } finally { + ComparisonDatabase.getInstance().writeUnLock(); + } + return checkpoint(); + } } diff --git a/src/main/java/asia/asoulcnki/api/service/impl/IRankingServiceImpl.java b/src/main/java/asia/asoulcnki/api/service/impl/IRankingServiceImpl.java index eea5c28..f5123ce 100644 --- a/src/main/java/asia/asoulcnki/api/service/impl/IRankingServiceImpl.java +++ b/src/main/java/asia/asoulcnki/api/service/impl/IRankingServiceImpl.java @@ -14,29 +14,29 @@ @CacheConfig(cacheNames = "checkCache") public class IRankingServiceImpl implements IRankingService { - @Override - @Cacheable(value = "leaderboard") - public RankingResultVo queryRankings(final SortMethodEnum sortMethod, final TimeRangeEnum timeRange, - final FilterRulesContainer container, final int pageSize, final int pageNum) { - LeaderBoardEntry leaderBoard; + @Override + @Cacheable(value = "leaderboard") + public RankingResultVo queryRankings(final SortMethodEnum sortMethod, final TimeRangeEnum timeRange, + final FilterRulesContainer container, final int pageSize, final int pageNum) { + LeaderBoardEntry leaderBoard; - switch (sortMethod) { - case SIMILAR_COUNT: - leaderBoard = LeaderBoard.getInstance().getSimilarCountLeaderBoard(); - break; - case LIKE_NUM: - leaderBoard = LeaderBoard.getInstance().getLikeLeaderBoard(); - break; - default: - leaderBoard = LeaderBoard.getInstance().getSimilarLikeSumLeaderboard(); - } + switch (sortMethod) { + case SIMILAR_COUNT: + leaderBoard = LeaderBoard.getInstance().getSimilarCountLeaderBoard(); + break; + case LIKE_NUM: + leaderBoard = LeaderBoard.getInstance().getLikeLeaderBoard(); + break; + default: + leaderBoard = LeaderBoard.getInstance().getSimilarLikeSumLeaderboard(); + } - return leaderBoard.query(container.getFilter(), timeRange, pageSize, pageNum); - } + return leaderBoard.query(container.getFilter(), timeRange, pageSize, pageNum); + } - @Override - @CacheEvict(value = "leaderboard", allEntries = true) - public void refresh() { - LeaderBoard.getInstance().refresh(); - } + @Override + @CacheEvict(value = "leaderboard", allEntries = true) + public void refresh() { + LeaderBoard.getInstance().refresh(); + } } diff --git a/src/main/java/asia/asoulcnki/api/service/impl/ISearchServiceImpl.java b/src/main/java/asia/asoulcnki/api/service/impl/ISearchServiceImpl.java index a818eb2..eba5862 100644 --- a/src/main/java/asia/asoulcnki/api/service/impl/ISearchServiceImpl.java +++ b/src/main/java/asia/asoulcnki/api/service/impl/ISearchServiceImpl.java @@ -21,7 +21,7 @@ @Service public class ISearchServiceImpl implements ISearchService { - private static final Logger logger = LoggerFactory.getLogger(ISearchServiceImpl.class); + private static final Logger log = LoggerFactory.getLogger(ISearchServiceImpl.class); private static final String DEFAULT_INDEX = "as"; diff --git a/src/test/java/asia/asoulcnki/api/common/duplicationcheck/ArticleCompareUtilTest.java b/src/test/java/asia/asoulcnki/api/common/duplicationcheck/ArticleCompareUtilTest.java new file mode 100644 index 0000000..d16b3f9 --- /dev/null +++ b/src/test/java/asia/asoulcnki/api/common/duplicationcheck/ArticleCompareUtilTest.java @@ -0,0 +1,17 @@ +package asia.asoulcnki.api.common.duplicationcheck; + +import org.junit.Test; +import org.springframework.boot.test.context.SpringBootTest; + +import static org.testng.Assert.*; + +@SpringBootTest +public class ArticleCompareUtilTest { + + @Test + public void testTrim() { + String s = "7月9日，认识了一个男人，他跟我聊叔本华，聊弗洛伊，聊庄子妻死，聊伽罗瓦和近世代数，聊彭罗斯和宇宙督察假说\n" + "\n" + "7月10日，我让他b站关注嘉然今天吃什么"; + int trimmedLength = ArticleCompareUtil.textLength(ArticleCompareUtil.trim(s)); + assertEquals(66, trimmedLength); + } +}