(improvement)(common|headless|chat|auth) 鉴权优化与召回优化

1 修复生成的用户token 一生成就失效的问题
2 如果用户设置的token ，需校验是否数据库存在，因为用户可设置一年的token 有泄露风险
3 结果解析优化， 去除不可以解析的情况，解析问题需要改写后的问，
4 召回样例，用相似度，保住至少有一个样例是高相似度的
5 数据集召回，填加完全匹配格式筛选逻辑

Author: guilinlewis

auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/adaptor/DefaultUserAdaptor.java

@@ -19,6 +19,7 @@
 import org.springframework.beans.BeanUtils;
 
 import java.sql.Timestamp;
+import java.util.Date;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
@@ -223,8 +224,8 @@ public UserToken generateToken(String name, String userName, long expireTime) {
                         userDO.getEmail(), userDO.getPassword(), userDO.getIsAdmin());
 
         // 使用令牌名称作为生成key ，这样可以区分正常请求和api 请求，api 的令牌失效时间很长，需考虑令牌泄露的情况
-        String token =
-                tokenService.generateToken(UserWithPassword.convert(userWithPassword),"SysDbToken:"+name, (new Date().getTime() + expireTime));
+        String token = tokenService.generateToken(UserWithPassword.convert(userWithPassword),
+                "SysDbToken:" + name, (new Date().getTime() + expireTime));
         UserTokenDO userTokenDO = saveUserToken(name, userName, token, expireTime);
         return convertUserToken(userTokenDO);
     }

auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/repository/UserRepository.java

@@ -21,6 +21,8 @@ public interface UserRepository {
 
     UserTokenDO getUserToken(Long tokenId);
 
+    UserTokenDO getUserTokenByName(String tokenName);
+
     void deleteUserTokenByName(String userName);
 
     void deleteUserToken(Long tokenId);

auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/repository/impl/UserRepositoryImpl.java

@@ -65,6 +65,13 @@ public UserTokenDO getUserToken(Long tokenId) {
         return userTokenDOMapper.selectById(tokenId);
     }
 
+    @Override
+    public UserTokenDO getUserTokenByName(String tokenName) {
+        QueryWrapper<UserTokenDO> queryWrapper = new QueryWrapper<>();
+        queryWrapper.lambda().eq(UserTokenDO::getName, tokenName);
+        return userTokenDOMapper.selectOne(queryWrapper);
+    }
+
     @Override
     public void deleteUserTokenByName(String userName) {
         QueryWrapper<UserTokenDO> queryWrapper = new QueryWrapper<>();

auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/utils/TokenService.java

@@ -94,10 +94,11 @@ private Optional<Claims> getClaims(String token, HttpServletRequest request) {
 
     public Optional<Claims> getClaims(String token, String appKey) {
         try {
-            if(StringUtils.isNotBlank(appKey)&&appKey.startsWith("SysDbToken:")) {// 如果是配置的长期令牌，需校验数据库是否存在该配置
+            if (StringUtils.isNotBlank(appKey) && appKey.startsWith("SysDbToken:")) {// 如果是配置的长期令牌，需校验数据库是否存在该配置
                 UserRepository userRepository = ContextUtils.getBean(UserRepository.class);
-                UserTokenDO dbToken= userRepository.getUserTokenByName(appKey.substring("SysDbToken:".length()));
-                if(dbToken==null||!dbToken.getToken().equals(token.replace("Bearer ",""))) {
+                UserTokenDO dbToken =
+                        userRepository.getUserTokenByName(appKey.substring("SysDbToken:".length()));
+                if (dbToken == null || !dbToken.getToken().equals(token.replace("Bearer ", ""))) {
                     throw new AccessException("Token does not exist :" + appKey);
                 }
             }
@@ -133,14 +134,14 @@ private String getTokenSecret(String appKey) {
         Map<String, String> appKeyToSecretMap = authenticationConfig.getAppKeyToSecretMap();
         String secret = appKeyToSecretMap.get(appKey);
         if (StringUtils.isBlank(secret)) {
-            if(StringUtils.isNotBlank(appKey)&&appKey.startsWith("SysDbToken:")) { // 是配置的长期令牌
-                String realAppKey=appKey.substring("SysDbToken:".length());
-                String tmp = "WIaO9YRRVt+7QtpPvyWsARFngnEcbaKBk783uGFwMrbJBaochsqCH62L4Kijcb0sZCYoSsiKGV/zPml5MnZ3uQ==";
-                if(tmp.length()<=realAppKey.length()) {
+            if (StringUtils.isNotBlank(appKey) && appKey.startsWith("SysDbToken:")) { // 是配置的长期令牌
+                String realAppKey = appKey.substring("SysDbToken:".length());
+                String tmp =
+                        "WIaO9YRRVt+7QtpPvyWsARFngnEcbaKBk783uGFwMrbJBaochsqCH62L4Kijcb0sZCYoSsiKGV/zPml5MnZ3uQ==";
+                if (tmp.length() <= realAppKey.length()) {
                     return realAppKey;
-                }
-                else{
-                    return realAppKey+tmp.substring(realAppKey.length());
+                } else {
+                    return realAppKey + tmp.substring(realAppKey.length());
                 }
             }
             throw new AccessException("get secret from appKey failed :" + appKey);

chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DataInterpretProcessor.java

@@ -47,8 +47,8 @@ public boolean accept(ExecuteContext executeContext) {
         Agent agent = executeContext.getAgent();
         ChatApp chatApp = agent.getChatAppConfig().get(APP_KEY);
         return Objects.nonNull(chatApp) && chatApp.isEnable()
-                && StringUtils.isNotBlank(executeContext.getResponse().getTextResult())  // 如果都没结果，则无法处理
-                && StringUtils.isBlank(executeContext.getResponse().getTextSummary());  // 如果已经有汇总的结果了，无法再次处理
+                && StringUtils.isNotBlank(executeContext.getResponse().getTextResult()) // 如果都没结果，则无法处理
+                && StringUtils.isBlank(executeContext.getResponse().getTextSummary()); // 如果已经有汇总的结果了，无法再次处理
     }
 
     @Override
@@ -59,10 +59,11 @@ public void process(ExecuteContext executeContext) {
 
         Map<String, Object> variable = new HashMap<>();
         String question = executeContext.getResponse().getTextResult();// 结果解析应该用改写的问题，因为改写的内容信息量更大
-        if(executeContext.getParseInfo().getProperties()!=null&&
-                executeContext.getParseInfo().getProperties().containsKey("CONTEXT")){
-            Map<String,Object> context = (Map<String, Object>) executeContext.getParseInfo().getProperties().get("CONTEXT");
-            if(context.get("queryText")!=null&&"".equals(context.get("queryText"))){
+        if (executeContext.getParseInfo().getProperties() != null
+                && executeContext.getParseInfo().getProperties().containsKey("CONTEXT")) {
+            Map<String, Object> context = (Map<String, Object>) executeContext.getParseInfo()
+                    .getProperties().get("CONTEXT");
+            if (context.get("queryText") != null && "".equals(context.get("queryText"))) {
                 question = context.get("queryText").toString();
             }
         }

common/src/main/java/com/hankcs/hanlp/LoadRemoveService.java

@@ -21,7 +21,8 @@ public List removeNatures(List value, Set<Long> modelIdOrDataSetIds) {
         List<String> resultList = new ArrayList<>(value);
         if (!CollectionUtils.isEmpty(modelIdOrDataSetIds)) {
             resultList.removeIf(nature -> {
-                if (Objects.isNull(nature)||!nature.startsWith("_")) { // 系统的字典是以 _ 开头的， 过滤因引用外部字典导致的异常
+                if (Objects.isNull(nature) || !nature.startsWith("_")) { // 系统的字典是以 _ 开头的，
+                                                                         // 过滤因引用外部字典导致的异常
                     return false;
                 }
                 Long id = getId(nature);

common/src/main/java/com/tencent/supersonic/common/pojo/Text2SQLExemplar.java

@@ -23,5 +23,5 @@ public class Text2SQLExemplar implements Serializable {
 
     private String sql;
 
-    protected double similarity; //  传递相似度，可以作为样本筛选的依据
+    protected double similarity; // 传递相似度，可以作为样本筛选的依据
 }

common/src/main/java/com/tencent/supersonic/common/service/impl/ExemplarServiceImpl.java

@@ -72,7 +72,7 @@ public List<Text2SQLExemplar> recallExemplars(String collection, String query, i
                 embeddingService.retrieveQuery(collection, retrieveQuery, num);
         results.forEach(ret -> {
             ret.getRetrieval().forEach(r -> {
-                Text2SQLExemplar tmp = //传递相似度，可以作为样本筛选的依据
+                Text2SQLExemplar tmp = // 传递相似度，可以作为样本筛选的依据
                         JsonUtil.mapToObject(r.getMetadata(), Text2SQLExemplar.class);
                 tmp.setSimilarity(r.getSimilarity());
                 exemplars.add(tmp);

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticParseInfo.java

@@ -66,7 +66,7 @@ public int compare(SemanticParseInfo o1, SemanticParseInfo o2) {
             DataSetMatchResult mr2 = getDataSetMatchResult(o2.getElementMatches());
 
             double difference = mr1.getMaxDatesetSimilarity() - mr2.getMaxDatesetSimilarity();
-            if (Math.abs(difference) < 0.0005) { //  看完全匹配的个数，实践证明，可以用户输入规范后，该逻辑具有优势
+            if (Math.abs(difference) < 0.0005) { // 看完全匹配的个数，实践证明，可以用户输入规范后，该逻辑具有优势
                 if (!o1.getDataSetId().equals(o2.getDataSetId())) {
                     List<SchemaElementMatch> elementMatches1 = o1.getElementMatches().stream()
                             .filter(e -> e.getSimilarity() == 1).collect(Collectors.toList());

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/PromptHelper.java

@@ -49,7 +49,7 @@ public List<List<Text2SQLExemplar>> getFewShotExemplars(LLMReq llmReq) {
         // use random collection of exemplars for each self-consistency inference
         for (int i = 0; i < selfConsistencyNumber; i++) {
             List<Text2SQLExemplar> shuffledList = new ArrayList<>(exemplars);
-            List<Text2SQLExemplar> same = shuffledList.stream() //  相似度极高的话，先找出来
+            List<Text2SQLExemplar> same = shuffledList.stream() // 相似度极高的话，先找出来
                     .filter(e -> e.getSimilarity() > 0.989).collect(Collectors.toList());
             List<Text2SQLExemplar> noSame = shuffledList.stream()
                     .filter(e -> e.getSimilarity() <= 0.989).collect(Collectors.toList());