Feat/0.2.2 (#235)

langchain升级到0.0.349

.github/workflows/release.yml

@@ -24,8 +24,8 @@ jobs:
       - name: get commit message
         id: get_commit_message
         run: |
-          echo "commit_message=$(git log -1 --pretty=%B)" >> "$GITHUB_OUTPUT"
-          echo "commit_name=$(github.event.head_commit.author.username)" >> "$GITHUB_OUTPUT"
+          echo "commit_message=$(git log --format=%B -n 1 ${{ github.event.after }})" >> "$GITHUB_OUTPUT"
+          echo "commit_name=$(git show -s --format='%an' ${{ github.event.after }})" >> "$GITHUB_OUTPUT"
 
       - name: notify feishu
         uses: fjogeleit/http-request-action@v1

docker/bisheng/config/config.yaml

@@ -11,11 +11,11 @@ redis_password:
 environment:
   env: dev
   uns_support: ['png','jpg','jpeg','bmp','doc', 'docx', 'ppt', 'pptx', 'xls', 'xlsx', 'txt', 'md', 'html', 'pdf', 'csv', 'tiff']
-
+              
 # admin 用户配置
 admin:
   user_name: "admin"
-  password: "1234"
+  password: "tzJkPSCZao5hEoZH"
 
 autogen_roles:
   AutoGenAssistant:

src/backend/bisheng/api/router.py

@@ -1,16 +1,15 @@
 # Router for base api
-from bisheng.api.v1 import (chat_router, endpoints_router, flow_styles_router, flows_router,
-                            knowledge_router, qa_router, report_router, server_router,
-                            skillcenter_router, user_router, validate_router, variable_router)
+from bisheng.api.v1 import (chat_router, endpoints_router, flows_router, knowledge_router,
+                            qa_router, report_router, server_router, skillcenter_router,
+                            user_router, validate_router, variable_router)
 from bisheng.api.v2 import chat_router_rpc, knowledge_router_rpc, rpc_router_rpc
 from fastapi import APIRouter
 
-router = APIRouter(prefix='/api/v1',)
+router = APIRouter(prefix='/api/v1', )
 router.include_router(chat_router)
 router.include_router(endpoints_router)
 router.include_router(validate_router)
 router.include_router(flows_router)
-router.include_router(flow_styles_router)
 router.include_router(skillcenter_router)
 router.include_router(knowledge_router)
 router.include_router(server_router)
@@ -19,7 +18,7 @@
 router.include_router(variable_router)
 router.include_router(report_router)
 
-router_rpc = APIRouter(prefix='/api/v2',)
+router_rpc = APIRouter(prefix='/api/v2', )
 router_rpc.include_router(knowledge_router_rpc)
 router_rpc.include_router(chat_router_rpc)
 router_rpc.include_router(rpc_router_rpc)

src/backend/bisheng/api/utils.py

@@ -1,11 +1,11 @@
 from bisheng.api.v1.schemas import StreamData
-from bisheng.database.base import get_session
+from bisheng.database.base import get_session, session_getter
 from bisheng.database.models.role_access import AccessType, RoleAccess
 from bisheng.database.models.variable_value import Variable
 from bisheng.graph.graph.base import Graph
 from bisheng.utils.logger import logger
 from sqlalchemy import delete
-from sqlmodel import Session, select
+from sqlmodel import select
 
 API_WORDS = ['api', 'key', 'token']
 
@@ -46,7 +46,8 @@ def build_input_keys_response(langchain_object, artifacts):
             input_keys_response['input_keys'][key] = value
     # If the object has memory, that memory will have a memory_variables attribute
     # memory variables should be removed from the input keys
-    if hasattr(langchain_object, 'memory') and hasattr(langchain_object.memory, 'memory_variables'):
+    if hasattr(langchain_object, 'memory') and hasattr(langchain_object.memory,
+                                                       'memory_variables'):
         # Remove memory variables from input keys
         input_keys_response['input_keys'] = {
             key: value
@@ -214,10 +215,10 @@ def build_flow_no_yield(graph_data: dict,
 def access_check(payload: dict, owner_user_id: int, target_id: int, type: AccessType) -> bool:
     if payload.get('role') != 'admin':
         # role_access
-        session = next(get_session())
-        role_access = session.exec(
-            select(RoleAccess).where(RoleAccess.role_id.in_(payload.get('role')),
-                                     RoleAccess.type == type.value)).all()
+        with next(get_session()) as session:
+            role_access = session.exec(
+                select(RoleAccess).where(RoleAccess.role_id.in_(payload.get('role')),
+                                         RoleAccess.type == type.value)).all()
         third_ids = [access.third_id for access in role_access]
         if owner_user_id != payload.get('user_id') and str(target_id) not in third_ids:
             return False
@@ -239,47 +240,47 @@ def get_L2_param_from_flow(
         elif node.vertex_type in {'VariableNode'}:
             variable_ids.append(node.id)
 
-    session: Session = next(get_session())
-    db_variables = session.exec(select(Variable).where(Variable.flow_id == flow_id)).all()
+    with session_getter() as session:
+        db_variables = session.exec(select(Variable).where(Variable.flow_id == flow_id)).all()
 
-    old_file_ids = {
-        variable.node_id: variable
-        for variable in db_variables if variable.value_type == 3
-    }
-    update = []
-    delete_node_ids = []
-    try:
-        for index, id in enumerate(node_id):
-            if id in old_file_ids:
-                if file_name[index] != old_file_ids.get(id).variable_name:
-                    old_file_ids.get(id).variable_name = file_name[index]
-                    update.append(old_file_ids.get(id))
-                old_file_ids.pop(id)
-            else:
-                # file type
-                db_new_var = Variable(flow_id=flow_id,
-                                      node_id=id,
-                                      variable_name=file_name[index],
-                                      value_type=3)
-                update.append(db_new_var)
-        # delete variable which not delete by edit
-        old_variable_ids = {
-            variable.node_id
-            for variable in db_variables if variable.value_type != 3
+        old_file_ids = {
+            variable.node_id: variable
+            for variable in db_variables if variable.value_type == 3
         }
+        update = []
+        delete_node_ids = []
+        try:
+            for index, id in enumerate(node_id):
+                if id in old_file_ids:
+                    if file_name[index] != old_file_ids.get(id).variable_name:
+                        old_file_ids.get(id).variable_name = file_name[index]
+                        update.append(old_file_ids.get(id))
+                    old_file_ids.pop(id)
+                else:
+                    # file type
+                    db_new_var = Variable(flow_id=flow_id,
+                                          node_id=id,
+                                          variable_name=file_name[index],
+                                          value_type=3)
+                    update.append(db_new_var)
+            # delete variable which not delete by edit
+            old_variable_ids = {
+                variable.node_id
+                for variable in db_variables if variable.value_type != 3
+            }
 
-        if old_file_ids:
-            delete_node_ids.extend(list(old_file_ids.keys()))
-
-        delete_node_ids.extend(old_variable_ids.difference(set(variable_ids)))
-
-        if update:
-            [session.add(var) for var in update]
-        if delete_node_ids:
-            session.exec(delete(Variable).where(Variable.node_id.in_(delete_node_ids)))
-        session.commit()
-        return True
-    except Exception as e:
-        logger.exception(e)
-        session.rollback()
-        return False
+            if old_file_ids:
+                delete_node_ids.extend(list(old_file_ids.keys()))
+
+            delete_node_ids.extend(old_variable_ids.difference(set(variable_ids)))
+
+            if update:
+                [session.add(var) for var in update]
+            if delete_node_ids:
+                session.exec(delete(Variable).where(Variable.node_id.in_(delete_node_ids)))
+            session.commit()
+            return True
+        except Exception as e:
+            logger.exception(e)
+            session.rollback()
+            return False

src/backend/bisheng/api/v1/__init__.py

@@ -1,6 +1,5 @@
 from bisheng.api.v1.chat import router as chat_router
 from bisheng.api.v1.endpoints import router as endpoints_router
-from bisheng.api.v1.flow_styles import router as flow_styles_router
 from bisheng.api.v1.flows import router as flows_router
 from bisheng.api.v1.knowledge import router as knowledge_router
 from bisheng.api.v1.qa import router as qa_router
@@ -16,7 +15,6 @@
     'endpoints_router',
     'validate_router',
     'flows_router',
-    'flow_styles_router',
     'skillcenter_router',
     'knowledge_router',
     'server_router',

src/backend/bisheng/api/v1/flows.py

@@ -72,7 +72,6 @@ def read_flows(*,
             sql = sql.where(Flow.status == status)
             count_sql = count_sql.where(Flow.status == status)
         total_count = session.scalar(count_sql)
-        logger.debug('flows_get end_count')
         sql = sql.order_by(Flow.update_time.desc())
         if page_num and page_size:
             sql = sql.offset((page_num - 1) * page_size).limit(page_size)

src/backend/bisheng/api/v1/knowledge.py

@@ -3,7 +3,6 @@
 import time
 from typing import List, Optional
 from uuid import uuid4
-from xml.dom.minidom import Document
 
 import requests
 from bisheng.api.utils import access_check
@@ -19,9 +18,10 @@
 from bisheng.settings import settings
 from bisheng.utils.logger import logger
 from bisheng.utils.minio_client import MinioClient
-from bisheng_langchain.document_loaders.elem_unstrcutured_loader import ElemUnstructuredLoader
-from bisheng_langchain.embeddings.host_embedding import HostEmbeddings
+from bisheng_langchain.document_loaders import ElemUnstructuredLoader
+from bisheng_langchain.embeddings import HostEmbeddings
 from bisheng_langchain.text_splitter import ElemCharacterTextSplitter
+from bisheng_langchain.vectorstores import Milvus
 from fastapi import APIRouter, BackgroundTasks, Depends, File, HTTPException, UploadFile
 from fastapi.encoders import jsonable_encoder
 from fastapi_jwt_auth import AuthJWT
@@ -30,8 +30,8 @@
                                         UnstructuredWordDocumentLoader)
 from langchain.embeddings.base import Embeddings
 from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.schema import Document
 from langchain.text_splitter import CharacterTextSplitter
-from langchain.vectorstores import Milvus
 from langchain.vectorstores.base import VectorStore
 from sqlalchemy import func, or_
 from sqlmodel import Session, select
@@ -572,6 +572,7 @@ def text_knowledge(
         documents: List[Document],
         session: Session = Depends(get_session),
 ):
+    """使用text 导入knowledge"""
     try:
         embeddings = decide_embeddings(db_knowledge.model)
         vectore_client = decide_vectorstores(db_knowledge.collection_name, 'Milvus', embeddings)