Feat/0.3.1

.github/workflows/release.yml

@@ -27,7 +27,7 @@ jobs:
       - name: Process git message
         id: process_message
         run: |
-          value=$(echo "${{ github.event.head_commit.message }}" | sed -e ':a' -e 'N' -e '$!ba' -e 's/\n/\\\\n/g')
+          value=$(echo "${{ github.event.head_commit.message }}" | sed -e ':a' -e 'N' -e '$!ba' -e 's/\n/%0A/g' )
           echo "message=${value}" >> $GITHUB_ENV
         shell: bash
 
@@ -38,6 +38,9 @@ jobs:
           method: 'POST'
           data: '{"msg_type":"post","content":{"post":{"zh_cn":{"title": "${{github.event.pusher.name}}提交代码，开始编译",  "content": [[{"tag":"text","text":"发布功能："},{"tag":"text","text":"${{ env.message }}"}]]}}}}'
 
+      - name: Set Environment Variable
+        run: echo "RELEASE_VERSION=99.99.99" >> $GITHUB_ENV
+
       - name: Set python version 3.8
         uses: actions/setup-python@v1
         with:
@@ -80,6 +83,7 @@ jobs:
       - name: build lock
         run: |
           cd ./src/backend
+          sed -i 's/^bisheng_langchain.*/bisheng_langchain = "'$RELEASE_VERSION'"/g' pyproject.toml
           poetry source add --priority=supplemental foo http://${{ secrets.NEXUS_PUBLIC }}:${{ secrets.NEXUS_PUBLIC_PASSWORD }}@${{ env.PY_NEXUS }}/repository/pypi-group/simple
           poetry lock
           cd ../../
@@ -136,4 +140,4 @@ jobs:
         with:
           url: ' https://open.feishu.cn/open-apis/bot/v2/hook/2cfe0d8d-647c-4408-9f39-c59134035c4b'
           method: 'POST'
-          data: '{"msg_type":"text","content":{"text":"release 部署成功"}}'
+          data: '{"msg_type":"text","content":{"text":"release 部署成功"}}'

docker/bisheng-uns/config.yaml

@@ -0,0 +1,56 @@
+# 可根据loguru的文档配置不同 handlers
+logger_conf:
+  # 默认输出到控制台的日志级别, 大于等于此级别都会输出
+  level: DEBUG
+  # 默认输出格式
+  format: '[{time:YYYY-MM-DD HH:mm:ss.SSSSSS}] [{level.name} process-{process.id}-{thread.id} {name}:{line}] - trace={extra[trace_id]} {message}'
+  # 参考loguru.add()中的参数可以配置多个handler
+  handlers:
+      # 文件路径，支持插入一些系统环境变量，若环境变量不存在则置空。例如 HOSTNAME: 主机名。后端会处理环境变量的替换
+    - sink: "/app/logs/bisheng_uns.log"
+      # 日志级别
+      level: INFO
+      # 和原生不一样，后端会将配置使用eval()执行转为函数用来过滤特定日志级别。推荐lambda
+      # filter: "lambda record: record['level'].name == 'INFO'"
+      # 日志格式化函数，extra内支持trace_id
+      format: "[{time:YYYY-MM-DD HH:mm:ss.SSSSSS}]|{level}|BISHENG|{extra[trace_id]}|{process.id}|{thread.id}|{message}"
+      # 每天的几点进行切割
+      rotation: "00:00"
+      retention: "3 Days"
+    - sink: "/app/logs/err-v0-BISHENG-UNS-{HOSTNAME}.log"
+      level: ERROR
+      filter: "lambda record: record['level'].name == 'ERROR'"
+      format: "[{time:YYYY-MM-DD HH:mm:ss.SSSSSS}]|{level}|BISHENG|{extra[trace_id]}||{process.id}|{thread.id}|||#EX_ERR:POS={name},line {line},ERR=500,EMSG={message}"
+      rotation: "00:00"
+      retention: "3 Days"
+
+# pdf解析需要用到的模型配置, 配置了rt_server环境变量的话会替换为对应的地址
+pdf_model_params:
+  layout_ep: "http://192.168.106.12:9001/v2.1/models/elem_layout_v1/infer"
+  cell_model_ep: "http://192.168.106.12:9001/v2.1/models/elem_table_cell_detect_v1/infer"
+  rowcol_model_ep: "http://192.168.106.12:9001/v2.1/models/elem_table_rowcol_detect_v1/infer"
+  table_model_ep: "http://192.168.106.12:9001/v2.1/models/elem_table_detect_v1/infer"
+  ocr_model_ep: "http://192.168.106.12:9001/v2.1/models/elem_ocr_collection_v3/infer"
+
+
+# ocr识别需要的配置项
+ocr_conf:
+  params:
+    sort_filter_boxes: true,
+    enable_huarong_box_adjust: true,
+    rotateupright: false,
+    support_long_image_segment: true,
+    split_long_sentence_blank: true
+  scene_mapping:
+    print:
+      det: general_text_det_mrcnn_v2.0
+      recog: transformer-blank-v0.2-faster
+    hand:
+      det: general_text_det_mrcnn_v2.0
+      recog: transformer-hand-v1.16-faster
+    print_recog:
+      recog: transformer-blank-v0.2-faster
+    hand_recog:
+      recog: transformer-hand-v1.16-faster
+    det:
+      det: general_text_det_mrcnn_v2.0

docker/bisheng/config/config.yaml

@@ -24,3 +24,27 @@ environment:
   env: dev
   uns_support: ['png','jpg','jpeg','bmp','doc', 'docx', 'ppt', 'pptx', 'xls', 'xlsx', 'txt', 'md', 'html', 'pdf', 'csv', 'tiff']
 
+# 可根据loguru的文档配置不同 handlers
+logger_conf:
+  # 默认输出到sys.stdout的日志级别, 大于等于此级别都会输出
+  level: DEBUG
+  # 默认输出格式
+  format: '<level>[{time:YYYY-MM-DD HH:mm:ss.SSSSSS}] [{level.name} process-{process.id}-{thread.id} {name}:{line}]</level> - <level>trace={extra[trace_id]} {message}</level>'
+  # 参考loguru.add()中的参数可以配置多个handler
+  handlers:
+      # 文件路径，支持插入一些系统环境变量，若环境变量不存在则置空。例如 HOSTNAME: 主机名。后端会处理环境变量的替换
+    - sink: "/app/data/bisheng.log"
+      # 日志级别
+      level: INFO
+      # 日志格式化函数，extra内支持trace_id
+      format: "[{time:YYYY-MM-DD HH:mm:ss.SSSSSS}]|{level}|BISHENG|{extra[trace_id]}|{process.id}|{thread.id}|{message}"
+      # 每天的几点进行切割
+      rotation: "00:00"
+      retention: "3 Days"
+    - sink: "/app/data/err-v0-BISHENG-{HOSTNAME}.log"
+      level: ERROR
+      # 和原生不一样，后端会将配置使用eval()执行转为函数用来过滤特定日志级别。推荐lambda
+      filter: "lambda record: record['level'].name == 'ERROR'"
+      format: "[{time:YYYY-MM-DD HH:mm:ss.SSSSSS}]|{level}|BISHENG|{extra[trace_id]}||{process.id}|{thread.id}|||#EX_ERR:POS={name},line {line},ERR=500,EMSG={message}"
+      rotation: "00:00"
+      retention: "3 Days"

docker/docker-compose-model.yml

@@ -65,6 +65,8 @@ services:
     environment:
       rt_server: bisheng-rt:9001
       TZ: Asia/Shanghai
+    volumes:
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/bisheng-uns/config.yaml:/opt/bisheng-unstructured/bisheng_unstructured/config/config.yaml
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:10001/health"]
       interval: 30s

src/backend/bisheng/api/errcode/assistant.py

@@ -20,3 +20,23 @@ class AssistantNameRepeatError(BaseErrorCode):
 class AssistantNotEditError(BaseErrorCode):
     Code: int = 10403
     Msg: str = '助手已上线，不可编辑'
+
+
+class ToolTypeRepeatError(BaseErrorCode):
+    Code: int = 10410
+    Msg: str = '工具名称已存在'
+
+
+class ToolTypeEmptyError(BaseErrorCode):
+    Code: int = 10411
+    Msg: str = '工具下的API不能为空'
+
+
+class ToolTypeNotExistsError(BaseErrorCode):
+    Code: int = 10412
+    Msg: str = '工具不存在'
+
+
+class ToolTypeIsPresetError(BaseErrorCode):
+    Code: int = 10413
+    Msg: str = '预置工具类别不可删除'

src/backend/bisheng/api/errcode/flow.py

@@ -0,0 +1,28 @@
+from bisheng.api.errcode.base import BaseErrorCode
+
+
+# RT服务相关的返回错误码，功能模块代码：105
+class NotFoundVersionError(BaseErrorCode):
+    Code: int = 10500
+    Msg: str = '未找到技能版本信息'
+
+
+class CurVersionDelError(BaseErrorCode):
+    Code: int = 10501
+    Msg: str = '当前正在使用版本无法删除'
+
+
+class VersionNameExistsError(BaseErrorCode):
+    Code: int = 10502
+    Msg: str = '版本名已存在'
+
+
+class NotFoundFlowError(BaseErrorCode):
+    Code: int = 10520
+    Msg: str = '技能不存在'
+
+
+class FlowOnlineEditError(BaseErrorCode):
+    Code: int = 10521
+    Msg: str = '技能已上线，不可编辑'
+