add flask example file

peng3307165 · peng3307165 · commit a75880c65917 · 2023-11-09T14:53:12.000+08:00
diff --git a/examples/__init__.py b/examples/__init__.py
@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+"""
+   Alipay.com Inc.
+   Copyright (c) 2004-2023 All Rights Reserved.
+   ------------------------------------------------------
+   File Name : __init__.py.py
+   Author : fuhui.phe
+   Create Time : 2023/11/9 14:21
+   Description : description what the main function of this file
+   Change Activity: 
+        version0 : 2023/11/9 14:21 by fuhui.phe  init
+"""
diff --git a/examples/flask/__init__.py b/examples/flask/__init__.py
@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-
diff --git a/examples/flask/data_insert.py b/examples/flask/data_insert.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+import json
+import requests
+
+
+def run():
+    url = 'http://127.0.0.1:5000/modelcache'
+    type = 'insert'
+    scope = {"model": "CODEGPT-1109"}
+    chat_info = [{"query": [{"role": "system", "content": "你是一个python助手"}, {"role": "user", "content": "hello"}],
+                  "answer": "你好，我是智能助手，请问有什么能帮您!"}]
+    data = {'type': type, 'scope': scope, 'chat_info': chat_info}
+    headers = {"Content-Type": "application/json"}
+    res = requests.post(url, headers=headers, json=json.dumps(data))
+    res_text = res.text
+    print('res_text: {}'.format(res_text))
+
+
+if __name__ == '__main__':
+    run()
diff --git a/examples/flask/data_query.py b/examples/flask/data_query.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+import json
+import requests
+
+
+def run():
+    url = 'http://127.0.0.1:5000/modelcache'
+    type = 'query'
+    scope = {"model": "CODEGPT-1109"}
+    query = [{"role": "system", "content": "你是一个python助手"}, {"role": "user", "content": "hello"}]
+    data = {'type': type, 'scope': scope, 'query': query}
+
+    headers = {"Content-Type": "application/json"}
+    res = requests.post(url, headers=headers, json=json.dumps(data))
+    res_text = res.text
+    print('res_text: {}'.format(res_text))
+
+
+if __name__ == '__main__':
+    run()
diff --git a/examples/flask/data_query_long.py b/examples/flask/data_query_long.py
@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+import json
+import requests
+
+
+def run():
+    url = 'http://127.0.0.1:5000/modelcache'
+    type = 'query'
+    scope = {"model": "CODEGPT-1109"}
+    system_conten = """
+<|role_start|>system<|role_end|>你是python助手, 你必须提供中立的、无害的答案帮助用户解决代码相关的问题，在回答用户问题过程中，你必须遵守如下准则：
+以用户选择的语言（如中文、英语）进行理解和交流
+回答应该是信息丰富的、直观的、合乎逻辑的和可操作的
+不泄漏模型的架构和内部实现细节
+不收集、存储或共享用户的个人信息或敏感信息，不使用未经许可的数据集，遵守数据集的许可协议和规定，并且不能改变数据集的原始内容
+不能生成涉及诽谤、歧视、侵犯知识产权等的内容，不违反法律和道德规范
+不能通过生成内容引起身体或精神上的伤害，例如，不包含暴力、恐怖、色情等内容
+不能使用或生成不准确、误导或伪造的信息，不能改变数据集的原始内容
+努力消除或减少内容中的偏见和歧视，包括种族、性别、性取向、宗教和政治观点等方面的偏见
+回答不会伤害人类、损害社会、危害环境和生态系统等方面
+<|end|><|role_start|>human<|role_end|>Analyze data from a survey and create visualizations to present the results.
+<|end|><|role_start|>bot<|role_end|>Sure thing! Let me just check if I can access the survey data.
+<|end|><|role_start|>human<|role_end|>What kind of visualizations can you create?
+<|end|><|role_start|>bot<|role_end|>I can create various types of visualizations such as bar charts, line graphs, scatter plots, pie charts, and more. I can also customize the visualizations according to your requirements.
+<|end|>
+    """
+    user_content = """xP3(Crosslingual Public Pool of Prompts)是一个多语言指令数据集，由46种语言的16种不同的自然语言任务组成。数据集中的每个实例都有两个组件:“inputs”和“targets”。“inputs”是一种自然语言的任务描述。“targets”是正确遵循“inputs”指令的文本结果。xP3中的原始数据来自三个来源:英语指令数据集P3, P3中的4个英语未见任务(例如，翻译，程序合成)和30个多语言NLP数据集。作者通过从PromptSource中采样人工编写的任务模板，然后填充模板，将不同的NLP任务转换为统一的形式化，构建了xP3数据集。Unnatural Instructions是一个包含大约24万个实例的指令数据集，使用InstructGPT构建。数据集中的每个实例都有四个组件: INSTRUCTION,INPUT, CONSTRAINTS,OUTPUT。“INSTRUCTION”是用自然语言对教学任务的描述。“INPUT”是自然语言中的参数，用于实例化指令任务。“CONSTRAINTS”是任务输出空间的限制。“OUTPUT”是在给定输入参数和约束条件下正确执行指令的文本序列。
+"""
+
+    query = [{"role": "system", "content": system_conten}, {"role": "user", "content": user_content}]
+    data = {'type': type, 'scope': scope, 'query': query}
+
+    headers = {"Content-Type": "application/json"}
+    res = requests.post(url, headers=headers, json=json.dumps(data))
+    res_text = res.text
+    print('res_text: {}'.format(res_text))
+
+
+if __name__ == '__main__':
+    run()
diff --git a/flask4modelcache.py b/flask4modelcache.py
@@ -0,0 +1,178 @@
+# -*- coding: utf-8 -*-
+import json
+from flask import Flask, request
+import logging
+from datetime import datetime
+import configparser
+import time
+import json
+from modelcache import cache
+from modelcache.adapter import adapter
+from modelcache.manager import CacheBase, VectorBase, get_data_manager
+from modelcache.similarity_evaluation.distance import SearchDistanceEvaluation
+from modelcache.processor.pre import query_multi_splicing
+from modelcache.processor.pre import insert_multi_splicing
+from concurrent.futures import ThreadPoolExecutor
+from modelcache.utils.model_filter import model_blacklist_filter
+from modelcache.embedding import Data2VecAudio
+# from modelcache.maya_embedding_service.maya_embedding_service import get_cache_embedding_text2vec
+
+
+# 创建一个Flask实例
+app = Flask(__name__)
+
+
+def response_text(cache_resp):
+    return cache_resp['data']
+
+
+def save_query_info(result, model, query, delta_time_log):
+    cache.data_manager.save_query_resp(result, model=model, query=json.dumps(query, ensure_ascii=False),
+                                       delta_time=delta_time_log)
+
+
+def response_hitquery(cache_resp):
+    return cache_resp['hitQuery']
+
+
+data2vec = Data2VecAudio()
+mysql_config = configparser.ConfigParser()
+mysql_config.read('modelcache/config/mysql_config.ini')
+milvus_config = configparser.ConfigParser()
+milvus_config.read('modelcache/config/milvus_config.ini')
+data_manager = get_data_manager(CacheBase("mysql", config=mysql_config),
+                                VectorBase("milvus", dimension=data2vec.dimension, milvus_config=milvus_config))
+
+
+cache.init(
+    embedding_func=data2vec.to_embeddings,
+    data_manager=data_manager,
+    similarity_evaluation=SearchDistanceEvaluation(),
+    query_pre_embedding_func=query_multi_splicing,
+    insert_pre_embedding_func=insert_multi_splicing,
+)
+
+# cache.set_openai_key()
+global executor
+executor = ThreadPoolExecutor(max_workers=6)
+
+
+@app.route('/welcome')
+def first_flask():  # 视图函数
+    return 'hello, modelcache!'
+
+
+@app.route('/modelcache', methods=['GET', 'POST'])
+def user_backend():
+    try:
+        if request.method == 'POST':
+            request_data = request.json
+        elif request.method == 'GET':
+            request_data = request.args
+        param_dict = json.loads(request_data)
+    except Exception as e:
+        result = {"errorCode": 101, "errorDesc": str(e), "cacheHit": False, "delta_time": 0, "hit_query": '',
+                  "answer": ''}
+        cache.data_manager.save_query_resp(result, model='', query='', delta_time=0)
+        return json.dumps(result)
+
+    # param parsing
+    try:
+        request_type = param_dict.get("type")
+
+        scope = param_dict.get("scope")
+        if scope is not None:
+            model = scope.get('model')
+            model = model.replace('-', '_')
+            model = model.replace('.', '_')
+        query = param_dict.get("query")
+        chat_info = param_dict.get("chat_info")
+        if request_type is None or request_type not in ['query', 'insert', 'detox', 'remove']:
+            result = {"errorCode": 102,
+                      "errorDesc": "type exception, should one of ['query', 'insert', 'detox', 'remove']",
+                      "cacheHit": False, "delta_time": 0, "hit_query": '', "answer": ''}
+            cache.data_manager.save_query_resp(result, model=model, query='', delta_time=0)
+            return json.dumps(result)
+    except Exception as e:
+        result = {"errorCode": 103, "errorDesc": str(e), "cacheHit": False, "delta_time": 0, "hit_query": '',
+                  "answer": ''}
+        return json.dumps(result)
+
+    # model filter
+    filter_resp = model_blacklist_filter(model, request_type)
+    if isinstance(filter_resp, dict):
+        return json.dumps(filter_resp)
+
+    if request_type == 'query':
+        try:
+            response = adapter.ChatCompletion.create_query(
+                scope={"model": model},
+                query=query
+            )
+            if response is None:
+                result = {"errorCode": 0, "errorDesc": '', "cacheHit": False, "delta_time": delta_time, "hit_query": '',
+                          "answer": ''}
+            elif response in ['adapt_query_exception']:
+                result = {"errorCode": 201, "errorDesc": response, "cacheHit": False, "delta_time": delta_time,
+                          "hit_query": '', "answer": ''}
+            else:
+                answer = response_text(response)
+                hit_query = response_hitquery(response)
+                result = {"errorCode": 0, "errorDesc": '', "cacheHit": True, "delta_time": delta_time,
+                          "hit_query": hit_query, "answer": answer}
+            future = executor.submit(save_query_info, result, model, query, delta_time_log)
+        except Exception as e:
+            result = {"errorCode": 202, "errorDesc": e, "cacheHit": False, "delta_time": 0,
+                      "hit_query": '', "answer": ''}
+            logging.info('result: {}'.format(result))
+
+        return json.dumps(result, ensure_ascii=False)
+
+    if request_type == 'insert':
+        try:
+            try:
+                response = adapter.ChatCompletion.create_insert(
+                    model=model,
+                    chat_info=chat_info
+                )
+            except Exception as e:
+                result = {"errorCode": 303, "errorDesc": e, "writeStatus": "exception"}
+                return json.dumps(result, ensure_ascii=False)
+
+            if response in ['adapt_insert_exception']:
+                result = {"errorCode": 301, "errorDesc": response, "writeStatus": "exception"}
+            elif response == 'success':
+                result = {"errorCode": 0, "errorDesc": "", "writeStatus": "success"}
+            else:
+                result = {"errorCode": 302, "errorDesc": response,
+                          "writeStatus": "exception"}
+            return json.dumps(result, ensure_ascii=False)
+        except Exception as e:
+            result = {"errorCode": 304, "errorDesc": e, "writeStatus": "exception"}
+            return json.dumps(result, ensure_ascii=False)
+
+    if request_type == 'remove':
+        remove_type = param_dict.get("remove_type")
+        id_list = param_dict.get("id_list", [])
+
+        response = adapter.ChatCompletion.create_remove(
+            model=model,
+            remove_type=remove_type,
+            id_list=id_list
+        )
+
+        if not isinstance(response, dict):
+            result = {"errorCode": 401, "errorDesc": "", "response": response, "removeStatus": "exception"}
+            return json.dumps(result)
+
+        state = response.get('status')
+
+        if state == 'success':
+            result = {"errorCode": 0, "errorDesc": "", "response": response, "writeStatus": "success"}
+        else:
+            result = {"errorCode": 402, "errorDesc": "", "response": response, "writeStatus": "exception"}
+        return json.dumps(result)
+
+
+if __name__ == '__main__':
+    app.run(host='0.0.0.0', port=5000, debug=True)
diff --git a/modelcache/embedding/data2vec.py b/modelcache/embedding/data2vec.py
@@ -5,7 +5,6 @@
 import torch
 from transformers import BertTokenizer, BertModel
 from modelcache.embedding.base import BaseEmbedding
-# from modelcache.utils.env_config import get_data2vec_model
 
 
 def mean_pooling(model_output, attention_mask):
