Streaming openai api support

scripts/openai_server_demo/README.md

@@ -8,7 +8,7 @@
 
 安装依赖
 ``` shell
-pip install fastapi uvicorn shortuuid
+pip install fastapi uvicorn shortuuid sse_starlette
 ```
 
 启动脚本
@@ -137,7 +137,7 @@ curl http://localhost:19327/v1/chat/completions \
   -H "Content-Type: application/json" \
   -d '{
     "messages": [
-      {"role": "user","message": "给我讲一些有关杭州的故事吧"}
+      {"role": "user","content": "给我讲一些有关杭州的故事吧"}
     ],
     "repetition_penalty": 1.0
   }'
@@ -179,9 +179,9 @@ curl http://localhost:19327/v1/chat/completions \
   -H "Content-Type: application/json" \
   -d '{
     "messages": [
-      {"role": "user","message": "给我讲一些有关杭州的故事吧"},
-      {"role": "assistant","message": "好的，请问您对杭州有什么特别的偏好吗？"},
-      {"role": "user","message": "我比较喜欢和西湖，可以给我讲一下西湖吗"}
+      {"role": "user","content": "给我讲一些有关杭州的故事吧"},
+      {"role": "assistant","content": "好的，请问您对杭州有什么特别的偏好吗？"},
+      {"role": "user","content": "我比较喜欢和西湖，可以给我讲一下西湖吗"}
     ],
     "repetition_penalty": 1.0
   }'
@@ -246,6 +246,8 @@ json返回体：
 
 `do_sample`: 启用随机采样策略。默认为true。
 
+`stream`: OpenAI格式的流式返回。默认为false，设置为true时，会按照OpenAI的格式流式返回数据，可以作为任意基于ChatGPT的应用的后端。
+
 ### 文本嵌入向量（text embedding）
 
 文本嵌入向量有很多作用，包括但不限于基于大型文档问答、总结一本书中的内容、为大语言模型找到与当前用户输入最相近的记忆等等。

scripts/openai_server_demo/openai_api_protocol.py

@@ -1,10 +1,11 @@
-from typing import Optional, List, Dict, Any, Union
+from typing import Optional, List, Dict, Any, Union, Literal
 
 import time
 
 import shortuuid
 from pydantic import BaseModel, Field
 
+
 class ChatCompletionRequest(BaseModel):
     model: str = "chinese-llama-alpaca-2"
     messages: Union[str, List[Dict[str, str]]]
@@ -26,17 +27,30 @@ class ChatMessage(BaseModel):
     content: str
 
 
+class DeltaMessage(BaseModel):
+    role: Optional[Literal["user", "assistant", "system"]] = None
+    content: Optional[str] = None
+
+
 class ChatCompletionResponseChoice(BaseModel):
     index: int
     message: ChatMessage
 
 
+class ChatCompletionResponseStreamChoice(BaseModel):
+    index: int
+    delta: DeltaMessage
+    finish_reason: Optional[Literal["stop", "length"]]
+
+
 class ChatCompletionResponse(BaseModel):
     id: str = Field(default_factory=lambda: f"chatcmpl-{shortuuid.random()}")
     object: str = "chat.completion"
     created: int = Field(default_factory=lambda: int(time.time()))
     model: str = "chinese-llama-alpaca-2"
-    choices: List[ChatCompletionResponseChoice]
+    choices: List[
+        Union[ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice]
+    ]
 
 
 class EmbeddingsRequest(BaseModel):
@@ -76,6 +90,5 @@ class CompletionResponse(BaseModel):
     id: Optional[str] = Field(default_factory=lambda: f"cmpl-{shortuuid.random()}")
     object: Optional[str] = "text_completion"
     created: Optional[int] = Field(default_factory=lambda: int(time.time()))
-    model: Optional[str] = 'chinese-llama-alpaca-2'
+    model: Optional[str] = "chinese-llama-alpaca-2"
     choices: List[CompletionResponseChoice]
-