Chat on Data - Stream

README.md

@@ -6,6 +6,7 @@ The repo provides a way to upload your own data so it's ready to try end to end.
 
 ## Updates
 
+* 8/5/2023 - Added Chat Interface with "Stream" Option.  This feature allows you to stream the conversation to the client.  You will need to add `OpenAiChat`, `OpenAiChat16k`, `OpenAiEmbedding`, `OpenAiEndPoint`, `OpenAiKey`, `OpenAiApiKey`, `OpenAiService`, `OpenAiVersion`, `PineconeEnv`, `PineconeIndex`, `PineconeKey`, `RedisAddress`, `RedisPassword`, `RedisPort` property in Azure App Service (Webapp) to enable the feature for existing deployment.
 * 7/30/2023 - Removed unused Code - SummaryAndQa and Chat
 * 7/28/2023 - Started removing the Davinci model usage.  For now removed the usage from all functionality except workshop.  Refactored Summarization functionality based on the feedback to allow user to specify the prompt and pre-defined Topics to summarize it on.
 * 7/26/2023 - Remove OpenAI Playground from Developer Tools as advanced features of that are available in ChatGPT section.

Workshop/10_PibCoPilot.ipynb

@@ -197,7 +197,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [],
    "source": [

app/backend/.env.example

@@ -35,4 +35,18 @@ SECDOCPERSIST_URL="http://localhost:7071/api/SecDocPersist?code="
 PIB_URL="http://localhost:7071/api/Pib?code="
 PIBCHAT_URL="http://localhost:7071/api/PibChat?code="
 FMPKEY=""
-CHATGPT_URL="http://localhost:7071/api/OpenChatGpt?code="
+CHATGPT_URL="http://localhost:7071/api/OpenChatGpt?code="
+OpenAiKey=""
+OpenAiService=""
+OpenAiEndPoint="https://.openai.azure.com/"
+OpenAiApiKey=""
+OpenAiChat="chat"
+OpenAiChat16k="chat16k"
+OpenAiVersion="2023-05-15"
+OpenAiEmbedding="embedding"
+RedisPassword=""
+RedisAddress=""
+RedisPort="6379"
+PineconeKey="9e8a4f2b-7dd2-43be-bf19-a0183cad3a7c"
+PineconeEnv="us-east-1-aws"
+PineconeIndex="oaiembed"

app/backend/Utilities/modelHelper.py

@@ -0,0 +1,50 @@
+import tiktoken
+
+MODELS_2_TOKEN_LIMITS = {
+    "gpt-35-turbo": 4000,
+    "gpt-3.5-turbo": 4000,
+    "gpt-35-turbo-16k": 16000,
+    "gpt-3.5-turbo-16k": 16000,
+    "gpt-4": 8100,
+    "gpt-4-32k": 32000
+}
+
+AOAI_2_OAI = {
+    "gpt-35-turbo": "gpt-3.5-turbo",
+    "gpt-35-turbo-16k": "gpt-3.5-turbo-16k"
+}
+
+def getTokenLimit(modelId: str) -> int:
+    if modelId not in MODELS_2_TOKEN_LIMITS:
+        raise ValueError("Expected model gpt-35-turbo and above")
+    return MODELS_2_TOKEN_LIMITS.get(modelId)
+
+
+def numTokenFromMessages(message: dict[str, str], model: str) -> int:
+    """
+    Calculate the number of tokens required to encode a message.
+    Args:
+        message (dict): The message to encode, represented as a dictionary.
+        model (str): The name of the model to use for encoding.
+    Returns:
+        int: The total number of tokens required to encode the message.
+    Example:
+        message = {'role': 'user', 'content': 'Hello, how are you?'}
+        model = 'gpt-3.5-turbo'
+        numTokenFromMessages(message, model)
+        output: 11
+    """
+    encoding = tiktoken.encoding_for_model(getOaiChatModel(model))
+    num_tokens = 2  # For "role" and "content" keys
+    for key, value in message.items():
+        num_tokens += len(encoding.encode(value))
+    return num_tokens
+
+
+def getOaiChatModel(aoaimodel: str) -> str:
+    message = "Expected Azure OpenAI ChatGPT model name"
+    if aoaimodel == "" or aoaimodel is None:
+        raise ValueError(message)
+    if aoaimodel not in AOAI_2_OAI and aoaimodel not in MODELS_2_TOKEN_LIMITS:
+        raise ValueError(message)
+    return AOAI_2_OAI.get(aoaimodel) or aoaimodel

app/backend/app.py

@@ -1,4 +1,4 @@
-from flask import Flask, request, jsonify, make_response
+from flask import Flask, request, jsonify, make_response, Response
 import requests
 import json
 from dotenv import load_dotenv
@@ -21,6 +21,7 @@
 from azure.cosmos import CosmosClient, PartitionKey
 from Utilities.fmp import *
 from distutils.util import strtobool
+from Utilities.ChatGptStream import *
 
 load_dotenv()
 app = Flask(__name__)
@@ -203,6 +204,93 @@ def chat():
         logging.exception("Exception in /chat")
         return jsonify({"error": str(e)}), 500
 
+def formatNdJson(r):
+    for data in r:
+        yield json.dumps(data).replace("\n", "\\n") + "\n"
+
+@app.route("/chatStream", methods=["POST"])
+def chatStream():
+    indexType=request.json["indexType"]
+    indexNs=request.json["indexNs"]
+    postBody=request.json["postBody"]
+
+    logging.info(f"indexType: {indexType}")
+    logging.info(f"indexNs: {indexNs}")
+
+    try:
+
+        OpenAiKey = os.environ['OpenAiKey']
+        OpenAiVersion = os.environ['OpenAiVersion']
+        OpenAiChat = os.environ['OpenAiChat']
+        OpenAiService = os.environ['OpenAiService']
+
+        if "OpenAiChat16k" in os.environ: 
+            OpenAiChat16k = os.getenv('OpenAiChat16k')
+        else:
+            OpenAiChat16k = "chat16k"
+
+        if "OpenAiApiKey" in os.environ: 
+            OpenAiApiKey = os.getenv('OpenAiApiKey')
+        else:
+            OpenAiApiKey = ""
+
+        if "SEARCHKEY" in os.environ: 
+            SearchKey = os.environ['SEARCHKEY']
+        else:
+            SearchKey = ""
+
+        if "SEARCHSERVICE" in os.environ: 
+            SearchService = os.environ['SEARCHSERVICE']
+        else:
+            SearchService = ""
+
+        if "OpenAiEmbedding" in os.environ: 
+            OpenAiEmbedding = os.environ['OpenAiEmbedding']
+        else:
+            OpenAiEmbedding = "embedding"
+
+        if "RedisAddress" in os.environ: 
+            RedisAddress = os.environ['RedisAddress']
+        else:
+            RedisAddress = ""
+
+        if "RedisPort" in os.environ: 
+            RedisPort = os.environ['RedisPort']
+        else:
+            RedisPort = ""
+
+        if "RedisPassword" in os.environ: 
+            RedisPassword = os.environ['RedisPassword']
+        else:
+            RedisPassword = "embedding"
+
+        if "PineconeEnv" in os.environ: 
+            PineconeEnv = os.environ['PineconeEnv']
+        else:
+            PineconeEnv = ""
+
+        if "PineconeKey" in os.environ: 
+            PineconeKey = os.environ['PineconeKey']
+        else:
+            PineconeKey = ""
+
+        if "PineconeIndex" in os.environ: 
+            PineconeIndex = os.environ['PineconeIndex']
+        else:
+            PineconeIndex = ""
+
+        # data = postBody
+        # params = {'indexType': indexType, "indexNs": indexNs }
+        # resp = requests.post(url, params=params, data=json.dumps(data), headers=headers)
+        chatStream = ChatGptStream(OpenAiService, OpenAiKey, OpenAiVersion, OpenAiChat, OpenAiChat16k, OpenAiApiKey, OpenAiEmbedding,
+                                    SearchService, SearchKey, RedisAddress, RedisPort, RedisPassword,
+                                    PineconeKey, PineconeEnv, PineconeIndex)
+        r = chatStream.run(indexType=indexType, indexNs=indexNs, postBody=postBody)
+        return Response(formatNdJson(r), mimetype='text/event-stream')
+    except Exception as e:
+        logging.exception("Exception in /chatStream")
+        return jsonify({"error": str(e)}), 500
+
 @app.route("/chatGpt", methods=["POST"])
 def chatGpt():
     indexType=request.json["indexType"]

app/backend/requirements.txt

@@ -8,5 +8,11 @@ azure-common==1.1.28
 azure-core==1.26.3
 azure-cosmos==4.4.0b1
 azure-identity==1.12.0
+openai==0.27.6
+tenacity==8.2.2
+tiktoken==0.3.0
+azure-core==1.26.3
+redis==4.4.2
+pinecone-client==2.2.1
 #azure-search-documents==11.4.0b4
 ./azure_search_documents-11.4.0b4-py3-none-any.whl

app/frontend/src/api/api.ts

@@ -343,7 +343,7 @@ export async function askTaskAgentApi(options: AskRequest): Promise<AskResponse>
   return parsedResponse.values[0].data
 
 }
-export async function chatGptApi(options: ChatRequest, indexNs: string, indexType:string): Promise<AskResponse> {
+export async function chat(options: ChatRequest, indexNs: string, indexType:string): Promise<AskResponse> {
     const response = await fetch('/chat' , {
         method: "POST",
         headers: {
@@ -384,6 +384,41 @@ export async function chatGptApi(options: ChatRequest, indexNs: string, indexTyp
     }
     return parsedResponse.values[0].data;
 }
+export async function chatStream(options: ChatRequest, indexNs: string, indexType:string): Promise<Response> {
+    return await fetch('/chatStream' , {
+      method: "POST",
+      headers: {
+          "Content-Type": "application/json"
+      },
+      body: JSON.stringify({
+        indexType:indexType,
+        indexNs: indexNs,
+        postBody: {
+          values: [
+            {
+              recordId: 0,
+              data: {
+                history: options.history,
+                approach: 'rrr',
+                overrides: {
+                  top: options.overrides?.top,
+                  temperature: options.overrides?.temperature,
+                  promptTemplate: options.overrides?.promptTemplate,
+                  suggest_followup_questions: options.overrides?.suggestFollowupQuestions,
+                  embeddingModelType: options.overrides?.embeddingModelType,
+                  firstSession:options.overrides?.firstSession,
+                  session:options.overrides?.session,
+                  sessionId:options.overrides?.sessionId,
+                  deploymentType: options.overrides?.deploymentType,
+                  chainType: options.overrides?.chainType,
+                }
+              }
+            }
+          ]
+        }
+      })
+  });
+}
 export async function chatGpt(options: ChatRequest, indexNs: string, indexType:string): Promise<AskResponse> {
   const response = await fetch('/chatGpt' , {
       method: "POST",