Markdown upload support

akshata29 · Aug 20, 2023 · 55cd798 · 55cd798
1 parent dd3ccc3
commit 55cd798
Show file tree

Hide file tree

Showing 36 changed files with 352 additions and 99 deletions.
diff --git a/.gitignore b/.gitignore
@@ -147,3 +147,4 @@ static/
 config.json
 local.settings.json
 Workshop/Data/PDF/Bumble Bee.pdf
+Workshop/Data/Markdown/security.zip
diff --git a/README.md b/README.md
@@ -6,6 +6,7 @@ The repo provides a way to upload your own data so it's ready to try end to end.
 
 ## Updates
 
+* 8/20/2023 - Added support for the Markdown files (as zip file) and removed the chunk_size=1 from Azure OpenAiEmbedding
 * 8/11/2023 - Fixed the issue with Streaming Chat feature.
 * 8/10/2023 - **Breaking Changes** - Refactored all code to use `OpenAiEndPoint` configuration value instead of `OpenAiService`.  It is to support the best practices as they are outlined in [Enterprise Logging](https://github.com/Azure-Samples/openai-python-enterprise-logging) via Azure API Management. Your `OpenAiEndPoint` if using APIM will be API Gateway URL and the `OpenAiKey` will be the Product/Unlimited key.   If not using APIM, you don't need to change the key, but ensure `OpenAiEndPoint` is fully qualified URL of your AOAI deployment.  `OpenAiService` is no longer used.  Changes did impact the working on Chat on Stream feature, so it's disabled for now and will be enabled once tested and fixed.
 * 8/9/2023 - Added Function calling in the ChatGpt interface as checkbox.  Sample demonstrate ability to call functions.  Currently [Weather API](https://rapidapi.com/apishub/api/yahoo-weather5), [Stock API](https://rapidapi.com/alphavantage/api/alpha-vantage) and Bing Search is supported. Function calling is in preview and supported only from "API Version" of "2023-07-01-preview", so make sure you update existing deployment to use that version.  Details on calling [Functions](https://github.com/Azure-Samples/openai/blob/main/Basic_Samples/Functions/working_with_functions.ipynb).  For existing deployment add `WeatherEndPoint`, `WeatherHost`, `StockEndPoint`, `StockHost` and `RapidApiKey` configuration to Azure Function App.

diff --git a/Workshop/10_PibCoPilot.ipynb b/Workshop/10_PibCoPilot.ipynb
@@ -108,7 +108,7 @@
     "    openai.api_type = \"azure\"\n",
     "    openai.api_key = OpenAiKey\n",
     "    openai.api_version = OpenAiVersion\n",
-    "    openai.api_base = OpenAiBase\n",
+    "    openai.api_base = OpenAiEndPoint\n",
     "    \n",
     "    llm = AzureChatOpenAI(\n",
     "                openai_api_base=openai.api_base,\n",
@@ -120,7 +120,7 @@
     "                max_tokens=tokenLength)\n",
     "    \n",
     "    logging.info(\"LLM Setup done\")\n",
-    "    embeddings = OpenAIEmbeddings(deployment=OpenAiEmbedding, chunk_size=1, openai_api_key=OpenAiKey)\n",
+    "    embeddings = OpenAIEmbeddings(deployment=OpenAiEmbedding, openai_api_key=OpenAiKey, openai_api_type=\"azure\")\n",
     "elif embeddingModelType == \"openai\":\n",
     "    openai.api_type = \"open_ai\"\n",
     "    openai.api_base = \"https://api.openai.com/v1\"\n",

diff --git a/Workshop/11_Prospectus.ipynb b/Workshop/11_Prospectus.ipynb
@@ -56,7 +56,6 @@
     "from langchain.chains import LLMChain\n",
     "# Import required libraries\n",
     "# Import required libraries\n",
-    "from langchain.embeddings.openai import OpenAIEmbeddings\n",
     "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
     "from langchain.document_loaders import (\n",
     "    PDFMinerLoader,\n",
@@ -82,7 +81,7 @@
     "    openai.api_type = \"azure\"\n",
     "    openai.api_key = OpenAiKey\n",
     "    openai.api_version = OpenAiVersion\n",
-    "    openai.api_base = OpenAiBase\n",
+    "    openai.api_base = OpenAiEndPoint\n",
     "\n",
     "    llm = AzureChatOpenAI(\n",
     "                openai_api_base=openai.api_base,\n",
@@ -94,7 +93,7 @@
     "                max_tokens=tokenLength)\n",
     "    \n",
     "    logging.info(\"LLM Setup done\")\n",
-    "    embeddings = OpenAIEmbeddings(deployment=OpenAiEmbedding, chunk_size=1, openai_api_key=OpenAiKey)\n",
+    "    embeddings = OpenAIEmbeddings(deployment=OpenAiEmbedding, openai_api_key=OpenAiKey, openai_api_type=\"azure\")\n",
     "elif embeddingModelType == \"openai\":\n",
     "    openai.api_type = \"open_ai\"\n",
     "    openai.api_base = \"https://api.openai.com/v1\"\n",
@@ -105,9 +104,7 @@
     "    llm = ChatOpenAI(temperature=temperature,\n",
     "        openai_api_key=OpenAiApiKey,\n",
     "        model_name=\"gpt-3.5-turbo\",\n",
-    "        max_tokens=tokenLength)\n",
-    "    \n",
-    "    embeddings = OpenAIEmbeddings(openai_api_key=OpenAiApiKey)"
+    "        max_tokens=tokenLength)"
    ]
   },
   {

diff --git a/Workshop/1A_GenerateQuestions.ipynb b/Workshop/1A_GenerateQuestions.ipynb
@@ -57,8 +57,6 @@
    "outputs": [],
    "source": [
     "# Import required libraries\n",
-    "from langchain.llms.openai import AzureOpenAI, OpenAI\n",
-    "from langchain.embeddings.openai import OpenAIEmbeddings\n",
     "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
     "from langchain.document_loaders import (\n",
     "    PDFMinerLoader,\n",
@@ -99,7 +97,7 @@
     "                openai_api_key=OpenAiKey,\n",
     "                openai_api_type=\"azure\",\n",
     "                max_tokens=tokenLength)\n",
-    "        embeddings = OpenAIEmbeddings(deployment=OpenAiEmbedding, chunk_size=1, openai_api_key=OpenAiKey)\n",
+    "        embeddings = OpenAIEmbeddings(deployment=OpenAiEmbedding, openai_api_key=OpenAiKey, openai_api_type=\"azure\")\n",
     "        logging.info(\"LLM Setup done\")\n",
     "elif embeddingModelType == \"openai\":\n",
     "        openai.api_type = \"open_ai\"\n",

diff --git a/Workshop/1_LoadData.ipynb b/Workshop/1_LoadData.ipynb
@@ -98,7 +98,6 @@
    "source": [
     "# Import required libraries\n",
     "from langchain.llms.openai import AzureOpenAI, OpenAI\n",
-    "from langchain.embeddings.openai import OpenAIEmbeddings\n",
     "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
     "from langchain.document_loaders import (\n",
     "    PDFMinerLoader,\n",

diff --git a/Workshop/2A_CacheQuestion.ipynb b/Workshop/2A_CacheQuestion.ipynb
@@ -19,7 +19,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -32,15 +32,9 @@
     "indexName = SearchIndex\n",
     "\n",
     "# Set OpenAI API key and endpoint\n",
-    "openai.api_type = \"azure\"\n",
-    "openai.api_version = OpenAiVersion\n",
-    "openai_api_key = OpenAiKey\n",
-    "assert openai_api_key, \"ERROR: Azure OpenAI Key is missing\"\n",
-    "openai.api_key = openai_api_key\n",
     "openAiEndPoint = f\"{OpenAiEndPoint}\"\n",
     "assert openAiEndPoint, \"ERROR: Azure OpenAI Endpoint is missing\"\n",
-    "assert \"openai.azure.com\" in openAiEndPoint.lower(), \"ERROR: Azure OpenAI Endpoint should be in the form: \\n\\n\\t<your unique endpoint identifier>.openai.azure.com\"\n",
-    "openai.api_base = openAiEndPoint"
+    "assert \"openai.azure.com\" in openAiEndPoint.lower(), \"ERROR: Azure OpenAI Endpoint should be in the form: \\n\\n\\t<your unique endpoint identifier>.openai.azure.com\""
    ]
   },
   {
@@ -76,7 +70,7 @@
     "                openai_api_key=OpenAiKey,\n",
     "                openai_api_type=\"azure\",\n",
     "                max_tokens=tokenLength)\n",
-    "        embeddings = OpenAIEmbeddings(deployment=OpenAiEmbedding, chunk_size=1, openai_api_key=OpenAiKey)\n",
+    "        embeddings = OpenAIEmbeddings(deployment=OpenAiEmbedding, openai_api_key=OpenAiKey, openai_api_type=\"azure\")\n",
     "        logging.info(\"LLM Setup done\")\n",
     "elif embeddingModelType == \"openai\":\n",
     "        openai.api_type = \"open_ai\"\n",