Context-based-search-Version2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a008e623-4a2e-4818-a711-638a867fd88f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Script Execution:  1.3523163795471191\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import time\n",
    "import pandas as pd\n",
    "import openai\n",
    "import re\n",
    "import requests\n",
    "import sys\n",
    "from num2words import num2words\n",
    "import numpy as np\n",
    "from openai.embeddings_utils import get_embedding, cosine_similarity\n",
    "import tiktoken\n",
    "\n",
    "openai.api_key = os.getenv(\"OPENAI_API_KEY\") \n",
    "openai.organization = os.getenv(\"OPENAI_ORGANIZATION\") \n",
    "\n",
    "start_time=time.time()\n",
    "path ='c:\\\\temp_storage\\\\articles\\\\cognitive-services\\\\'\n",
    "\n",
    "########### This helps takes care of removing metadata\n",
    "search_string = \"---\" \n",
    "metadata_counter = 0\n",
    "############\n",
    "d = []\n",
    "text=\"\"\n",
    "\n",
    "for root, directories, files in os.walk(path , topdown=False):\n",
    "    for file in files:\n",
    "        if file.lower().endswith(\".md\"):\n",
    "            name =(os.path.join(root,file))\n",
    "            f = open(name, \"r\",encoding=\"utf-8\")\n",
    "            for line in f:\n",
    "                if line.find(search_string) !=-1 and metadata_counter !=2:\n",
    "                    metadata_counter+=1\n",
    "                if line.find(search_string) != 0 and metadata_counter==2:\n",
    "                    text +=line\n",
    "            f.close()\n",
    "            d.append({'FILE NAME': file ,'CONTENT': text})\n",
    "            pd.DataFrame(d)\n",
    "            metadata_counter = 0\n",
    "            text=\"\"\n",
    "end_time = time.time()\n",
    "duration = end_time - start_time\n",
    "\n",
    "print (\"Script Execution: \", duration)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a771194f-f5bb-423e-a75e-ed0873313e2f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FILE NAME</th>\n",
       "      <th>CONTENT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>anomaly-detection-best-practices.md</td>\n",
       "      <td>\\n# Best practices for using the Anomaly Detec...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>best-practices-multivariate.md</td>\n",
       "      <td>\\n# Best practices for using the Multivariate ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>multivariate-architecture.md</td>\n",
       "      <td>\\n# Predictive maintenance solution with Multi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>troubleshoot.md</td>\n",
       "      <td>\\n# Troubleshoot the multivariate API\\n\\nThis ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>batch-inference.md</td>\n",
       "      <td>\\n# Trigger batch inference with trained model...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1837</th>\n",
       "      <td>rotate-keys.md</td>\n",
       "      <td>\\n# Rotate subscription keys in Cognitive Serv...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1838</th>\n",
       "      <td>security-controls-policy.md</td>\n",
       "      <td># Azure Policy Regulatory Compliance controls ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1839</th>\n",
       "      <td>security-features.md</td>\n",
       "      <td>\\n# Azure Cognitive Services security\\n\\nSecur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1840</th>\n",
       "      <td>use-key-vault.md</td>\n",
       "      <td>\\n# Develop Azure Cognitive Services applicati...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1841</th>\n",
       "      <td>what-are-cognitive-services.md</td>\n",
       "      <td>\\n# What are Azure Cognitive Services?\\n\\nAzur...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1842 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                FILE NAME  \\\n",
       "0     anomaly-detection-best-practices.md   \n",
       "1          best-practices-multivariate.md   \n",
       "2            multivariate-architecture.md   \n",
       "3                         troubleshoot.md   \n",
       "4                      batch-inference.md   \n",
       "...                                   ...   \n",
       "1837                       rotate-keys.md   \n",
       "1838          security-controls-policy.md   \n",
       "1839                 security-features.md   \n",
       "1840                     use-key-vault.md   \n",
       "1841       what-are-cognitive-services.md   \n",
       "\n",
       "                                                CONTENT  \n",
       "0     \\n# Best practices for using the Anomaly Detec...  \n",
       "1     \\n# Best practices for using the Multivariate ...  \n",
       "2     \\n# Predictive maintenance solution with Multi...  \n",
       "3     \\n# Troubleshoot the multivariate API\\n\\nThis ...  \n",
       "4     \\n# Trigger batch inference with trained model...  \n",
       "...                                                 ...  \n",
       "1837  \\n# Rotate subscription keys in Cognitive Serv...  \n",
       "1838  # Azure Policy Regulatory Compliance controls ...  \n",
       "1839  \\n# Azure Cognitive Services security\\n\\nSecur...  \n",
       "1840  \\n# Develop Azure Cognitive Services applicati...  \n",
       "1841  \\n# What are Azure Cognitive Services?\\n\\nAzur...  \n",
       "\n",
       "[1842 rows x 2 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(d)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "efca7fd2-d9d4-4422-b133-4803c5769772",
   "metadata": {},
   "outputs": [],
   "source": [
    "# s is input text\n",
    "def normalize_text(s, sep_token = \" \\n \"):\n",
    "    s = re.sub(r'\\s+',  ' ', s).strip()\n",
    "    s = re.sub(r\". ,\",\"\",s)\n",
    "    # remove all instances of multiple spaces\n",
    "    s = s.replace(\"..\",\".\")\n",
    "    s = s.replace(\". .\",\".\")\n",
    "    s = s.replace(\"\\n\", \"\")\n",
    "    s = s.replace(\"#\",\"\")\n",
    "    s = s.strip()\n",
    "    if s ==\"\":\n",
    "        s = \"<blank>\"\n",
    "    return s\n",
    "df_normalized=df.copy()\n",
    "df_normalized['CONTENT'] = df[\"CONTENT\"].apply(lambda x : normalize_text(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "38fd0780-a2b8-42f9-aea5-cd2b81a9e245",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FILE NAME</th>\n",
       "      <th>CONTENT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>anomaly-detection-best-practices.md</td>\n",
       "      <td>Best practices for using the Anomaly Detector ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>best-practices-multivariate.md</td>\n",
       "      <td>Best practices for using the Multivariate Anom...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>multivariate-architecture.md</td>\n",
       "      <td>Predictive maintenance solution with Multivari...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>troubleshoot.md</td>\n",
       "      <td>Troubleshoot the multivariate API This article...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>batch-inference.md</td>\n",
       "      <td>Trigger batch inference with trained model You...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1837</th>\n",
       "      <td>rotate-keys.md</td>\n",
       "      <td>Rotate subscription keys in Cognitive Services...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1838</th>\n",
       "      <td>security-controls-policy.md</td>\n",
       "      <td>Azure Policy Regulatory Compliance controls fo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1839</th>\n",
       "      <td>security-features.md</td>\n",
       "      <td>Azure Cognitive Services security Security sho...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1840</th>\n",
       "      <td>use-key-vault.md</td>\n",
       "      <td>Develop Azure Cognitive Services applications ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1841</th>\n",
       "      <td>what-are-cognitive-services.md</td>\n",
       "      <td>What are Azure Cognitive Services? Azure Cogni...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1842 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                FILE NAME  \\\n",
       "0     anomaly-detection-best-practices.md   \n",
       "1          best-practices-multivariate.md   \n",
       "2            multivariate-architecture.md   \n",
       "3                         troubleshoot.md   \n",
       "4                      batch-inference.md   \n",
       "...                                   ...   \n",
       "1837                       rotate-keys.md   \n",
       "1838          security-controls-policy.md   \n",
       "1839                 security-features.md   \n",
       "1840                     use-key-vault.md   \n",
       "1841       what-are-cognitive-services.md   \n",
       "\n",
       "                                                CONTENT  \n",
       "0     Best practices for using the Anomaly Detector ...  \n",
       "1     Best practices for using the Multivariate Anom...  \n",
       "2     Predictive maintenance solution with Multivari...  \n",
       "3     Troubleshoot the multivariate API This article...  \n",
       "4     Trigger batch inference with trained model You...  \n",
       "...                                                 ...  \n",
       "1837  Rotate subscription keys in Cognitive Services...  \n",
       "1838  Azure Policy Regulatory Compliance controls fo...  \n",
       "1839  Azure Cognitive Services security Security sho...  \n",
       "1840  Develop Azure Cognitive Services applications ...  \n",
       "1841  What are Azure Cognitive Services? Azure Cogni...  \n",
       "\n",
       "[1842 rows x 2 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_normalized\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "91c1c4db-2c0d-475b-b2ac-dec03a8a071d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                              FILE NAME  CONTENT\n",
      "57    cognitive-services-bing-custom-search-signup-r...  <blank>\n",
      "61                                    publish-revert.md  <blank>\n",
      "1089                  quickstart-sdk-cognitive-model.md  <blank>\n",
      "1099                                  tip-debug-json.md  <blank>\n",
      "1526                              quickstart-list-go.md  <blank>\n",
      "1527                                 quickstart-list.md  <blank>\n"
     ]
    }
   ],
   "source": [
    "df_blank = df_normalized[df_normalized.CONTENT=='<blank>']\n",
    "print(df_blank)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8209d9e-09cf-4a08-930e-953beb0ab98a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d3113b03-9031-4ea9-8167-5cbd25c12914",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FILE NAME</th>\n",
       "      <th>CONTENT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>anomaly-detection-best-practices.md</td>\n",
       "      <td>Best practices for using the Anomaly Detector ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>best-practices-multivariate.md</td>\n",
       "      <td>Best practices for using the Multivariate Anom...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>multivariate-architecture.md</td>\n",
       "      <td>Predictive maintenance solution with Multivari...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>troubleshoot.md</td>\n",
       "      <td>Troubleshoot the multivariate API This article...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>batch-inference.md</td>\n",
       "      <td>Trigger batch inference with trained model You...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1837</th>\n",
       "      <td>rotate-keys.md</td>\n",
       "      <td>Rotate subscription keys in Cognitive Services...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1838</th>\n",
       "      <td>security-controls-policy.md</td>\n",
       "      <td>Azure Policy Regulatory Compliance controls fo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1839</th>\n",
       "      <td>security-features.md</td>\n",
       "      <td>Azure Cognitive Services security Security sho...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1840</th>\n",
       "      <td>use-key-vault.md</td>\n",
       "      <td>Develop Azure Cognitive Services applications ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1841</th>\n",
       "      <td>what-are-cognitive-services.md</td>\n",
       "      <td>What are Azure Cognitive Services? Azure Cogni...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1836 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                FILE NAME  \\\n",
       "0     anomaly-detection-best-practices.md   \n",
       "1          best-practices-multivariate.md   \n",
       "2            multivariate-architecture.md   \n",
       "3                         troubleshoot.md   \n",
       "4                      batch-inference.md   \n",
       "...                                   ...   \n",
       "1837                       rotate-keys.md   \n",
       "1838          security-controls-policy.md   \n",
       "1839                 security-features.md   \n",
       "1840                     use-key-vault.md   \n",
       "1841       what-are-cognitive-services.md   \n",
       "\n",
       "                                                CONTENT  \n",
       "0     Best practices for using the Anomaly Detector ...  \n",
       "1     Best practices for using the Multivariate Anom...  \n",
       "2     Predictive maintenance solution with Multivari...  \n",
       "3     Troubleshoot the multivariate API This article...  \n",
       "4     Trigger batch inference with trained model You...  \n",
       "...                                                 ...  \n",
       "1837  Rotate subscription keys in Cognitive Services...  \n",
       "1838  Azure Policy Regulatory Compliance controls fo...  \n",
       "1839  Azure Cognitive Services security Security sho...  \n",
       "1840  Develop Azure Cognitive Services applications ...  \n",
       "1841  What are Azure Cognitive Services? Azure Cogni...  \n",
       "\n",
       "[1836 rows x 2 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_normalized = df_normalized[df_normalized.CONTENT !='<blank>']\n",
    "len(df_normalized)\n",
    "df_normalized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "78fbbfba-ab31-4f45-bf1b-58159435a19c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FILE NAME</th>\n",
       "      <th>CONTENT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>anomaly-detection-best-practices.md</td>\n",
       "      <td>Best practices for using the Anomaly Detector ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>best-practices-multivariate.md</td>\n",
       "      <td>Best practices for using the Multivariate Anom...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>multivariate-architecture.md</td>\n",
       "      <td>Predictive maintenance solution with Multivari...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>troubleshoot.md</td>\n",
       "      <td>Troubleshoot the multivariate API This article...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>batch-inference.md</td>\n",
       "      <td>Trigger batch inference with trained model You...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1837</th>\n",
       "      <td>rotate-keys.md</td>\n",
       "      <td>Rotate subscription keys in Cognitive Services...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1838</th>\n",
       "      <td>security-controls-policy.md</td>\n",
       "      <td>Azure Policy Regulatory Compliance controls fo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1839</th>\n",
       "      <td>security-features.md</td>\n",
       "      <td>Azure Cognitive Services security Security sho...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1840</th>\n",
       "      <td>use-key-vault.md</td>\n",
       "      <td>Develop Azure Cognitive Services applications ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1841</th>\n",
       "      <td>what-are-cognitive-services.md</td>\n",
       "      <td>What are Azure Cognitive Services? Azure Cogni...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1836 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                FILE NAME  \\\n",
       "0     anomaly-detection-best-practices.md   \n",
       "1          best-practices-multivariate.md   \n",
       "2            multivariate-architecture.md   \n",
       "3                         troubleshoot.md   \n",
       "4                      batch-inference.md   \n",
       "...                                   ...   \n",
       "1837                       rotate-keys.md   \n",
       "1838          security-controls-policy.md   \n",
       "1839                 security-features.md   \n",
       "1840                     use-key-vault.md   \n",
       "1841       what-are-cognitive-services.md   \n",
       "\n",
       "                                                CONTENT  \n",
       "0     Best practices for using the Anomaly Detector ...  \n",
       "1     Best practices for using the Multivariate Anom...  \n",
       "2     Predictive maintenance solution with Multivari...  \n",
       "3     Troubleshoot the multivariate API This article...  \n",
       "4     Trigger batch inference with trained model You...  \n",
       "...                                                 ...  \n",
       "1837  Rotate subscription keys in Cognitive Services...  \n",
       "1838  Azure Policy Regulatory Compliance controls fo...  \n",
       "1839  Azure Cognitive Services security Security sho...  \n",
       "1840  Develop Azure Cognitive Services applications ...  \n",
       "1841  What are Azure Cognitive Services? Azure Cogni...  \n",
       "\n",
       "[1836 rows x 2 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_normalized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ab5bc67-46e4-45a3-adbf-363375f40759",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "96b2508b-4b0c-417e-b3ff-0bb4ef738dd2",
   "metadata": {},
   "source": [
    "| GENERATION |TOKENIZER    | MAX INPUT TOKENS| KNOWLEDGE CUTOFF|\n",
    "|------------|-------------|-----------------|-----------------|\n",
    "| V2         | cl100k_base | 8191            | Sep 2021        |\n",
    "| V1         | GPT-2/GPT-3 | 2046            | Aug 2020        |\n",
    "\n",
    "\n",
    "https://beta.openai.com/docs/guides/embeddings/what-are-embeddings\n",
    "\n",
    "https://openai.com/blog/new-and-improved-embedding-model/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "bfe07d0a-ce24-4210-b1ce-a50faed184b5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FILE NAME</th>\n",
       "      <th>CONTENT</th>\n",
       "      <th>n_tokens</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>anomaly-detection-best-practices.md</td>\n",
       "      <td>Best practices for using the Anomaly Detector ...</td>\n",
       "      <td>1244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>best-practices-multivariate.md</td>\n",
       "      <td>Best practices for using the Multivariate Anom...</td>\n",
       "      <td>3318</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>multivariate-architecture.md</td>\n",
       "      <td>Predictive maintenance solution with Multivari...</td>\n",
       "      <td>789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>troubleshoot.md</td>\n",
       "      <td>Troubleshoot the multivariate API This article...</td>\n",
       "      <td>2196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>batch-inference.md</td>\n",
       "      <td>Trigger batch inference with trained model You...</td>\n",
       "      <td>2638</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1837</th>\n",
       "      <td>rotate-keys.md</td>\n",
       "      <td>Rotate subscription keys in Cognitive Services...</td>\n",
       "      <td>310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1838</th>\n",
       "      <td>security-controls-policy.md</td>\n",
       "      <td>Azure Policy Regulatory Compliance controls fo...</td>\n",
       "      <td>215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1839</th>\n",
       "      <td>security-features.md</td>\n",
       "      <td>Azure Cognitive Services security Security sho...</td>\n",
       "      <td>1551</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1840</th>\n",
       "      <td>use-key-vault.md</td>\n",
       "      <td>Develop Azure Cognitive Services applications ...</td>\n",
       "      <td>4827</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1841</th>\n",
       "      <td>what-are-cognitive-services.md</td>\n",
       "      <td>What are Azure Cognitive Services? Azure Cogni...</td>\n",
       "      <td>1888</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1836 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                FILE NAME  \\\n",
       "0     anomaly-detection-best-practices.md   \n",
       "1          best-practices-multivariate.md   \n",
       "2            multivariate-architecture.md   \n",
       "3                         troubleshoot.md   \n",
       "4                      batch-inference.md   \n",
       "...                                   ...   \n",
       "1837                       rotate-keys.md   \n",
       "1838          security-controls-policy.md   \n",
       "1839                 security-features.md   \n",
       "1840                     use-key-vault.md   \n",
       "1841       what-are-cognitive-services.md   \n",
       "\n",
       "                                                CONTENT  n_tokens  \n",
       "0     Best practices for using the Anomaly Detector ...      1244  \n",
       "1     Best practices for using the Multivariate Anom...      3318  \n",
       "2     Predictive maintenance solution with Multivari...       789  \n",
       "3     Troubleshoot the multivariate API This article...      2196  \n",
       "4     Trigger batch inference with trained model You...      2638  \n",
       "...                                                 ...       ...  \n",
       "1837  Rotate subscription keys in Cognitive Services...       310  \n",
       "1838  Azure Policy Regulatory Compliance controls fo...       215  \n",
       "1839  Azure Cognitive Services security Security sho...      1551  \n",
       "1840  Develop Azure Cognitive Services applications ...      4827  \n",
       "1841  What are Azure Cognitive Services? Azure Cogni...      1888  \n",
       "\n",
       "[1836 rows x 3 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer = tiktoken.get_encoding(\"cl100k_base\")\n",
    "df_tok=df_normalized.copy()\n",
    "df_tok['n_tokens'] = df_normalized[\"CONTENT\"].apply(lambda x: len(tokenizer.encode(x)))\n",
    "df_tok"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "93b3f794-d584-4565-8ef5-86e127aed559",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of tokens: 2532696\n",
      "\n",
      "MODEL        VERSION    COST\n",
      "-----------------------------------\n",
      "Ada\t\tv1\t$10.13078\n",
      "Babbage\t\tv1\t$12.66348\n",
      "Curie\t\tv1\t$50.65392\n",
      "Davinci\t\tv1\t$506.5392\n",
      "Ada\t\tv2\t$1.013078\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Based on https://openai.com/api/pricing/ on 01/29/2023\n",
    "# If you were using this for approximating pricing with Azure OpenAI adjust the values below with: https://azure.microsoft.com/pricing/details/cognitive-services/openai-service/\n",
    "\n",
    "#MODEL\tUSAGE\n",
    "#Ada     v1\t$0.0040 / 1K tokens\n",
    "#Babbage v1\t$0.0050 / 1K tokens\n",
    "#Curie   v1\t$0.0200 / 1K tokens\n",
    "#Davinci v1\t$0.2000 / 1K tokens\n",
    "\n",
    "#MODEL\tUSAGE\n",
    "#Ada     v2\t$0.0004 / 1K tokens\n",
    "#This Ada model, text-embedding-ada-002, is a better and lower cost replacement for our older embedding models. \n",
    "\n",
    "n_tokens_sum = df_tok['n_tokens'].sum()\n",
    "\n",
    "ada_v1_embeddings_cost = (n_tokens_sum/1000) *.0040\n",
    "babbage_v1_embeddings_cost = (n_tokens_sum/1000) *.0050\n",
    "curie_v1_embeddings_cost = (n_tokens_sum/1000) *.02\n",
    "davinci_v1_embeddings_cost = (n_tokens_sum/1000) *.2\n",
    "\n",
    "ada_v2_embeddings_cost = (n_tokens_sum/1000) *.0004\n",
    "\n",
    "print(\"Number of tokens: \" + str(n_tokens_sum) + \"\\n\")\n",
    "\n",
    "print(\"MODEL        VERSION    COST\")\n",
    "print(\"-----------------------------------\")\n",
    "print(\"Ada\" + \"\\t\\t\" + \"v1\" + \"\\t$\" + '%.8s' % str(ada_v1_embeddings_cost))\n",
    "print(\"Babbage\" + \"\\t\\t\" + \"v1\" + \"\\t$\" + '%.8s' % str(babbage_v1_embeddings_cost))\n",
    "print(\"Curie\" + \"\\t\\t\" + \"v1\" + \"\\t$\" + '%.8s' % str(curie_v1_embeddings_cost))\n",
    "print(\"Davinci\" + \"\\t\\t\" + \"v1\" + \"\\t$\" + '%.8s' % str(davinci_v1_embeddings_cost))\n",
    "print(\"Ada\" + \"\\t\\t\" + \"v2\" + \"\\t$\" + '%.8s' %str(ada_v2_embeddings_cost))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "258a9094-98bf-4d47-84a8-ff57ac7f590e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1842"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "556b1d9c-e904-42d3-9584-dbf9733db562",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "534"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tok = df_tok[df_tok.n_tokens<400]\n",
    "\n",
    "len(df_tok)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d7a8b721-aaeb-4c7a-a3d0-5edd27a48275",
   "metadata": {},
   "source": [
    "# OpenAI Text & Embedding Rate Limits?\n",
    "\n",
    "Rate limits are enforced at the **organization level, not user level**, based on the specific endpoint used as well as the type of account you have. \n",
    "\n",
    "Rate limits are measured in two ways: **RPM (requests per minute)** and **TPM (tokens per minute)**. \n",
    "\n",
    "## TEXT & EMBEDDING\n",
    "\n",
    "Free trial users •20 RPM •150,000 TPM\n",
    "\n",
    "Pay-as-you-go users (first 48 hours)\t•60 RPM •250,000 TPM\n",
    "\n",
    "Pay-as-you-go users (after 48 hours)\t•3,000 RPM •250,000 TPM\n",
    "\n",
    "https://beta.openai.com/docs/guides/rate-limits/overview"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c704c1bd-d448-4260-9b37-1f79e181382e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "from IPython.display import clear_output\n",
    "\n",
    "request_counter = 0\n",
    "total_requests_sent = 0\n",
    "rate_limit= 3000\n",
    "\n",
    "start_timer=time.time()\n",
    "\n",
    "def generate_embeddings(text, model=\"text-embedding-ada-002\"):\n",
    "    global request_counter\n",
    "    global rate_limit\n",
    "    global total_requests_sent\n",
    "    global start_timer\n",
    "    clear_output(wait=True)\n",
    "    check_timer = time.time()\n",
    "    duration = check_timer-start_timer\n",
    "    print(duration)\n",
    "    \n",
    "    if int(duration) >= 60:\n",
    "        start_timer=time.time()\n",
    "        request_counter=0\n",
    "    if request_counter == rate_limit and int(duration) <= 59:\n",
    "        sleep_for = 60-int(duration)\n",
    "        print(\"Sleeping for \" + str(sleep_for) +\" seconds\")\n",
    "        print(\"Total requests sent: \", total_requests_sent)\n",
    "        time.sleep(sleep_for)\n",
    "        start_timer = time.time()\n",
    "        request_counter =0\n",
    "    if request_counter < rate_limit:\n",
    "        request_counter+=1\n",
    "        total_requests_sent+=1\n",
    "        print(\"Request counter: \", request_counter)\n",
    "        print(\"Total requests sent: \", total_requests_sent)\n",
    "        \n",
    "    return openai.Embedding.create(input = [text], model=model)['data'][0]['embedding']\n",
    "\n",
    "df_embeddings = df_tok.copy()\n",
    "df_embeddings['ada_v2_embedding'] = df_tok.CONTENT.transform(lambda x: generate_embeddings(x, model='text-embedding-ada-002'))\n",
    "\n",
    "df_embeddings\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "b4468ce5-fe77-4e67-8f23-1fac4e285e6a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "How can I help you?\n",
      "\n",
      " test\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FILE NAME</th>\n",
       "      <th>CONTENT</th>\n",
       "      <th>n_tokens</th>\n",
       "      <th>ada_v2_embedding</th>\n",
       "      <th>similarities</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>713</th>\n",
       "      <td>test-model.md</td>\n",
       "      <td>To test your model from Language Studio 1. Sel...</td>\n",
       "      <td>165</td>\n",
       "      <td>[0.0032791579142212868, -0.0011860255617648363...</td>\n",
       "      <td>0.780951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>874</th>\n",
       "      <td>do-not-use-too-few-utterances.md</td>\n",
       "      <td>These few utterances are for demonstration pur...</td>\n",
       "      <td>53</td>\n",
       "      <td>[-0.0060370368883013725, 0.005788491573184729,...</td>\n",
       "      <td>0.772646</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>867</th>\n",
       "      <td>best-practice-utterances.md</td>\n",
       "      <td>For best results make sure that example uttera...</td>\n",
       "      <td>54</td>\n",
       "      <td>[-0.014310219325125217, 0.03167257085442543, 0...</td>\n",
       "      <td>0.770794</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            FILE NAME  \\\n",
       "713                     test-model.md   \n",
       "874  do-not-use-too-few-utterances.md   \n",
       "867       best-practice-utterances.md   \n",
       "\n",
       "                                               CONTENT  n_tokens  \\\n",
       "713  To test your model from Language Studio 1. Sel...       165   \n",
       "874  These few utterances are for demonstration pur...        53   \n",
       "867  For best results make sure that example uttera...        54   \n",
       "\n",
       "                                      ada_v2_embedding  similarities  \n",
       "713  [0.0032791579142212868, -0.0011860255617648363...      0.780951  \n",
       "874  [-0.0060370368883013725, 0.005788491573184729,...      0.772646  \n",
       "867  [-0.014310219325125217, 0.03167257085442543, 0...      0.770794  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# search embedded docs based on cosine similarity\n",
    "\n",
    "df_similarities = df_embeddings.copy()\n",
    "\n",
    "def get_embedding(text, model=\"text-embedding-ada-002\"):\n",
    "   return openai.Embedding.create(input = [text], model=model)['data'][0]['embedding']\n",
    "\n",
    "def search_docs(df, user_query, top_n=3, to_print=True):\n",
    "    embedding = get_embedding(\n",
    "        user_query,\n",
    "        model=\"text-embedding-ada-002\"\n",
    "    )\n",
    "\n",
    "    df_similarities[\"similarities\"] = df_embeddings.ada_v2_embedding.apply(lambda x: cosine_similarity(x, embedding))\n",
    "\n",
    "    res = (\n",
    "        df_similarities.sort_values(\"similarities\", ascending=False)\n",
    "        .head(top_n)\n",
    "    )\n",
    "    if to_print:\n",
    "        display(res)\n",
    "    return res\n",
    "\n",
    "question = input(\"How can I help you?\\n\\n\")\n",
    "\n",
    "res = search_docs(df, question, top_n=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "438e5363-b22e-4564-8c21-bd388d3fae8d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'To test your model from Language Studio 1. Select **Testing deployments** from the left side menu. 3. Select the model you want to test. You can only test models that are assigned to deployments. 4. From deployment name dropdown, select your deployment name. 5. In the text box, enter an utterance to test. 6. From the top menu, click on **Run the test**. 7. After you run the test, you should see the response of the model in the result. You can view the results in entities cards view, or view it in JSON format. :::image type=\"content\" source=\"././media/test-model.png\" alt-text=\"A screenshot showing how to test a model in Language Studio.\" lightbox=\"././media/test-model.png\":::'"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res.CONTENT.values[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "79b8934c-c47f-431d-b46b-2a5a7daab3ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FILE NAME</th>\n",
       "      <th>CONTENT</th>\n",
       "      <th>n_tokens</th>\n",
       "      <th>ada_v2_embedding</th>\n",
       "      <th>similarities</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>quickstart-cleanup-next-steps.md</td>\n",
       "      <td>Clean up resources If you want to clean up and...</td>\n",
       "      <td>188</td>\n",
       "      <td>[-0.002563188085332513, 0.005606868304312229, ...</td>\n",
       "      <td>0.706685</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>client-libraries-multivariate.md</td>\n",
       "      <td>Quickstart: Use the Multivariate Anomaly Detec...</td>\n",
       "      <td>150</td>\n",
       "      <td>[-0.019996939226984978, 0.011973044835031033, ...</td>\n",
       "      <td>0.729733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>client-libraries.md</td>\n",
       "      <td>Quickstart: Use the Univariate Anomaly Detecto...</td>\n",
       "      <td>147</td>\n",
       "      <td>[-0.003396739950403571, 0.013678478077054024, ...</td>\n",
       "      <td>0.727968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>client-libraries.md</td>\n",
       "      <td>Quickstart: Use the Bing Autosuggest client li...</td>\n",
       "      <td>101</td>\n",
       "      <td>[-0.014453585259616375, 0.017982948571443558, ...</td>\n",
       "      <td>0.711676</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>client-libraries.md</td>\n",
       "      <td>Quickstart: Use the Bing Custom Search client ...</td>\n",
       "      <td>130</td>\n",
       "      <td>[-0.000941423000767827, 0.024544676765799522, ...</td>\n",
       "      <td>0.719405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1767</th>\n",
       "      <td>prerequisites-go.md</td>\n",
       "      <td>Prerequisites This quickstart requires: * [Go]...</td>\n",
       "      <td>39</td>\n",
       "      <td>[0.0007572504691779613, 0.0019182561663910747,...</td>\n",
       "      <td>0.734317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1768</th>\n",
       "      <td>prerequisites-java.md</td>\n",
       "      <td>Prerequisites This quickstart requires: * [JDK...</td>\n",
       "      <td>64</td>\n",
       "      <td>[0.010907812044024467, -0.005134829320013523, ...</td>\n",
       "      <td>0.737251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1769</th>\n",
       "      <td>prerequisites-nodejs.md</td>\n",
       "      <td>Prerequisites This quickstart requires: * [Nod...</td>\n",
       "      <td>46</td>\n",
       "      <td>[0.0030772036407142878, 4.518704008660279e-05,...</td>\n",
       "      <td>0.726393</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1770</th>\n",
       "      <td>prerequisites-python.md</td>\n",
       "      <td>Prerequisites This quickstart requires: * [Pyt...</td>\n",
       "      <td>48</td>\n",
       "      <td>[0.005925570148974657, 0.0006811613566242158, ...</td>\n",
       "      <td>0.729803</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1776</th>\n",
       "      <td>setup-env-variables.md</td>\n",
       "      <td>Set up  Create a Translator resource Azure Cog...</td>\n",
       "      <td>166</td>\n",
       "      <td>[-2.2200711100595072e-05, -0.00724548101425170...</td>\n",
       "      <td>0.722860</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>330 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                             FILE NAME  \\\n",
       "25    quickstart-cleanup-next-steps.md   \n",
       "26    client-libraries-multivariate.md   \n",
       "27                 client-libraries.md   \n",
       "46                 client-libraries.md   \n",
       "62                 client-libraries.md   \n",
       "...                                ...   \n",
       "1767               prerequisites-go.md   \n",
       "1768             prerequisites-java.md   \n",
       "1769           prerequisites-nodejs.md   \n",
       "1770           prerequisites-python.md   \n",
       "1776            setup-env-variables.md   \n",
       "\n",
       "                                                CONTENT  n_tokens  \\\n",
       "25    Clean up resources If you want to clean up and...       188   \n",
       "26    Quickstart: Use the Multivariate Anomaly Detec...       150   \n",
       "27    Quickstart: Use the Univariate Anomaly Detecto...       147   \n",
       "46    Quickstart: Use the Bing Autosuggest client li...       101   \n",
       "62    Quickstart: Use the Bing Custom Search client ...       130   \n",
       "...                                                 ...       ...   \n",
       "1767  Prerequisites This quickstart requires: * [Go]...        39   \n",
       "1768  Prerequisites This quickstart requires: * [JDK...        64   \n",
       "1769  Prerequisites This quickstart requires: * [Nod...        46   \n",
       "1770  Prerequisites This quickstart requires: * [Pyt...        48   \n",
       "1776  Set up  Create a Translator resource Azure Cog...       166   \n",
       "\n",
       "                                       ada_v2_embedding  similarities  \n",
       "25    [-0.002563188085332513, 0.005606868304312229, ...      0.706685  \n",
       "26    [-0.019996939226984978, 0.011973044835031033, ...      0.729733  \n",
       "27    [-0.003396739950403571, 0.013678478077054024, ...      0.727968  \n",
       "46    [-0.014453585259616375, 0.017982948571443558, ...      0.711676  \n",
       "62    [-0.000941423000767827, 0.024544676765799522, ...      0.719405  \n",
       "...                                                 ...           ...  \n",
       "1767  [0.0007572504691779613, 0.0019182561663910747,...      0.734317  \n",
       "1768  [0.010907812044024467, -0.005134829320013523, ...      0.737251  \n",
       "1769  [0.0030772036407142878, 4.518704008660279e-05,...      0.726393  \n",
       "1770  [0.005925570148974657, 0.0006811613566242158, ...      0.729803  \n",
       "1776  [-2.2200711100595072e-05, -0.00724548101425170...      0.722860  \n",
       "\n",
       "[330 rows x 5 columns]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_similarities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "1bdac244-b3ba-4b1c-a8cd-5215e49d3a99",
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "How can I help you?\n",
      "\n",
      " Tell me about Multivariate Anomaly Detection?\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Multivariate Anomaly Detection (MAD) is a type of anomaly detection technology that uses machine learning to detect anomalies across multiple datasets. It is used to uncover hidden relationships between variables, to detect outliers and anomalies, and to assess the degree of change. MAD can help identify potential problems in areas such as credit card fraud and financial fraud, or detect patterns in industrial process control systems.\n"
     ]
    }
   ],
   "source": [
    "def search_docs(df, user_query, top_n=3, to_print=True):\n",
    "    embedding = get_embedding(\n",
    "        user_query,\n",
    "        model=\"text-embedding-ada-002\"\n",
    "    )\n",
    "    \n",
    "    df_question = df_similarities.copy()\n",
    "    df_question[\"similarities\"] = df_similarities.ada_v2_embedding.apply(lambda x: cosine_similarity(x, embedding))\n",
    "\n",
    "    res = (\n",
    "        df_question.sort_values(\"similarities\", ascending=False)\n",
    "        .head(top_n)\n",
    "    )\n",
    "    return res\n",
    "\n",
    "res = search_docs(df, question, top_n=1)\n",
    "\n",
    "ai_question = input(\"How can I help you?\\n\\n\")\n",
    "context= res.CONTENT.values\n",
    "completion_model='text-davinci-003'\n",
    "\n",
    "initial_prompt = \"The following is a conversation with an AI assistant. The assistant is helpful, creative, clever, and very friendly.\"\n",
    "\n",
    "combined_prompt = initial_prompt + str(context) + \"Q: \" + ai_question\n",
    "response = openai.Completion.create(model=completion_model, prompt=combined_prompt, max_tokens=100)\n",
    "ai_response = response['choices'][0]['text'].replace('\\n', '').replace(' .', '.').strip()\n",
    "\n",
    "print(\"\\n\"+ ai_response)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}