experiment.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Install Python Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Create or update your local Python environment\n",
    "\n",
    "%pip install azure-functions\n",
    "%pip install azure-core\n",
    "%pip install azure-cosmos\n",
    "%pip install openai\n",
    "%pip install numpy\n",
    "%pip install requests\n",
    "%pip install pandas\n",
    "%pip install azure-storage-blob \n",
    "%pip install azure-identity\n",
    "%pip install smart_open\n",
    "%pip install tenacity\n",
    "%pip install pinecone-client\n",
    "%pip install redis\n",
    "%pip install tiktoken\n",
    "%pip install azure-storage-file-share\n",
    "%pip install python-dotenv\n",
    "%pip install azure-search-documents==11.4.0b3\n",
    "%pip install azure-ai-formrecognizer\n",
    "%pip install beautifulsoup4\n",
    "%pip install lxml\n",
    "%pip install openpyxl\n",
    "%pip install azure-ai-textanalytics\n",
    "%pip install langchain==0.0.139\n",
    "%pip install flask\n",
    "%pip install flask-socketio\n",
    "\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from dotenv import load_dotenv\n",
    "load_dotenv(override=True)\n",
    "\n",
    "import shutil\n",
    "import sys\n",
    "sys.path.append('./utils')\n",
    "\n",
    "### Uncomment below imports as needed -- make sure that all relevant values and keys in the .env file are properly populated\n",
    "# from utils import redis_helpers\n",
    "# from utils import helpers\n",
    "# from utils import language\n",
    "# from utils import openai_helpers\n",
    "# from utils import storage\n",
    "# from utils import bot_helpers\n",
    "\n",
    "from utils.env_vars import *\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://searchseh000000.search.windows.net'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "COG_SEARCH_ENDPOINT"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Activate Cognitive Search Ingestion \n",
    "#### First Run - Create Index and Indexer \n",
    "##### Caution: this will destroy any data you might already have in your index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index vec-index Deleted\n",
      "Index vec-index created\n",
      "Index km-openai Deleted\n",
      "Index km-openai created\n",
      "Deleted Skillset - km-openai-skills\n",
      "Created new Skillset - km-openai-skills\n",
      "Deleted Indexer - km-openai-indexer\n",
      "Deleted Data Source - km-openai-skills\n",
      "Created new Data Source Connection - km-openai-docs\n",
      "Created new Indexer - km-openai-indexer\n",
      "Running Indexer km-openai-indexer\n"
     ]
    }
   ],
   "source": [
    "#### Ingest all knowledge base documents\n",
    "from utils import cogsearch_helpers\n",
    "\n",
    "cogsearch_helpers.ingest_kb(container = KB_BLOB_CONTAINER)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Additional Runs - Re-indexing with delta documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running Indexer km-openai-indexer\n"
     ]
    }
   ],
   "source": [
    "### Re-index additional documents\n",
    "from utils import cogsearch_helpers\n",
    "\n",
    "cogsearch_helpers.run_indexer()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Activate Form Recognizer Ingestion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### Ingest all form documents\n",
    "\n",
    "from utils import storage\n",
    "from utils import fr_helpers\n",
    "\n",
    "fr_helpers.process_forms(in_container = FR_CONTAINER, out_container = OUTPUT_BLOB_CONTAINER)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Deleted Indexer - km-openai-indexer\n",
      "Deleted Data Source - km-openai-skills\n",
      "Created new Data Source Connection - km-openai-docs\n",
      "Created new Indexer - km-openai-indexer\n",
      "Running Indexer km-openai-indexer\n"
     ]
    }
   ],
   "source": [
    "### other containers could be used as the sources of documents to index\n",
    "\n",
    "cogsearch_helpers.create_indexer('kmoaidemo2') \n",
    "cogsearch_helpers.run_indexer()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scrape Web Pages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.lufthansa.com\n",
      "HTTP Error 403: Forbidden\n",
      "HTTP Error 403: Forbidden\n"
     ]
    }
   ],
   "source": [
    "### Scrape data from web pages\n",
    "### This saves the text from the web pages into the processed folder, and any files which are found are saved\n",
    "### to the demo folder. For the files to be processed, you need to run the cognitive search re-indexer\n",
    "\n",
    "# TODO: Some files may be better parsed by form recognizer. Need to decide how to know which is more suitable, \n",
    "# and place them in the correct folder. \n",
    "\n",
    "# Enter the URLs (only enter the root url, the crawler traverses the hierarchy of the webpages below the root) \n",
    "# that you want to scrape in the urls list. Each url should be a string\n",
    "urls = ['https://www.lufthansa.com']\n",
    "\n",
    "from utils import web_crawler\n",
    "\n",
    "for url in urls:\n",
    "    web_crawler.crawl(url, KB_BLOB_CONN_STR, KB_BLOB_CONTAINER, OUTPUT_BLOB_CONTAINER)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from urllib.parse import urlparse\n",
    "\n",
    "local_domain = urlparse(url).netloc"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Interrogate the APIs with the sample Knowledge Base"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Use this cell to query Redis with the below queries\n",
    "import os\n",
    "import json\n",
    "from utils import bot_helpers\n",
    "from dotenv import load_dotenv\n",
    "load_dotenv(override=True)\n",
    "from utils import km_agents\n",
    "\n",
    "from utils import redis_helpers\n",
    "from utils import openai_helpers\n",
    "redis_conn = redis_helpers.get_new_conn() \n",
    "\n",
    "params_dict = {\n",
    "    'enable_unified_search': False,\n",
    "    'enable_redis_search': True,\n",
    "    'enable_cognitive_search': False,\n",
    "    'evaluate_step': False,\n",
    "    'check_adequacy': False,\n",
    "    'check_intent': False,\n",
    "    'use_calendar': False,\n",
    "    'use_bing': False\n",
    "    }\n",
    "\n",
    "stream = True\n",
    "\n",
    "queries = [\n",
    "    \"what is mentioned about the Lost City hotel?\",\n",
    "    \"what is mentioned about the Strip in Vegas?\",\n",
    "    \"what is mentioned about the Fountain Hotel?\"\n",
    "]\n",
    "\n",
    "agent_name = 'ccr' # or 'os' or 'zs'\n",
    "\n",
    "for q in queries:\n",
    "    agent = km_agents.KMOAI_Agent(agent_name = agent_name, params_dict=params_dict, verbose = False, stream=stream)\n",
    "    answer, sources, likely_sources, s_id = agent.run(q, redis_conn)\n",
    "    print(\"\\n\\n\", answer, '\\n', sources, '\\n', likely_sources, '\\n', s_id, '\\n\\n\\n###############################')\n",
    "    \n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Try out the summarization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import summarization\n",
    "\n",
    "folder = './docs_to_summarize/'\n",
    "\n",
    "ref_summ_df = summarization.summarize_folder(folder, mode='refine', verbose=False)\n",
    "mp_summ_df  = summarization.summarize_folder(folder, mode='map_reduce', verbose=False)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Experimentation Code Below - NO NEED TO RUN \n",
    "### For your reference only"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reset Redis Index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### Reset Index in Redis\n",
    "from utils import redis_helpers\n",
    "\n",
    "reset_index = True\n",
    "\n",
    "if reset_index:\n",
    "    redis_conn = redis_helpers.get_new_conn()\n",
    "    redis_helpers.redis_reset_index(redis_conn)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate Embeddings and Load into Redis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Use this cell to load embeddings directly into Redis from this notebook\n",
    "\n",
    "import json\n",
    "\n",
    "\n",
    "emb_documents = []\n",
    "\n",
    "\n",
    "for item in os.listdir(\"dump\"):\n",
    "    path = os.path.join(\"dump\", item)\n",
    "\n",
    "    with open(path, 'r') as openfile:\n",
    "        data = json.load(openfile)\n",
    "        \n",
    "    emb_documents += helpers.generate_embeddings(data, CHOSEN_EMB_MODEL, SMALL_EMB_TOKEN_NUM,  text_suffix = 'S')\n",
    "\n",
    "    if MEDIUM_EMB_TOKEN_NUM != 0:\n",
    "        emb_documents += helpers.generate_embeddings(data, CHOSEN_EMB_MODEL, MEDIUM_EMB_TOKEN_NUM, text_suffix = 'M')\n",
    "\n",
    "    if LARGE_EMB_TOKEN_NUM != 0:\n",
    "        emb_documents += helpers.generate_embeddings(data, CHOSEN_EMB_MODEL, LARGE_EMB_TOKEN_NUM,  text_suffix = 'L')\n",
    "\n",
    "\n",
    "helpers.load_embedding_docs_in_redis(emb_documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "emb_documents = []\n",
    "\n",
    "emb_documents += helpers.generate_embeddings_from_json_docs('dump', ADA_002_EMBEDDING_MODEL, ADA_002_MODEL_MAX_TOKENS, text_suffix='XL', limit=-1)\n",
    "\n",
    "print(f\"Generated {len(emb_documents)} embeddings.\")\n",
    "helpers.save_embdding_docs_to_pkl(emb_documents, \"test.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading 141 embeddings into Redis\n"
     ]
    }
   ],
   "source": [
    "emb_documents = helpers.load_embedding_docs_from_pkl(\"test.pkl\")\n",
    "helpers.load_embedding_docs_in_redis(emb_documents)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "openai",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "4cd1745cb8ae6036b9e0c2149eea914cda9394cff45b1c8e288267fc5648a26d"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}