openai/embeddings_faiss_urls.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DESCRIPTION:\n",
    "    This example shows how to create embeddings from urls and saves it in a faiss DB in the /dbs/urls/ folder\n",
    "    Then query these faiss index and get an answer using OpenAI GPT3.5\n",
    "### REQUIREMENTS:\n",
    "    Create an .env file with your OpenAI API key and save it in the root directory of this project.\n",
    "\n",
    "  For more information about Faiss index, see:\n",
    "      https://github.com/facebookresearch/faiss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "from langchain.document_loaders import UnstructuredURLLoader\n",
    "from langchain.chains import RetrievalQA\n",
    "from langchain.vectorstores import FAISS\n",
    "from langchain.chains.question_answering import load_qa_chain\n",
    "\n",
    "import utils\n",
    "from dotenv import load_dotenv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# IMPORTANT!!\n",
    "# you only need to run this once to create the embeddings, then you can use the faiss_index for retrieval\n",
    "# you can use the already created faiss_index in the dbs folder - ./dbs/urls/faiss_index\n",
    "\n",
    "# create embeddings from urls and save it to a FAISS index.\n",
    "utils.init_OpenAI()\n",
    "embeddings = utils.init_embeddings()\n",
    "\n",
    "# you can add as many urls as you want, but for this example we will only use one\n",
    "# moby dick book online \n",
    "urls = [\"https://www.gutenberg.org/files/2701/2701-0.txt\"]\n",
    "\n",
    "loader = UnstructuredURLLoader(urls=urls)\n",
    "documents = loader.load()\n",
    "\n",
    "text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100, separators= [\"\\n\\n\", \"\\n\", \".\", \";\", \",\", \" \", \"\"])\n",
    "texts = text_splitter.split_documents(documents)\n",
    "db = FAISS.from_documents(documents=texts, embedding=embeddings)\n",
    "db.save_local(\"./dbs/urls/faiss_index\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "llm = utils.init_llm()\n",
    "embeddings = utils.init_embeddings()\n",
    "vectorStore = FAISS.load_local(\"./dbs/urls/faiss_index\", embeddings)\n",
    "retriever = vectorStore.as_retriever(search_type=\"similarity\", search_kwargs={\"k\":2})\n",
    "qa = RetrievalQA.from_chain_type(llm=llm, chain_type=\"stuff\", retriever=retriever, return_source_documents=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'query': \"Why does the coffin prepared for Queequeg become Ishmael's life buoy once the Pequod sinks?\",\n",
       " 'result': \"There is no direct connection between Queequeg's coffin and Ishmael's life buoy. The life-buoy that Ishmael clings to is described as a long slender cask that had been hanging from the stern of the ship, but it had filled with water and sank with the sailor who was trying to grab it. Queequeg's coffin, on the other hand, was prepared for him by his own request and contained his personal belongings and supplies for his afterlife.\"}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qa({\"query\": \"Why does the coffin prepared for Queequeg become Ishmael's life buoy once the Pequod sinks?\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'query': 'Why does Ahab pursue Moby Dick so single-mindedly?',\n",
       " 'result': 'Ahab is consumed with the hot fire of his purpose, and in all his thoughts and actions ever had in view the ultimate capture of Moby Dick. It is suggested that his vindictiveness towards the White Whale might have possibly extended itself in some degree to all sperm whales, and that the more monsters he slew by so much the more he multiplied the chances that each subsequently encountered whale would prove to be the hated one he hunted.'}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qa({\"query\": \"Why does Ahab pursue Moby Dick so single-mindedly?\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'query': \"Why does the novel's narrator begin his story with 'Call me Ishmael'?\",\n",
       " 'result': 'The novel\\'s narrator begins his story with \"Call me Ishmael\" to introduce himself to the reader by using a pseudonym.'}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qa({\"query\": \"Why does the novel's narrator begin his story with 'Call me Ishmael'?\"})"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.2"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}