Semnatic_Search.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "from PIL import Image               # to load images\n",
        "from IPython.display import display # to display images"
      ],
      "metadata": {
        "id": "C_-uyi19vNjG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aYpmmfQ-ZbIh",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9e7d83b4-8f46-482c-ea32-2acd7dd84773"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/873.5 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m873.5/873.5 kB\u001b[0m \u001b[31m31.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m72.0/72.0 kB\u001b[0m \u001b[31m4.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m60.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m90.0/90.0 kB\u001b[0m \u001b[31m10.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.5/114.5 kB\u001b[0m \u001b[31m10.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m268.8/268.8 kB\u001b[0m \u001b[31m29.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m149.6/149.6 kB\u001b[0m \u001b[31m14.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.1/49.1 kB\u001b[0m \u001b[31m5.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ],
      "source": [
        "!pip install --upgrade langchain openai  -q"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "os.environ[\"OPENAI_API_KEY\"] = \"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\""
      ],
      "metadata": {
        "id": "5D5W_BeBZf5X"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install unstructured -q\n",
        "!pip install unstructured[local-inference] -q\n",
        "!pip install detectron2@git+https://github.com/facebookresearch/detectron2.git@v0.6#egg=detectron2 -q"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XbL5zdf9aG8a",
        "outputId": "60cf911c-3d70-4b71-edb0-4b04f8a6d6fa"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!apt-get install poppler-utils  "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HwalYTVZoRlH",
        "outputId": "9f475bea-721c-479f-c1fb-ea0ad263c9b7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "poppler-utils is already the newest version (0.86.1-0ubuntu1.1).\n",
            "0 upgraded, 0 newly installed, 0 to remove and 24 not upgraded.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.document_loaders import DirectoryLoader\n",
        "\n",
        "directory = '/content/....'\n",
        "\n",
        "def load_docs(directory):\n",
        "  loader = DirectoryLoader(directory)\n",
        "  documents = loader.load()\n",
        "  return documents\n",
        "\n",
        "documents = load_docs(directory)\n",
        "len(documents)"
      ],
      "metadata": {
        "id": "fulYnj9nZr3n"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "\n",
        "def split_docs(documents,chunk_size=1000,chunk_overlap=20):\n",
        "  text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)\n",
        "  docs = text_splitter.split_documents(documents)\n",
        "  return docs\n",
        "\n",
        "docs = split_docs(documents)\n",
        "print(len(docs))"
      ],
      "metadata": {
        "id": "5lF8jA6xZ0Hm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(docs[0].page_content)"
      ],
      "metadata": {
        "id": "tVcFjgHJZ8S9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(docs[5].page_content)"
      ],
      "metadata": {
        "id": "PGXlt51N2xyJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#requires for open ai embedding\n",
        "!pip install tiktoken -q"
      ],
      "metadata": {
        "id": "6PiPwt-FaYwl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import openai\n",
        "from langchain.embeddings.openai import OpenAIEmbeddings\n",
        "\n",
        "embeddings = OpenAIEmbeddings(model_name=\"ada\")\n",
        "\n",
        "query_result = embeddings.embed_query(\"Hello world\")\n",
        "len(query_result)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "F5GY9voPa0av",
        "outputId": "5fd540c3-8e46-4863-aa07-8fcde3e0d822"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1024"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install pinecone-client -q"
      ],
      "metadata": {
        "id": "LXhIY5SrrRec"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pinecone \n",
        "from langchain.vectorstores import Pinecone\n",
        "# initialize pinecone\n",
        "pinecone.init(\n",
        "    api_key= \"....\",  # find at app.pinecone.io\n",
        "    environment= \"....\"  # next to api key in console\n",
        ")\n",
        "\n",
        "index_name = \"langchain\"\n",
        "\n",
        "index = Pinecone.from_documents(docs, embeddings, index_name=index_name)"
      ],
      "metadata": {
        "id": "hfIpYLV-acks"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_similiar_docs(query,k=2,score=False):\n",
        "  if score:\n",
        "    similar_docs = index.similarity_search_with_score(query,k=k)\n",
        "  else:\n",
        "    similar_docs = index.similarity_search(query,k=k)\n",
        "  return similar_docs\n",
        "\n",
        "query = \"How is the economy\"\n",
        "similar_docs = get_similiar_docs(query)\n",
        "similar_docs"
      ],
      "metadata": {
        "id": "o5r7YLpbchAD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.llms import OpenAI\n",
        "\n",
        "# model_name = \"text-davinci-003\"\n",
        "# model_name = \"gpt-3.5-turbo\"\n",
        "model_name = \"gpt-4\"\n",
        "llm = OpenAI(model_name=model_name)"
      ],
      "metadata": {
        "id": "DuevPx4dbI4W"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.chains.question_answering import load_qa_chain\n",
        "chain = load_qa_chain(llm, chain_type=\"stuff\")\n",
        "\n",
        "def get_answer(query):\n",
        "  similar_docs = get_similiar_docs(query)\n",
        "  # print(similar_docs)\n",
        "  answer =  chain.run(input_documents=similar_docs, question=query)\n",
        "  return  answer\n",
        "\n",
        "query = \"How is the economy\"  \n",
        "get_answer(query)"
      ],
      "metadata": {
        "id": "RqCE-C3Ubty0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "query = \"How did the relations improve?\"\n",
        "get_answer(query)"
      ],
      "metadata": {
        "id": "RTnCqRJ3r1kf"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}