Custom_OpenAI_Chatbot_with_ML.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NKoeyn5BlD_r",
        "outputId": "4d490d74-3320-4643-cc58-c3bd54c9372a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting openai\n",
            "  Downloading openai-1.14.1-py3-none-any.whl (257 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m257.5/257.5 kB\u001b[0m \u001b[31m6.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai) (3.7.1)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai) (1.7.0)\n",
            "Collecting httpx<1,>=0.23.0 (from openai)\n",
            "  Downloading httpx-0.27.0-py3-none-any.whl (75 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m9.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pydantic<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from openai) (2.6.4)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai) (1.3.1)\n",
            "Requirement already satisfied: tqdm>4 in /usr/local/lib/python3.10/dist-packages (from openai) (4.66.2)\n",
            "Requirement already satisfied: typing-extensions<5,>=4.7 in /usr/local/lib/python3.10/dist-packages (from openai) (4.10.0)\n",
            "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai) (3.6)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai) (1.2.0)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai) (2024.2.2)\n",
            "Collecting httpcore==1.* (from httpx<1,>=0.23.0->openai)\n",
            "  Downloading httpcore-1.0.4-py3-none-any.whl (77 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.8/77.8 kB\u001b[0m \u001b[31m10.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting h11<0.15,>=0.13 (from httpcore==1.*->httpx<1,>=0.23.0->openai)\n",
            "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m7.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai) (0.6.0)\n",
            "Requirement already satisfied: pydantic-core==2.16.3 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai) (2.16.3)\n",
            "Installing collected packages: h11, httpcore, httpx, openai\n",
            "Successfully installed h11-0.14.0 httpcore-1.0.4 httpx-0.27.0 openai-1.14.1\n"
          ]
        }
      ],
      "source": [
        "!pip install openai"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import openai\n",
        "openai.api_key = \"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\"\n",
        "import json"
      ],
      "metadata": {
        "id": "DwYNVaMVlVuc"
      },
      "execution_count": 46,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "ukraine_prompt = \"\"\"\n",
        "Question: \"When did Russia invade Ukraine?\"\n",
        "Answer:\n",
        "\"\"\""
      ],
      "metadata": {
        "id": "EjqSlXo-lqFl"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "initial_ukraine_answer = openai.completions.create(\n",
        "    model=\"gpt-3.5-turbo-instruct\",\n",
        "    prompt=ukraine_prompt,\n",
        "    max_tokens=150\n",
        ")"
      ],
      "metadata": {
        "id": "usvwq_N-l6x2"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "initial_ukraine_answer"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ob4g2sqOl-Fx",
        "outputId": "f9949e7a-d138-460a-ef24-6d34c9ee95c6"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Completion(id='cmpl-93pFmpfT4cSrpnLMDXO6Q9hsMuo5o', choices=[CompletionChoice(finish_reason='stop', index=0, logprobs=None, text='Russia invaded Ukraine in February 2014.')], created=1710699030, model='gpt-3.5-turbo-instruct', object='text_completion', system_fingerprint=None, usage=CompletionUsage(completion_tokens=9, prompt_tokens=12, total_tokens=21))"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "initial_ukraine_answer = initial_ukraine_answer.choices[0].text"
      ],
      "metadata": {
        "id": "ySPvlMnbm-tD"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from dateutil.parser import parse\n",
        "import pandas as pd\n",
        "import requests\n",
        "\n",
        "# Get the Wikipedia page for \"2022\" since OpenAI's models stop in 2021\n",
        "resp = requests.get(\"https://en.wikipedia.org/w/api.php?action=query&prop=extracts&exlimit=1&titles=2022&explaintext=1&formatversion=2&format=json\")\n",
        "\n",
        "# Load page text into a dataframe\n",
        "df = pd.DataFrame()\n",
        "df[\"text\"] = resp.json()[\"query\"][\"pages\"][0][\"extract\"].split(\"\\n\")\n",
        "\n",
        "# Clean up text to remove empty lines and headings\n",
        "df = df[(df[\"text\"].str.len() > 0) & (~df[\"text\"].str.startswith(\"==\"))]\n",
        "\n",
        "# In some cases dates are used as headings instead of being part of the\n",
        "# text sample; adjust so dated text samples start with dates\n",
        "prefix = \"\"\n",
        "for (i, row) in df.iterrows():\n",
        "    # If the row already has \" - \", it already has the needed date prefix\n",
        "    if \" – \" not in row[\"text\"]:\n",
        "        try:\n",
        "            # If the row's text is a date, set it as the new prefix\n",
        "            parse(row[\"text\"])\n",
        "            prefix = row[\"text\"]\n",
        "        except:\n",
        "            # If the row's text isn't a date, add the prefix\n",
        "            row[\"text\"] = prefix + \" – \" + row[\"text\"]\n",
        "df = df[df[\"text\"].str.contains(\" – \")]\n",
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "hZYESAV3nDPG",
        "outputId": "ede96638-82e2-4656-a977-c42ef419dd4a"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                  text\n",
              "0     – 2022 (MMXXII) was a common year starting on...\n",
              "1     – The year saw the removal of nearly all COVI...\n",
              "2     – 2022 was also dominated by wars and armed c...\n",
              "9     January 1 – The Regional Comprehensive Econom...\n",
              "10   January 2 – Abdalla Hamdok resigns as Prime Mi...\n",
              "..                                                 ...\n",
              "239  December 21–December 26 – A major winter storm...\n",
              "240  December 24 – 2022 Fijian general election: Th...\n",
              "241  December 29 – Brazilian football legend Pelé d...\n",
              "242  December 31 – Former Pope Benedict XVI dies at...\n",
              "246  December 7 – The world population was estimate...\n",
              "\n",
              "[182 rows x 1 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-dea9cb2f-8c48-4081-9af6-8ec38112af00\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>– 2022 (MMXXII) was a common year starting on...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>– The year saw the removal of nearly all COVI...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>– 2022 was also dominated by wars and armed c...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>January 1 – The Regional Comprehensive Econom...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>January 2 – Abdalla Hamdok resigns as Prime Mi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>239</th>\n",
              "      <td>December 21–December 26 – A major winter storm...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>240</th>\n",
              "      <td>December 24 – 2022 Fijian general election: Th...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>241</th>\n",
              "      <td>December 29 – Brazilian football legend Pelé d...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>242</th>\n",
              "      <td>December 31 – Former Pope Benedict XVI dies at...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>246</th>\n",
              "      <td>December 7 – The world population was estimate...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>182 rows × 1 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-dea9cb2f-8c48-4081-9af6-8ec38112af00')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-dea9cb2f-8c48-4081-9af6-8ec38112af00 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-dea9cb2f-8c48-4081-9af6-8ec38112af00');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-4f85cf94-db9a-44cd-978a-e7d652ab8a4b\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-4f85cf94-db9a-44cd-978a-e7d652ab8a4b')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-4f85cf94-db9a-44cd-978a-e7d652ab8a4b button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "  <div id=\"id_a308164d-e828-420c-8540-975930187ea8\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('df')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_a308164d-e828-420c-8540-975930187ea8 button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('df');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df",
              "summary": "{\n  \"name\": \"df\",\n  \"rows\": 182,\n  \"fields\": [\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 182,\n        \"samples\": [\n          \"February 4 \\u2013 February 20 \\u2013 The 2022 Winter Olympics are held in Beijing, China, making it the first city ever to host both the Summer Olympics and Winter Olympics.\",\n          \"March 7 \\u2013 COVID-19 pandemic: The global death toll from COVID-19 surpasses 6 million.\",\n          \"October 30 \\u2013 2022 Brazilian general election: Luiz In\\u00e1cio Lula da Silva defeats incumbent Jair Bolsonaro in a runoff election after neither candidate secured a majority in the first round of voting. Lula da Silva is the first president elected three times, and also the oldest one, at 77.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#We'll use the Embedding tooling from OpenAI\n",
        "#https://platform.openai.com/docs/guides/embeddings/embeddings\n",
        "#In order to avoid a RateLimitError we'll send our data in batches to the Embedding.create function."
      ],
      "metadata": {
        "id": "7-tqD_GgneEp"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from openai import OpenAI\n",
        "#client = OpenAI()\n",
        "\n",
        "def get_embedding(text, model=\"text-embedding-3-small\"):\n",
        "   text = text.replace(\"\\n\", \" \")\n",
        "   return openai.embeddings.create(input = [text], model=model).data[0].embedding\n",
        "\n",
        "df['ada_embedding'] =df['text'].apply(get_embedding)\n",
        "\n"
      ],
      "metadata": {
        "id": "YIrMhgO3uZbh"
      },
      "execution_count": 60,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "ypXJmJLH-s-j",
        "outputId": "97271c19-f3f0-4610-9670-aadb07ea52ca"
      },
      "execution_count": 80,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                 text  \\\n",
              "0    – 2022 (MMXXII) was a common year starting on...   \n",
              "1    – The year saw the removal of nearly all COVI...   \n",
              "2    – 2022 was also dominated by wars and armed c...   \n",
              "9    January 1 – The Regional Comprehensive Econom...   \n",
              "10  January 2 – Abdalla Hamdok resigns as Prime Mi...   \n",
              "\n",
              "                                        ada_embedding  \n",
              "0   [0.014851835556328297, -0.019367381930351257, ...  \n",
              "1   [-0.0037521650083363056, 0.027671359479427338,...  \n",
              "2   [-0.055763185024261475, 0.00849777553230524, 0...  \n",
              "9   [-0.03662966191768646, 0.017913617193698883, 0...  \n",
              "10  [0.027225859463214874, -0.024198608472943306, ...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-d8410048-c078-4af4-8a43-71f5e523c29c\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>ada_embedding</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>– 2022 (MMXXII) was a common year starting on...</td>\n",
              "      <td>[0.014851835556328297, -0.019367381930351257, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>– The year saw the removal of nearly all COVI...</td>\n",
              "      <td>[-0.0037521650083363056, 0.027671359479427338,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>– 2022 was also dominated by wars and armed c...</td>\n",
              "      <td>[-0.055763185024261475, 0.00849777553230524, 0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>January 1 – The Regional Comprehensive Econom...</td>\n",
              "      <td>[-0.03662966191768646, 0.017913617193698883, 0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>January 2 – Abdalla Hamdok resigns as Prime Mi...</td>\n",
              "      <td>[0.027225859463214874, -0.024198608472943306, ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d8410048-c078-4af4-8a43-71f5e523c29c')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-d8410048-c078-4af4-8a43-71f5e523c29c button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-d8410048-c078-4af4-8a43-71f5e523c29c');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-9a2faff4-1224-47b8-bf05-16edad84146d\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-9a2faff4-1224-47b8-bf05-16edad84146d')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-9a2faff4-1224-47b8-bf05-16edad84146d button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df",
              "summary": "{\n  \"name\": \"df\",\n  \"rows\": 182,\n  \"fields\": [\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 182,\n        \"samples\": [\n          \"February 4 \\u2013 February 20 \\u2013 The 2022 Winter Olympics are held in Beijing, China, making it the first city ever to host both the Summer Olympics and Winter Olympics.\",\n          \"March 7 \\u2013 COVID-19 pandemic: The global death toll from COVID-19 surpasses 6 million.\",\n          \"October 30 \\u2013 2022 Brazilian general election: Luiz In\\u00e1cio Lula da Silva defeats incumbent Jair Bolsonaro in a runoff election after neither candidate secured a majority in the first round of voting. Lula da Silva is the first president elected three times, and also the oldest one, at 77.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"ada_embedding\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 80
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.to_csv(\"embeddings.csv\")"
      ],
      "metadata": {
        "id": "1QZzcSPrpLGm"
      },
      "execution_count": 61,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install scikit-learn"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Oty1k2G9569J",
        "outputId": "de318467-0468-4fea-8c58-a840c95669af"
      },
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (1.2.2)\n",
            "Requirement already satisfied: numpy>=1.17.3 in /usr/local/lib/python3.10/dist-packages (from scikit-learn) (1.25.2)\n",
            "Requirement already satisfied: scipy>=1.3.2 in /usr/local/lib/python3.10/dist-packages (from scikit-learn) (1.11.4)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn) (1.3.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn) (3.3.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install openai[embeddings]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "09CN4QH95vQE",
        "outputId": "d9e21b12-acfc-46b6-95bd-abb1bc00cd47"
      },
      "execution_count": 76,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: openai[embeddings] in /usr/local/lib/python3.10/dist-packages (1.14.1)\n",
            "\u001b[33mWARNING: openai 1.14.1 does not provide the extra 'embeddings'\u001b[0m\u001b[33m\n",
            "\u001b[0mRequirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai[embeddings]) (3.7.1)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai[embeddings]) (1.7.0)\n",
            "Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from openai[embeddings]) (0.27.0)\n",
            "Requirement already satisfied: pydantic<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from openai[embeddings]) (2.6.4)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai[embeddings]) (1.3.1)\n",
            "Requirement already satisfied: tqdm>4 in /usr/local/lib/python3.10/dist-packages (from openai[embeddings]) (4.66.2)\n",
            "Requirement already satisfied: typing-extensions<5,>=4.7 in /usr/local/lib/python3.10/dist-packages (from openai[embeddings]) (4.10.0)\n",
            "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai[embeddings]) (3.6)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai[embeddings]) (1.2.0)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai[embeddings]) (2024.2.2)\n",
            "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai[embeddings]) (1.0.4)\n",
            "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai[embeddings]) (0.14.0)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai[embeddings]) (0.6.0)\n",
            "Requirement already satisfied: pydantic-core==2.16.3 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai[embeddings]) (2.16.3)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import openai\n",
        "import numpy as np"
      ],
      "metadata": {
        "id": "Uk4Hh1b2pjRh"
      },
      "execution_count": 77,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def distances_from_embeddings(embedding1, embeddings2, distance_metric=\"cosine\"):\n",
        "    \"\"\"\n",
        "    Function to compute distances between two sets of embeddings.\n",
        "    \"\"\"\n",
        "    distances = []\n",
        "    for embedding2 in embeddings2:\n",
        "        # Compute cosine distance between embeddings\n",
        "        distance = np.dot(embedding1, embedding2) / (np.linalg.norm(embedding1) * np.linalg.norm(embedding2))\n",
        "        distances.append(distance)\n",
        "    return distances\n",
        "\n",
        "def get_rows_sorted_by_relevance(question, df):\n",
        "    \"\"\"\n",
        "    Function that takes in a question string and a dataframe containing\n",
        "    rows of text and associated embeddings, and returns that dataframe\n",
        "    sorted from least to most relevant for that question\n",
        "    \"\"\"\n",
        "    # Get embeddings for the question text\n",
        "    question_embedding = get_embedding(question)\n",
        "\n",
        "    # Compute cosine distances between question embedding and embeddings in the DataFrame\n",
        "    distances = distances_from_embeddings(question_embedding, df[\"ada_embedding\"].values)\n",
        "\n",
        "    # Make a copy of the DataFrame and add a \"distances\" column\n",
        "    df_copy = df.copy()\n",
        "    df_copy[\"distances\"] = distances\n",
        "\n",
        "    # Sort the copied DataFrame by the distances and return it\n",
        "    # (shorter distance = more relevant so we sort in ascending order)\n",
        "    df_copy.sort_values(\"distances\", ascending=False, inplace=True)\n",
        "    return df_copy"
      ],
      "metadata": {
        "id": "ScDGorYh-Bhh"
      },
      "execution_count": 108,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "get_rows_sorted_by_relevance(\"When did Russia invade Ukraine?\", df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 597
        },
        "id": "ww-FaLeI-FT5",
        "outputId": "eae87cde-82ce-46f2-a6e7-1e3c763762c6"
      },
      "execution_count": 109,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                  text  \\\n",
              "51   March 2 – Russian invasion of Ukraine: Russia ...   \n",
              "194  October 8 – Russian invasion of Ukraine: An ex...   \n",
              "177  September 21 – Russian invasion of Ukraine: Fo...   \n",
              "49   March 1 – Russian invasion of Ukraine: In an e...   \n",
              "204  October 29 – Russian invasion of Ukraine: In r...   \n",
              "..                                                 ...   \n",
              "224  November 19–November 26 – The 2022 Central Ame...   \n",
              "213  November 1 – 2022 Danish general election: A b...   \n",
              "81   April 4 – The Intergovernmental Panel on Clima...   \n",
              "242  December 31 – Former Pope Benedict XVI dies at...   \n",
              "241  December 29 – Brazilian football legend Pelé d...   \n",
              "\n",
              "                                         ada_embedding  distances  \n",
              "51   [-0.028382889926433563, 0.0018178632017225027,...   0.642622  \n",
              "194  [-0.0289325974881649, 0.04709221050143242, 0.0...   0.641142  \n",
              "177  [-0.05499464273452759, 0.018673701211810112, 0...   0.633080  \n",
              "49   [-0.030894501134753227, 0.012962727807462215, ...   0.609966  \n",
              "204  [-0.08263245224952698, 0.014531219378113747, 0...   0.609376  \n",
              "..                                                 ...        ...  \n",
              "224  [0.011729566380381584, -0.036321211606264114, ...   0.020352  \n",
              "213  [-0.011684443801641464, -0.0028280820697546005...   0.016193  \n",
              "81   [0.04204146936535835, 0.00775872590020299, 0.0...   0.015254  \n",
              "242  [-0.01116782333701849, -0.0330800898373127, 0....   0.004484  \n",
              "241  [0.04179256409406662, -0.005502953194081783, -...  -0.041605  \n",
              "\n",
              "[182 rows x 3 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-83904a27-8731-4cfc-be33-cd4ce245ca12\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>ada_embedding</th>\n",
              "      <th>distances</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>51</th>\n",
              "      <td>March 2 – Russian invasion of Ukraine: Russia ...</td>\n",
              "      <td>[-0.028382889926433563, 0.0018178632017225027,...</td>\n",
              "      <td>0.642622</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>194</th>\n",
              "      <td>October 8 – Russian invasion of Ukraine: An ex...</td>\n",
              "      <td>[-0.0289325974881649, 0.04709221050143242, 0.0...</td>\n",
              "      <td>0.641142</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>177</th>\n",
              "      <td>September 21 – Russian invasion of Ukraine: Fo...</td>\n",
              "      <td>[-0.05499464273452759, 0.018673701211810112, 0...</td>\n",
              "      <td>0.633080</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>March 1 – Russian invasion of Ukraine: In an e...</td>\n",
              "      <td>[-0.030894501134753227, 0.012962727807462215, ...</td>\n",
              "      <td>0.609966</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>204</th>\n",
              "      <td>October 29 – Russian invasion of Ukraine: In r...</td>\n",
              "      <td>[-0.08263245224952698, 0.014531219378113747, 0...</td>\n",
              "      <td>0.609376</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>224</th>\n",
              "      <td>November 19–November 26 – The 2022 Central Ame...</td>\n",
              "      <td>[0.011729566380381584, -0.036321211606264114, ...</td>\n",
              "      <td>0.020352</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>213</th>\n",
              "      <td>November 1 – 2022 Danish general election: A b...</td>\n",
              "      <td>[-0.011684443801641464, -0.0028280820697546005...</td>\n",
              "      <td>0.016193</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>81</th>\n",
              "      <td>April 4 – The Intergovernmental Panel on Clima...</td>\n",
              "      <td>[0.04204146936535835, 0.00775872590020299, 0.0...</td>\n",
              "      <td>0.015254</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>242</th>\n",
              "      <td>December 31 – Former Pope Benedict XVI dies at...</td>\n",
              "      <td>[-0.01116782333701849, -0.0330800898373127, 0....</td>\n",
              "      <td>0.004484</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>241</th>\n",
              "      <td>December 29 – Brazilian football legend Pelé d...</td>\n",
              "      <td>[0.04179256409406662, -0.005502953194081783, -...</td>\n",
              "      <td>-0.041605</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>182 rows × 3 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-83904a27-8731-4cfc-be33-cd4ce245ca12')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-83904a27-8731-4cfc-be33-cd4ce245ca12 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-83904a27-8731-4cfc-be33-cd4ce245ca12');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-992ac23b-4255-4d21-a6cc-c7bf4f3eca9c\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-992ac23b-4255-4d21-a6cc-c7bf4f3eca9c')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-992ac23b-4255-4d21-a6cc-c7bf4f3eca9c button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"get_rows_sorted_by_relevance(\\\"When did Russia invade Ukraine?\\\", df)\",\n  \"rows\": 182,\n  \"fields\": [\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 182,\n        \"samples\": [\n          \"April 8 \\u2013 Russian invasion of Ukraine: Russia is condemned by world leaders following a missile attack on Kramatorsk train station, which kills 59 civilians trying to evacuate, including seven children.\",\n          \"January 5 \\u2013 A nationwide state of emergency is declared in Kazakhstan in response to the 2022 Kazakh unrest. The cabinet of prime minister Askar Mamin resigns, while president Kassym-Jomart Tokayev removes former president Nursultan Nazarbayev, widely regarded as being the real power in the country, from his position as Chairman of the Security Council of Kazakhstan.\",\n          \"October 30 \\u2013 2022 Brazilian general election: Luiz In\\u00e1cio Lula da Silva defeats incumbent Jair Bolsonaro in a runoff election after neither candidate secured a majority in the first round of voting. Lula da Silva is the first president elected three times, and also the oldest one, at 77.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"ada_embedding\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"distances\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.17443947579458244,\n        \"min\": -0.04160466514185154,\n        \"max\": 0.6426215630551203,\n        \"num_unique_values\": 182,\n        \"samples\": [\n          0.5173531005158769,\n          0.25090979103288247,\n          0.04794675755292717\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 109
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "get_rows_sorted_by_relevance(\"Who owns Twitter?\", df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 597
        },
        "id": "KHIAOilq-OWU",
        "outputId": "388b00bd-d7bd-40f8-d572-67057bcca207"
      },
      "execution_count": 110,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                  text  \\\n",
              "199  October 28 – Elon Musk completes his $44 billi...   \n",
              "97   April 25 – Elon Musk reaches an agreement to a...   \n",
              "217  November 11 – The cryptocurrency exchange FTX,...   \n",
              "1     – The year saw the removal of nearly all COVI...   \n",
              "228  November 30 – OpenAI releases ChatGPT, an arti...   \n",
              "..                                                 ...   \n",
              "72   March 27 – The M23 offensive begins in North K...   \n",
              "227  November 21 – A 5.6 earthquake strikes near Ci...   \n",
              "59   March 5 – Researchers in the Antarctic find En...   \n",
              "224  November 19–November 26 – The 2022 Central Ame...   \n",
              "172  September 12 – September 2022 Armenia–Azerbaij...   \n",
              "\n",
              "                                         ada_embedding  distances  \n",
              "199  [-0.015518752858042717, -0.06498844921588898, ...   0.570004  \n",
              "97   [0.01241325493901968, -0.044606246054172516, -...   0.560778  \n",
              "217  [-0.04513799771666527, -0.025073962286114693, ...   0.235093  \n",
              "1    [-0.0037521650083363056, 0.027671359479427338,...   0.149112  \n",
              "228  [0.0053897760808467865, 0.020050955936312675, ...   0.148426  \n",
              "..                                                 ...        ...  \n",
              "72   [0.019150277599692345, 0.025272000581026077, 0...  -0.006288  \n",
              "227  [-0.04527856409549713, 0.042859405279159546, 0...  -0.013649  \n",
              "59   [0.031138721853494644, 0.0263734869658947, 0.0...  -0.013952  \n",
              "224  [0.011729566380381584, -0.036321211606264114, ...  -0.020350  \n",
              "172  [-0.029692646116018295, 0.014991875737905502, ...  -0.029827  \n",
              "\n",
              "[182 rows x 3 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-298a9600-be06-4221-8412-19826dfa10bd\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>ada_embedding</th>\n",
              "      <th>distances</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>199</th>\n",
              "      <td>October 28 – Elon Musk completes his $44 billi...</td>\n",
              "      <td>[-0.015518752858042717, -0.06498844921588898, ...</td>\n",
              "      <td>0.570004</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>97</th>\n",
              "      <td>April 25 – Elon Musk reaches an agreement to a...</td>\n",
              "      <td>[0.01241325493901968, -0.044606246054172516, -...</td>\n",
              "      <td>0.560778</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>217</th>\n",
              "      <td>November 11 – The cryptocurrency exchange FTX,...</td>\n",
              "      <td>[-0.04513799771666527, -0.025073962286114693, ...</td>\n",
              "      <td>0.235093</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>– The year saw the removal of nearly all COVI...</td>\n",
              "      <td>[-0.0037521650083363056, 0.027671359479427338,...</td>\n",
              "      <td>0.149112</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>228</th>\n",
              "      <td>November 30 – OpenAI releases ChatGPT, an arti...</td>\n",
              "      <td>[0.0053897760808467865, 0.020050955936312675, ...</td>\n",
              "      <td>0.148426</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>72</th>\n",
              "      <td>March 27 – The M23 offensive begins in North K...</td>\n",
              "      <td>[0.019150277599692345, 0.025272000581026077, 0...</td>\n",
              "      <td>-0.006288</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>227</th>\n",
              "      <td>November 21 – A 5.6 earthquake strikes near Ci...</td>\n",
              "      <td>[-0.04527856409549713, 0.042859405279159546, 0...</td>\n",
              "      <td>-0.013649</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59</th>\n",
              "      <td>March 5 – Researchers in the Antarctic find En...</td>\n",
              "      <td>[0.031138721853494644, 0.0263734869658947, 0.0...</td>\n",
              "      <td>-0.013952</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>224</th>\n",
              "      <td>November 19–November 26 – The 2022 Central Ame...</td>\n",
              "      <td>[0.011729566380381584, -0.036321211606264114, ...</td>\n",
              "      <td>-0.020350</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>172</th>\n",
              "      <td>September 12 – September 2022 Armenia–Azerbaij...</td>\n",
              "      <td>[-0.029692646116018295, 0.014991875737905502, ...</td>\n",
              "      <td>-0.029827</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>182 rows × 3 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-298a9600-be06-4221-8412-19826dfa10bd')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-298a9600-be06-4221-8412-19826dfa10bd button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-298a9600-be06-4221-8412-19826dfa10bd');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-612391d9-c41f-485b-ad70-8e1dcbae6074\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-612391d9-c41f-485b-ad70-8e1dcbae6074')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-612391d9-c41f-485b-ad70-8e1dcbae6074 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"get_rows_sorted_by_relevance(\\\"Who owns Twitter?\\\", df)\",\n  \"rows\": 182,\n  \"fields\": [\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 182,\n        \"samples\": [\n          \"April 3 \\u2013 The second round of voting of the 2022 Costa Rican general election is held, and Social Democratic Progress Party presidential candidate Rodrigo Chaves Robles is elected president.\",\n          \"May 15 \\u2013 Former President of Somalia Hassan Sheikh Mohamud is elected president again, beating President Mohamed Abdullahi Mohamed.\",\n          \"July 28\\u2013August 8 \\u2013 The 2022 Commonwealth Games is held in Birmingham, England.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"ada_embedding\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"distances\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.06626085372170983,\n        \"min\": -0.02982682811427173,\n        \"max\": 0.5700036459865164,\n        \"num_unique_values\": 182,\n        \"samples\": [\n          0.10969186424638085,\n          0.0877128090274494,\n          0.018586408333345666\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 110
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install tiktoken"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TX3B-Nmw_YxF",
        "outputId": "9d43b61f-5c7b-40de-e942-867ac0e3b0ff"
      },
      "execution_count": 85,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting tiktoken\n",
            "  Downloading tiktoken-0.6.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m14.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken) (2023.12.25)\n",
            "Requirement already satisfied: requests>=2.26.0 in /usr/local/lib/python3.10/dist-packages (from tiktoken) (2.31.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (2024.2.2)\n",
            "Installing collected packages: tiktoken\n",
            "Successfully installed tiktoken-0.6.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import tiktoken\n",
        "\n",
        "def create_prompt(question, df, max_token_count):\n",
        "    \"\"\"\n",
        "    Given a question and a dataframe containing rows of text and their\n",
        "    embeddings, return a text prompt to send to a Completion model\n",
        "    \"\"\"\n",
        "    # Create a tokenizer that is designed to align with our embeddings\n",
        "    tokenizer = tiktoken.get_encoding(\"cl100k_base\")\n",
        "\n",
        "    # Count the number of tokens in the prompt template and question\n",
        "    prompt_template = \"\"\"\n",
        "Answer the question based on the context below, and if the question\n",
        "can't be answered based on the context, say \"I don't know\"\n",
        "\n",
        "Context:\n",
        "\n",
        "{}\n",
        "\n",
        "---\n",
        "\n",
        "Question: {}\n",
        "Answer:\"\"\"\n",
        "\n",
        "    current_token_count = len(tokenizer.encode(prompt_template)) + \\\n",
        "                            len(tokenizer.encode(question))\n",
        "\n",
        "    context = []\n",
        "    for text in get_rows_sorted_by_relevance(question, df)[\"text\"].values:\n",
        "\n",
        "        # Increase the counter based on the number of tokens in this row\n",
        "        text_token_count = len(tokenizer.encode(text))\n",
        "        current_token_count += text_token_count\n",
        "\n",
        "        # Add the row of text to the list if we haven't exceeded the max\n",
        "        if current_token_count <= max_token_count:\n",
        "            context.append(text)\n",
        "        else:\n",
        "            break\n",
        "\n",
        "    return prompt_template.format(\"\\n\\n###\\n\\n\".join(context), question)\n",
        ""
      ],
      "metadata": {
        "id": "A8VMNpv7-5ES"
      },
      "execution_count": 111,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(create_prompt(\"When did Russia invade Ukraine?\", df, 200))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WFL1FpMd_V_-",
        "outputId": "0c1c64e8-a031-4a4f-f01f-9c8f0791e2cd"
      },
      "execution_count": 112,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Answer the question based on the context below, and if the question\n",
            "can't be answered based on the context, say \"I don't know\"\n",
            "\n",
            "Context: \n",
            "\n",
            "March 2 – Russian invasion of Ukraine: Russia captures its first large city, the Black Sea port of Kherson, as shelling intensifies across many parts of Ukraine, including civilian areas.\n",
            "\n",
            "###\n",
            "\n",
            "October 8 – Russian invasion of Ukraine: An explosion occurs on the Crimean Bridge connecting Crimea and Russia, killing three and causing a partial collapse of the only road bridge between the Crimean Peninsula and the Russian mainland. Two days later, retaliatory missile strikes are conducted by Russia across Ukraine, the most widespread since the start of the invasion, notably including attacks on Kyiv.\n",
            "\n",
            "---\n",
            "\n",
            "Question: When did Russia invade Ukraine?\n",
            "Answer:\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(create_prompt(\"Who owns Twitter?\", df, 100))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s50XJOJ4_tOJ",
        "outputId": "4f4887b6-9ab3-42b0-c802-18b0b3100cb7"
      },
      "execution_count": 113,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Answer the question based on the context below, and if the question\n",
            "can't be answered based on the context, say \"I don't know\"\n",
            "\n",
            "Context: \n",
            "\n",
            "October 28 – Elon Musk completes his $44 billion acquisition of Twitter.\n",
            "\n",
            "###\n",
            "\n",
            "April 25 – Elon Musk reaches an agreement to acquire the social media network Twitter (which he later rebrands as X) for $44 billion USD, which later closes in October.\n",
            "\n",
            "---\n",
            "\n",
            "Question: Who owns Twitter?\n",
            "Answer:\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "COMPLETION_MODEL_NAME = \"gpt-3.5-turbo-instruct\"\n",
        "\n",
        "def answer_question(\n",
        "    question, df, max_prompt_tokens=1800, max_answer_tokens=150\n",
        "):\n",
        "    \"\"\"\n",
        "    Given a question, a dataframe containing rows of text, and a maximum\n",
        "    number of desired tokens in the prompt and response, return the\n",
        "    answer to the question according to an OpenAI Completion model\n",
        "\n",
        "    If the model produces an error, return an empty string\n",
        "    \"\"\"\n",
        "\n",
        "    prompt = create_prompt(question, df, max_prompt_tokens)\n",
        "\n",
        "    try:\n",
        "        response = openai.completions.create(\n",
        "            model=COMPLETION_MODEL_NAME,\n",
        "            prompt=prompt,\n",
        "            max_tokens=max_answer_tokens\n",
        "        )\n",
        "        return response.choices[0].text\n",
        "    except Exception as e:\n",
        "        print(e)\n",
        "        return \"\"\n",
        ""
      ],
      "metadata": {
        "id": "vragB5tH_2-x"
      },
      "execution_count": 114,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "custom_ukraine_answer = answer_question(\"When did Russia invade Ukraine?\", df)\n",
        "print(custom_ukraine_answer)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8RxFFeohAhfl",
        "outputId": "9689129f-58be-46d9-fc76-3de34c8f475f"
      },
      "execution_count": 115,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " February 24\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "custom_twitter_answer = answer_question(\"Who owns Twitter?\", df)\n",
        "print(custom_twitter_answer)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FQDRpxzNAk3W",
        "outputId": "af7475c9-9d09-4ce5-f5b6-8abff33da330"
      },
      "execution_count": 116,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " Elon Musk owns Twitter.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "Pq8fRjYABHt5"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}