notebooks/rag/anthropic_mongodb_pam_ai_stack.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mongodb-developer/GenAI-Showcase/blob/main/notebooks/rag/anthropic_mongodb_pam_ai_stack.ipynb)\n",
        "\n",
        "[![View Article](https://img.shields.io/badge/View%20Article-blue)](https://www.mongodb.com/developer/products/atlas/rag_with_claude_opus_mongodb/)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zkePkAfZLQ_R"
      },
      "outputs": [],
      "source": [
        "!pip install pymongo datasets pandas anthropic openai"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "VoY9qBGJNo2F"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import requests\n",
        "from io import BytesIO\n",
        "import pandas as pd\n",
        "from google.colab import userdata\n",
        "\n",
        "def download_and_combine_parquet_files(parquet_file_urls, hf_token):\n",
        "    \"\"\"\n",
        "    Downloads Parquet files from the provided URLs using the given Hugging Face token,\n",
        "    and returns a combined DataFrame.\n",
        "\n",
        "    Parameters:\n",
        "    - parquet_file_urls: List of strings, URLs to the Parquet files.\n",
        "    - hf_token: String, Hugging Face authorization token.\n",
        "\n",
        "    Returns:\n",
        "    - combined_df: A pandas DataFrame containing the combined data from all Parquet files.\n",
        "    \"\"\"\n",
        "    headers = {\"Authorization\": f\"Bearer {hf_token}\"}\n",
        "    all_dataframes = []\n",
        "\n",
        "    for parquet_file_url in parquet_file_urls:\n",
        "        response = requests.get(parquet_file_url, headers=headers)\n",
        "        if response.status_code == 200:\n",
        "            parquet_bytes = BytesIO(response.content)\n",
        "            df = pd.read_parquet(parquet_bytes)\n",
        "            all_dataframes.append(df)\n",
        "        else:\n",
        "            print(f\"Failed to download Parquet file from {parquet_file_url}: {response.status_code}\")\n",
        "\n",
        "    if all_dataframes:\n",
        "        combined_df = pd.concat(all_dataframes, ignore_index=True)\n",
        "        return combined_df\n",
        "    else:\n",
        "        print(\"No dataframes to concatenate.\")\n",
        "        return None"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "8sa8PrlAOawR"
      },
      "outputs": [],
      "source": [
        "# Uncomment the links below to load more data\n",
        "# For the full list of data visit: https://huggingface.co/datasets/MongoDB/tech-news-embeddings/tree/refs%2Fconvert%2Fparquet/default/train\n",
        "parquet_files = [\n",
        "    \"https://huggingface.co/api/datasets/MongoDB/tech-news-embeddings/parquet/default/train/0000.parquet\",\n",
        "    # \"https://huggingface.co/api/datasets/MongoDB/tech-news-embeddings/parquet/default/train/0001.parquet\",\n",
        "    # \"https://huggingface.co/api/datasets/MongoDB/tech-news-embeddings/parquet/default/train/0002.parquet\",\n",
        "    # \"https://huggingface.co/api/datasets/MongoDB/tech-news-embeddings/parquet/default/train/0003.parquet\",\n",
        "    # \"https://huggingface.co/api/datasets/MongoDB/tech-news-embeddings/parquet/default/train/0004.parquet\",\n",
        "    # \"https://huggingface.co/api/datasets/MongoDB/tech-news-embeddings/parquet/default/train/0005.parquet\",\n",
        "]\n",
        "\n",
        "hf_token = userdata.get(\"HF_TOKEN\")\n",
        "combined_df = download_and_combine_parquet_files(parquet_files, hf_token)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "GEx4i1ehRliI"
      },
      "outputs": [],
      "source": [
        "# Remove the _id coloum from the intital dataset\n",
        "combined_df = combined_df.drop(columns=['_id'])\n",
        "\n",
        "# Convert each numpy array in the 'embedding' column to a normal Python list\n",
        "combined_df['embedding'] = combined_df['embedding'].apply(lambda x: x.tolist())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 293
        },
        "id": "6UsKPi32OWX3",
        "outputId": "231e7041-7b4d-49f7-c7c5-1d81e7e4adea"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "combined_df"
            },
            "text/html": [
              "\n",
              "  <div id=\"df-3ae8e6a4-5609-4514-bf94-1ccf25e2a336\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>companyName</th>\n",
              "      <th>companyUrl</th>\n",
              "      <th>published_at</th>\n",
              "      <th>url</th>\n",
              "      <th>title</th>\n",
              "      <th>main_image</th>\n",
              "      <th>description</th>\n",
              "      <th>embedding</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>01Synergy</td>\n",
              "      <td>https://hackernoon.com/company/01synergy</td>\n",
              "      <td>2023-05-16 02:09:00</td>\n",
              "      <td>https://www.businesswire.com/news/home/2023051...</td>\n",
              "      <td>onsemi and Sineng Electric Spearhead the Devel...</td>\n",
              "      <td>https://firebasestorage.googleapis.com/v0/b/ha...</td>\n",
              "      <td>(Nasdaq: ON) a leader in intelligent power and...</td>\n",
              "      <td>[0.05243798345327377, -0.10347484797239304, -0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>01Synergy</td>\n",
              "      <td>https://hackernoon.com/company/01synergy</td>\n",
              "      <td>2023-05-02 00:07:00</td>\n",
              "      <td>https://elkodaily.com/news/local/adobe-student...</td>\n",
              "      <td>Adobe student receives national Information an...</td>\n",
              "      <td>https://firebasestorage.googleapis.com/v0/b/ha...</td>\n",
              "      <td>ELKO — An eighth grader at Adobe Middle School...</td>\n",
              "      <td>[0.0036485784221440554, -0.05992984399199486, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>01Synergy</td>\n",
              "      <td>https://hackernoon.com/company/01synergy</td>\n",
              "      <td>2023-05-01 22:22:00</td>\n",
              "      <td>https://www.aei.org/technology-and-innovation/...</td>\n",
              "      <td>Modernizing State Services: Harnessing Technol...</td>\n",
              "      <td>https://firebasestorage.googleapis.com/v0/b/ha...</td>\n",
              "      <td>To deliver 21st-century government services Go...</td>\n",
              "      <td>[0.012319465167820454, -0.0807630866765976, 0....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>01Synergy</td>\n",
              "      <td>https://hackernoon.com/company/01synergy</td>\n",
              "      <td>2023-05-02 13:12:00</td>\n",
              "      <td>https://www.crn.com/news/managed-services/terr...</td>\n",
              "      <td>Terry Richardson On Why He Left AMD GreenPages...</td>\n",
              "      <td>https://firebasestorage.googleapis.com/v0/b/ha...</td>\n",
              "      <td>In February GreenPages acquired Toronto-based ...</td>\n",
              "      <td>[-0.02363203465938568, 0.021521812304854393, 0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>01Synergy</td>\n",
              "      <td>https://hackernoon.com/company/01synergy</td>\n",
              "      <td>2023-05-15 20:01:00</td>\n",
              "      <td>https://www.benzinga.com/pressreleases/23/05/3...</td>\n",
              "      <td>Synex Renewable Energy Corporation (Formerly S...</td>\n",
              "      <td>https://firebasestorage.googleapis.com/v0/b/ha...</td>\n",
              "      <td>The conference will bring together growth orie...</td>\n",
              "      <td>[0.08473014086484909, -0.07019763439893723, 0....</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3ae8e6a4-5609-4514-bf94-1ccf25e2a336')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-3ae8e6a4-5609-4514-bf94-1ccf25e2a336 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-3ae8e6a4-5609-4514-bf94-1ccf25e2a336');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-2d77cf66-d123-4be8-b06c-c9f8f53d4a16\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-2d77cf66-d123-4be8-b06c-c9f8f53d4a16')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-2d77cf66-d123-4be8-b06c-c9f8f53d4a16 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "text/plain": [
              "  companyName                                companyUrl         published_at  \\\n",
              "0   01Synergy  https://hackernoon.com/company/01synergy  2023-05-16 02:09:00   \n",
              "1   01Synergy  https://hackernoon.com/company/01synergy  2023-05-02 00:07:00   \n",
              "2   01Synergy  https://hackernoon.com/company/01synergy  2023-05-01 22:22:00   \n",
              "3   01Synergy  https://hackernoon.com/company/01synergy  2023-05-02 13:12:00   \n",
              "4   01Synergy  https://hackernoon.com/company/01synergy  2023-05-15 20:01:00   \n",
              "\n",
              "                                                 url  \\\n",
              "0  https://www.businesswire.com/news/home/2023051...   \n",
              "1  https://elkodaily.com/news/local/adobe-student...   \n",
              "2  https://www.aei.org/technology-and-innovation/...   \n",
              "3  https://www.crn.com/news/managed-services/terr...   \n",
              "4  https://www.benzinga.com/pressreleases/23/05/3...   \n",
              "\n",
              "                                               title  \\\n",
              "0  onsemi and Sineng Electric Spearhead the Devel...   \n",
              "1  Adobe student receives national Information an...   \n",
              "2  Modernizing State Services: Harnessing Technol...   \n",
              "3  Terry Richardson On Why He Left AMD GreenPages...   \n",
              "4  Synex Renewable Energy Corporation (Formerly S...   \n",
              "\n",
              "                                          main_image  \\\n",
              "0  https://firebasestorage.googleapis.com/v0/b/ha...   \n",
              "1  https://firebasestorage.googleapis.com/v0/b/ha...   \n",
              "2  https://firebasestorage.googleapis.com/v0/b/ha...   \n",
              "3  https://firebasestorage.googleapis.com/v0/b/ha...   \n",
              "4  https://firebasestorage.googleapis.com/v0/b/ha...   \n",
              "\n",
              "                                         description  \\\n",
              "0  (Nasdaq: ON) a leader in intelligent power and...   \n",
              "1  ELKO — An eighth grader at Adobe Middle School...   \n",
              "2  To deliver 21st-century government services Go...   \n",
              "3  In February GreenPages acquired Toronto-based ...   \n",
              "4  The conference will bring together growth orie...   \n",
              "\n",
              "                                           embedding  \n",
              "0  [0.05243798345327377, -0.10347484797239304, -0...  \n",
              "1  [0.0036485784221440554, -0.05992984399199486, ...  \n",
              "2  [0.012319465167820454, -0.0807630866765976, 0....  \n",
              "3  [-0.02363203465938568, 0.021521812304854393, 0...  \n",
              "4  [0.08473014086484909, -0.07019763439893723, 0....  "
            ]
          },
          "execution_count": 5,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "combined_df.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GXv9MrqLSZw8"
      },
      "source": [
        "Create Database and Collection\n",
        "Create Vector Search Index"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Yv0dMtWnQhDy",
        "outputId": "5bc5276b-46b3-4bc3-f5e4-19802ff910c3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Connection to MongoDB successful\n"
          ]
        }
      ],
      "source": [
        "import pymongo\n",
        "from google.colab import userdata\n",
        "\n",
        "def get_mongo_client(mongo_uri):\n",
        "  \"\"\"Establish connection to the MongoDB.\"\"\"\n",
        "  try:\n",
        "    client = pymongo.MongoClient(mongo_uri, appname=\"devrel.content.python\")\n",
        "    print(\"Connection to MongoDB successful\")\n",
        "    return client\n",
        "  except pymongo.errors.ConnectionFailure as e:\n",
        "    print(f\"Connection failed: {e}\")\n",
        "    return None\n",
        "\n",
        "mongo_uri = userdata.get('MONGO_URI')\n",
        "if not mongo_uri:\n",
        "  print(\"MONGO_URI not set in environment variables\")\n",
        "\n",
        "mongo_client = get_mongo_client(mongo_uri)\n",
        "\n",
        "DB_NAME=\"tech_news\"\n",
        "COLLECTION_NAME=\"hacker_noon_tech_news\"\n",
        "\n",
        "db = mongo_client[DB_NAME]\n",
        "collection = db[COLLECTION_NAME]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4NowJ2mlQrfT",
        "outputId": "31bfa7ac-dc62-421e-92c5-f644e03d7c1f"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "DeleteResult({'n': 228012, 'electionId': ObjectId('7fffffff000000000000000e'), 'opTime': {'ts': Timestamp(1709660559, 7341), 't': 14}, 'ok': 1.0, '$clusterTime': {'clusterTime': Timestamp(1709660559, 7341), 'signature': {'hash': b'jT\\xf1\\xb4\\xa9\\xd3\\xe3suu\\x03`\\x15(}\\x8f\\x00\\x9f\\xe9\\x8a', 'keyId': 7320226449804230661}}, 'operationTime': Timestamp(1709660559, 7341)}, acknowledged=True)"
            ]
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# To ensure we are working with a fresh collection\n",
        "# delete any existing records in the collection\n",
        "collection.delete_many({})"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tdFsJXXRjEZe"
      },
      "outputs": [],
      "source": [
        "# Data Ingestion\n",
        "combined_df_json = combined_df.to_dict(orient='records')\n",
        "collection.insert_many(combined_df_json)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "import openai\n",
        "from google.colab import userdata\n",
        "\n",
        "openai.api_key = userdata.get(\"OPENAI_API_KEY\")\n",
        "\n",
        "EMBEDDING_MODEL = \"text-embedding-3-small\"\n",
        "\n",
        "def get_embedding(text):\n",
        "    \"\"\"Generate an embedding for the given text using OpenAI's API.\"\"\"\n",
        "\n",
        "    # Check for valid input\n",
        "    if not text or not isinstance(text, str):\n",
        "        return None\n",
        "\n",
        "    try:\n",
        "        # Call OpenAI API to get the embedding\n",
        "        embedding = openai.embeddings.create(input=text, model=EMBEDDING_MODEL, dimensions=256).data[0].embedding\n",
        "        return embedding\n",
        "    except Exception as e:\n",
        "        print(f\"Error in get_embedding: {e}\")\n",
        "        return None"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "QsSBrAu4nSJC"
      },
      "outputs": [],
      "source": [
        "def vector_search(user_query, collection):\n",
        "    \"\"\"\n",
        "    Perform a vector search in the MongoDB collection based on the user query.\n",
        "\n",
        "    Args:\n",
        "    user_query (str): The user's query string.\n",
        "    collection (MongoCollection): The MongoDB collection to search.\n",
        "\n",
        "    Returns:\n",
        "    list: A list of matching documents.\n",
        "    \"\"\"\n",
        "\n",
        "    # Generate embedding for the user query\n",
        "    query_embedding = get_embedding(user_query)\n",
        "\n",
        "    if query_embedding is None:\n",
        "        return \"Invalid query or embedding generation failed.\"\n",
        "\n",
        "    # Define the vector search pipeline\n",
        "    pipeline = [\n",
        "        {\n",
        "            \"$vectorSearch\": {\n",
        "                \"index\": \"vector_index\",\n",
        "                \"queryVector\": query_embedding,\n",
        "                \"path\": \"embedding\",\n",
        "                \"numCandidates\": 150,  # Number of candidate matches to consider\n",
        "                \"limit\": 5  # Return top 5 matches\n",
        "            }\n",
        "        },\n",
        "        {\n",
        "            \"$project\": {\n",
        "                \"_id\": 0,  # Exclude the _id field\n",
        "                \"embedding\": 0,  # Exclude the embedding field\n",
        "                \"score\": {\n",
        "                    \"$meta\": \"vectorSearchScore\"  # Include the search score\n",
        "                }\n",
        "            }\n",
        "        }\n",
        "    ]\n",
        "\n",
        "    # Execute the search\n",
        "    results = collection.aggregate(pipeline)\n",
        "    return list(results)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "wADfSOFOnw0s"
      },
      "outputs": [],
      "source": [
        "import anthropic\n",
        "client = anthropic.Client(api_key=userdata.get(\"ANTHROPIC_API_KEY\"))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "JrG8Ho50nhVR"
      },
      "outputs": [],
      "source": [
        "def handle_user_query(query, collection):\n",
        "\n",
        "  get_knowledge = vector_search(query, collection)\n",
        "\n",
        "  search_result = ''\n",
        "  for result in get_knowledge:\n",
        "    search_result += (\n",
        "        f\"Title: {result.get('title', 'N/A')}, \"\n",
        "        f\"Company Name: {result.get('companyName', 'N/A')}, \"\n",
        "        f\"Company URL: {result.get('companyUrl', 'N/A')}, \"\n",
        "        f\"Date Published: {result.get('published_at', 'N/A')}, \"\n",
        "        f\"Article URL: {result.get('url', 'N/A')}, \"\n",
        "        f\"Description: {result.get('description', 'N/A')}, \\n\"\n",
        "    )\n",
        "\n",
        "  response = client.messages.create(\n",
        "    model=\"claude-3-opus-20240229\",\n",
        "    max_tokens=1024,\n",
        "    system=\"You are Venture Captital Tech Analyst with access to some tech company articles and information. You use the information you are given to provide advice.\",\n",
        "    messages=[\n",
        "        {\"role\": \"user\", \"content\": \"Answer this user query: \" + query + \" with the following context: \" + search_result}\n",
        "    ]\n",
        "  )\n",
        "\n",
        "  return (response.content[0].text), search_result"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_3OcaBbmoY2H",
        "outputId": "06c231ef-cbaa-48cd-e226-9d1c8450b737"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Response: Based on the information provided in the article titles and descriptions, Alibaba Group Holding Limited appears to be a top technology stock pick for 2023 according to renowned investor Ray Dalio. The article \"Top 10 Technology Stocks to Buy in 2023 According to Ray Dalio\" suggests that Alibaba is one of Dalio's favored tech investments for the year.\n",
            "\n",
            "As a venture capital tech analyst, I would recommend considering an investment in Alibaba for the following reasons:\n",
            "\n",
            "1. Endorsement from a respected investor: Ray Dalio, known for his successful investment strategies, has included Alibaba in his top 10 technology stock picks for 2023. His backing lends credibility to the investment potential of the company.\n",
            "\n",
            "2. Strong market position: Alibaba is a leading e-commerce company in China with a significant market share. It has a diversified business model spanning e-commerce, cloud computing, digital media, and entertainment.\n",
            "\n",
            "3. Growth potential: With China's large and growing middle class, Alibaba is well-positioned to benefit from increasing consumer spending and the shift towards online shopping.\n",
            "\n",
            "However, it's essential to consider the following points as well:\n",
            "\n",
            "1. Regulatory risks: Chinese tech companies, including Alibaba, have faced increased regulatory scrutiny in recent times. Changes in government policies could impact the company's growth and profitability.\n",
            "\n",
            "2. Competition: While Alibaba is a dominant player, it faces competition from other tech giants like Tencent and JD.com in various business segments.\n",
            "\n",
            "3. Geopolitical tensions: Ongoing tensions between the U.S. and China could lead to market volatility and impact investor sentiment towards Chinese stocks.\n",
            "\n",
            "As with any investment, it's crucial to conduct thorough research, consider your risk tolerance, and diversify your portfolio. Keep in mind that the information provided here is based on limited data points, and stock prices can be influenced by various factors beyond the scope of this context.\n",
            "Source Information: \\nTitle: Top 10 Technology Stocks to Buy in 2023 According to Ray Dalio, Company Name: ALIBABA GROUP HOLDING LIMITED, Company URL: https://hackernoon.com/company/alibabagroupholdinglimited, Date Published: 2023-04-21 11:58:00, Article URL: https://uk.finance.yahoo.com/news/top-10-technology-stocks-buy-155830366.html, Description: In this article we discuss the top 10 technology stocks to buy in 2023 according to Ray Dalio. If you want to skip our detailed analysis of Dalio’s investment philosophy and portfolio construction, \n",
            "Title: 3 Tech Stocks I Love Right Now, Company Name: 10Clouds, Company URL: https://hackernoon.com/company/10clouds, Date Published: 2023-04-02 11:30:00, Article URL: https://www.msn.com/en-xl/money/other/3-tech-stocks-i-love-right-now/ar-AA19n9Ht, Description: These are tech giants but they''re also great investments., \n",
            "Title: 3 Millionaire-Maker Hydrogen Stocks to Buy Before the Window Closes, Company Name: Air Products & Chemicals, Company URL: https://hackernoon.com/company/airproductschemicals, Date Published: 2023-07-28 12:18:00, Article URL: https://www.msn.com/en-us/money/topstocks/3-millionaire-maker-hydrogen-stocks-to-buy-before-the-window-closes/ar-AA1etN8O, Description: These are the best hydrogen stocks to buy with multibagger returns potential.More From InvestorPlace Buy This $5 Stock BEFORE This Apple Project Goes Live Wall Street Titan: Here’s My #1 Stock for 2023 The $1 Investment You MUST Take Advantage of Right Now It doesn’t matter if you have $500 or $5 million., \n",
            "Title: Why it may be time to sell the pop in tech stocks: BlackRock, Company Name: BlackRock, Company URL: https://hackernoon.com/company/blackrock, Date Published: 2023-02-13 19:06:00, Article URL: https://news.yahoo.com/why-it-may-be-time-to-sell-the-pop-in-tech-stocks-blackrock-190606866.html, Description: Household tech names like Apple Meta and Netflix have soared so far in 2023 but one strategist says the gains aren''t likely to last. It''s time to take profits on tech stocks — the early sector winner of 2023 — as the Federal Reserve may soon dash hopes for a pivot on interest rates, \n",
            "Title: The harsh reality for investors eyeing tech stocks in 2023: Morning Brief, Company Name: 10Clouds, Company URL: https://hackernoon.com/company/10clouds, Date Published: 2023-01-02 11:18:00, Article URL: https://news.yahoo.com/the-harsh-reality-for-investors-eyeing-tech-stocks-in-2023-morning-brief-111854804.html, Description: Curious on how to buy battered tech stocks? Here''s a quick tip. More on that and what else to watch in business on Monday., \n",
            "\n"
          ]
        }
      ],
      "source": [
        "# Conduct query with retrieval of sources\n",
        "query = \"Give me the best tech stock to invest in and tell me why\"\n",
        "response, source_information = handle_user_query(query, collection)\n",
        "\n",
        "print(f\"Response: {response}\")\n",
        "print(f\"\\\\nSource Information: \\\\n{source_information}\")"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}