stocks_fetch_and_pol.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "stocks_fetch_and_pol.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "# Sourcing the most recent posts in the r/stocks subReddit using PRAW, and performing sentiment analysis using textblob"
      ],
      "metadata": {
        "id": "EzBZXJayvsJe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GPczUnju8o3q",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5df6213d-f897-4eab-ddde-2363d3f33581"
      },
      "source": [
        "pip install praw"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting praw\n",
            "  Downloading praw-7.5.0-py3-none-any.whl (176 kB)\n",
            "\u001b[K     |████████████████████████████████| 176 kB 5.0 MB/s \n",
            "\u001b[?25hCollecting prawcore<3,>=2.1\n",
            "  Downloading prawcore-2.3.0-py3-none-any.whl (16 kB)\n",
            "Collecting update-checker>=0.18\n",
            "  Downloading update_checker-0.18.0-py3-none-any.whl (7.0 kB)\n",
            "Collecting websocket-client>=0.54.0\n",
            "  Downloading websocket_client-1.2.3-py3-none-any.whl (53 kB)\n",
            "\u001b[K     |████████████████████████████████| 53 kB 1.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests<3.0,>=2.6.0 in /usr/local/lib/python3.7/dist-packages (from prawcore<3,>=2.1->praw) (2.23.0)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0,>=2.6.0->prawcore<3,>=2.1->praw) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0,>=2.6.0->prawcore<3,>=2.1->praw) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0,>=2.6.0->prawcore<3,>=2.1->praw) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0,>=2.6.0->prawcore<3,>=2.1->praw) (2021.10.8)\n",
            "Installing collected packages: websocket-client, update-checker, prawcore, praw\n",
            "Successfully installed praw-7.5.0 prawcore-2.3.0 update-checker-0.18.0 websocket-client-1.2.3\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-DCTlP3tBuhE"
      },
      "source": [
        "import praw\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import datetime\n",
        "from google.colab import files"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# authentication information"
      ],
      "metadata": {
        "id": "qnlMa1sTwEY9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6liBNAW0Cc_C"
      },
      "source": [
        "reddit = praw.Reddit(\n",
        "    client_id=\"Jef0auDOdnMZ9vD6zlcWRg\",\n",
        "    client_secret=\"XqXCuNc590taBMgauwFmMBpDqvZp0A\",\n",
        "    user_agent=\"poppaearedditapilink\",\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# creating an array for data, and a for loop for submissions in the sub, and a dummy for each post\n",
        "# for the information I am interested in, which includes the post body text, title and time of posting\n",
        "# then appending the posts to the array."
      ],
      "metadata": {
        "id": "Unzitj69wKJD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VtUTpIFMLPIM",
        "outputId": "1a051d80-8a20-4e6e-bd73-aa453c9d7490"
      },
      "source": [
        "data = []\n",
        "for submission in reddit.subreddit('stocks').new(limit=1000):\n",
        "    # print(\"Title:\", submission.title)\n",
        "    # print(\"Post:\", submission.selftext)\n",
        "    # print(\"Posted at:\", submission.created)\n",
        "    dummy = {\"Title\": submission.title, \"Post\": submission.selftext, \"DateTime\": submission.created}\n",
        "    data.append(dummy)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "It appears that you are using PRAW in an asynchronous environment.\n",
            "It is strongly recommended to use Async PRAW: https://asyncpraw.readthedocs.io.\n",
            "See https://praw.readthedocs.io/en/latest/getting_started/multiple_instances.html#discord-bots-and-asynchronous-environments for more info.\n",
            "\n",
            "It appears that you are using PRAW in an asynchronous environment.\n",
            "It is strongly recommended to use Async PRAW: https://asyncpraw.readthedocs.io.\n",
            "See https://praw.readthedocs.io/en/latest/getting_started/multiple_instances.html#discord-bots-and-asynchronous-environments for more info.\n",
            "\n",
            "It appears that you are using PRAW in an asynchronous environment.\n",
            "It is strongly recommended to use Async PRAW: https://asyncpraw.readthedocs.io.\n",
            "See https://praw.readthedocs.io/en/latest/getting_started/multiple_instances.html#discord-bots-and-asynchronous-environments for more info.\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# putting the data array into a dataframe"
      ],
      "metadata": {
        "id": "ISH6_IHB2fEA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.DataFrame(data, columns=['Title', 'Post', 'DateTime'])\n",
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "Ay78JfNXg4eJ",
        "outputId": "0f966356-c74e-4521-84a6-8011fcd423da"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-5a14a939-b059-4350-921e-b468ae6d10a3\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Title</th>\n",
              "      <th>Post</th>\n",
              "      <th>DateTime</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Tip request: which market to buy a specific st...</td>\n",
              "      <td>Hello, I am not sure which market to choose so...</td>\n",
              "      <td>1.642250e+09</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>My first (concise) stock analysis, looking for...</td>\n",
              "      <td>\\n\\nI've owned some individual stocks in the ...</td>\n",
              "      <td>1.642249e+09</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Insider trade alerts?</td>\n",
              "      <td>I'm looking for a way to receive automatic, re...</td>\n",
              "      <td>1.642247e+09</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Alphawave IP $AWE.L undervalued semiconductor ...</td>\n",
              "      <td>I can’t find any reddit posts or any content a...</td>\n",
              "      <td>1.642233e+09</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Loan from stocks</td>\n",
              "      <td>Seeking guidance, I have over 50k in stocks. W...</td>\n",
              "      <td>1.642233e+09</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>979</th>\n",
              "      <td>Question on losses between years and then gains</td>\n",
              "      <td>Say in this example an Investor had 35,000 in ...</td>\n",
              "      <td>1.640706e+09</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>980</th>\n",
              "      <td>I am looking for a high risk and high reward s...</td>\n",
              "      <td>I am almost done with my end of year stock sho...</td>\n",
              "      <td>1.640705e+09</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>981</th>\n",
              "      <td>Small cap ETF?</td>\n",
              "      <td>I was thinking of investing in a few small cap...</td>\n",
              "      <td>1.640704e+09</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>982</th>\n",
              "      <td>$PRDS - an asymmetrical risk/reward profile</td>\n",
              "      <td>Background:\\n\\n$PRDS is a biotech company deve...</td>\n",
              "      <td>1.640703e+09</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>983</th>\n",
              "      <td>(12/28) Tuesday's Pre-Market Stock Movers &amp; News</td>\n",
              "      <td>#Good morning traders and investors of the r/s...</td>\n",
              "      <td>1.640697e+09</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>984 rows × 3 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-5a14a939-b059-4350-921e-b468ae6d10a3')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-5a14a939-b059-4350-921e-b468ae6d10a3 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-5a14a939-b059-4350-921e-b468ae6d10a3');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                                 Title  ...      DateTime\n",
              "0    Tip request: which market to buy a specific st...  ...  1.642250e+09\n",
              "1    My first (concise) stock analysis, looking for...  ...  1.642249e+09\n",
              "2                                Insider trade alerts?  ...  1.642247e+09\n",
              "3    Alphawave IP $AWE.L undervalued semiconductor ...  ...  1.642233e+09\n",
              "4                                     Loan from stocks  ...  1.642233e+09\n",
              "..                                                 ...  ...           ...\n",
              "979    Question on losses between years and then gains  ...  1.640706e+09\n",
              "980  I am looking for a high risk and high reward s...  ...  1.640705e+09\n",
              "981                                     Small cap ETF?  ...  1.640704e+09\n",
              "982        $PRDS - an asymmetrical risk/reward profile  ...  1.640703e+09\n",
              "983   (12/28) Tuesday's Pre-Market Stock Movers & News  ...  1.640697e+09\n",
              "\n",
              "[984 rows x 3 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# converting unix timestamps into readable time (GMT to be exact)"
      ],
      "metadata": {
        "id": "62EGrk9f2jzg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "datetimes = pd.to_datetime(df[\"DateTime\"], unit = \"s\")\n",
        "df[\"DateTime\"] = datetimes"
      ],
      "metadata": {
        "id": "P2JbVzka4vp9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "iEjCl_pT6U9w",
        "outputId": "c9da27fc-46db-43a7-cad7-64e2e5d99b44"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-53531869-22fb-4f09-bff2-ea15f3f4b775\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Title</th>\n",
              "      <th>Post</th>\n",
              "      <th>DateTime</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Tip request: which market to buy a specific st...</td>\n",
              "      <td>Hello, I am not sure which market to choose so...</td>\n",
              "      <td>2022-01-15 12:30:33</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>My first (concise) stock analysis, looking for...</td>\n",
              "      <td>\\n\\nI've owned some individual stocks in the ...</td>\n",
              "      <td>2022-01-15 12:09:07</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Insider trade alerts?</td>\n",
              "      <td>I'm looking for a way to receive automatic, re...</td>\n",
              "      <td>2022-01-15 11:45:02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Alphawave IP $AWE.L undervalued semiconductor ...</td>\n",
              "      <td>I can’t find any reddit posts or any content a...</td>\n",
              "      <td>2022-01-15 07:53:53</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Loan from stocks</td>\n",
              "      <td>Seeking guidance, I have over 50k in stocks. W...</td>\n",
              "      <td>2022-01-15 07:49:09</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>979</th>\n",
              "      <td>Question on losses between years and then gains</td>\n",
              "      <td>Say in this example an Investor had 35,000 in ...</td>\n",
              "      <td>2021-12-28 15:38:55</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>980</th>\n",
              "      <td>I am looking for a high risk and high reward s...</td>\n",
              "      <td>I am almost done with my end of year stock sho...</td>\n",
              "      <td>2021-12-28 15:27:08</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>981</th>\n",
              "      <td>Small cap ETF?</td>\n",
              "      <td>I was thinking of investing in a few small cap...</td>\n",
              "      <td>2021-12-28 14:58:51</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>982</th>\n",
              "      <td>$PRDS - an asymmetrical risk/reward profile</td>\n",
              "      <td>Background:\\n\\n$PRDS is a biotech company deve...</td>\n",
              "      <td>2021-12-28 14:49:16</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>983</th>\n",
              "      <td>(12/28) Tuesday's Pre-Market Stock Movers &amp; News</td>\n",
              "      <td>#Good morning traders and investors of the r/s...</td>\n",
              "      <td>2021-12-28 13:02:39</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>984 rows × 3 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-53531869-22fb-4f09-bff2-ea15f3f4b775')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-53531869-22fb-4f09-bff2-ea15f3f4b775 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-53531869-22fb-4f09-bff2-ea15f3f4b775');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                                 Title  ...            DateTime\n",
              "0    Tip request: which market to buy a specific st...  ... 2022-01-15 12:30:33\n",
              "1    My first (concise) stock analysis, looking for...  ... 2022-01-15 12:09:07\n",
              "2                                Insider trade alerts?  ... 2022-01-15 11:45:02\n",
              "3    Alphawave IP $AWE.L undervalued semiconductor ...  ... 2022-01-15 07:53:53\n",
              "4                                     Loan from stocks  ... 2022-01-15 07:49:09\n",
              "..                                                 ...  ...                 ...\n",
              "979    Question on losses between years and then gains  ... 2021-12-28 15:38:55\n",
              "980  I am looking for a high risk and high reward s...  ... 2021-12-28 15:27:08\n",
              "981                                     Small cap ETF?  ... 2021-12-28 14:58:51\n",
              "982        $PRDS - an asymmetrical risk/reward profile  ... 2021-12-28 14:49:16\n",
              "983   (12/28) Tuesday's Pre-Market Stock Movers & News  ... 2021-12-28 13:02:39\n",
              "\n",
              "[984 rows x 3 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df[\"text\"] = df[\"Title\"] + df[\"Post\"]\n",
        "df"
      ],
      "metadata": {
        "id": "D4B1R5OdS3QY",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 641
        },
        "outputId": "f6d2ac27-eb2a-4232-8d2d-05f04982b957"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-97ca0cd1-a2c8-467c-9b46-fa8d0ea26065\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Title</th>\n",
              "      <th>Post</th>\n",
              "      <th>DateTime</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Tip request: which market to buy a specific st...</td>\n",
              "      <td>Hello, I am not sure which market to choose so...</td>\n",
              "      <td>2022-01-15 12:30:33</td>\n",
              "      <td>Tip request: which market to buy a specific st...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>My first (concise) stock analysis, looking for...</td>\n",
              "      <td>\\n\\nI've owned some individual stocks in the ...</td>\n",
              "      <td>2022-01-15 12:09:07</td>\n",
              "      <td>My first (concise) stock analysis, looking for...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Insider trade alerts?</td>\n",
              "      <td>I'm looking for a way to receive automatic, re...</td>\n",
              "      <td>2022-01-15 11:45:02</td>\n",
              "      <td>Insider trade alerts?I'm looking for a way to ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Alphawave IP $AWE.L undervalued semiconductor ...</td>\n",
              "      <td>I can’t find any reddit posts or any content a...</td>\n",
              "      <td>2022-01-15 07:53:53</td>\n",
              "      <td>Alphawave IP $AWE.L undervalued semiconductor ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Loan from stocks</td>\n",
              "      <td>Seeking guidance, I have over 50k in stocks. W...</td>\n",
              "      <td>2022-01-15 07:49:09</td>\n",
              "      <td>Loan from stocksSeeking guidance, I have over ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>979</th>\n",
              "      <td>Question on losses between years and then gains</td>\n",
              "      <td>Say in this example an Investor had 35,000 in ...</td>\n",
              "      <td>2021-12-28 15:38:55</td>\n",
              "      <td>Question on losses between years and then gain...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>980</th>\n",
              "      <td>I am looking for a high risk and high reward s...</td>\n",
              "      <td>I am almost done with my end of year stock sho...</td>\n",
              "      <td>2021-12-28 15:27:08</td>\n",
              "      <td>I am looking for a high risk and high reward s...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>981</th>\n",
              "      <td>Small cap ETF?</td>\n",
              "      <td>I was thinking of investing in a few small cap...</td>\n",
              "      <td>2021-12-28 14:58:51</td>\n",
              "      <td>Small cap ETF?I was thinking of investing in a...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>982</th>\n",
              "      <td>$PRDS - an asymmetrical risk/reward profile</td>\n",
              "      <td>Background:\\n\\n$PRDS is a biotech company deve...</td>\n",
              "      <td>2021-12-28 14:49:16</td>\n",
              "      <td>$PRDS - an asymmetrical risk/reward profileBac...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>983</th>\n",
              "      <td>(12/28) Tuesday's Pre-Market Stock Movers &amp; News</td>\n",
              "      <td>#Good morning traders and investors of the r/s...</td>\n",
              "      <td>2021-12-28 13:02:39</td>\n",
              "      <td>(12/28) Tuesday's Pre-Market Stock Movers &amp; Ne...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>984 rows × 4 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-97ca0cd1-a2c8-467c-9b46-fa8d0ea26065')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-97ca0cd1-a2c8-467c-9b46-fa8d0ea26065 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-97ca0cd1-a2c8-467c-9b46-fa8d0ea26065');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                                 Title  ...                                               text\n",
              "0    Tip request: which market to buy a specific st...  ...  Tip request: which market to buy a specific st...\n",
              "1    My first (concise) stock analysis, looking for...  ...  My first (concise) stock analysis, looking for...\n",
              "2                                Insider trade alerts?  ...  Insider trade alerts?I'm looking for a way to ...\n",
              "3    Alphawave IP $AWE.L undervalued semiconductor ...  ...  Alphawave IP $AWE.L undervalued semiconductor ...\n",
              "4                                     Loan from stocks  ...  Loan from stocksSeeking guidance, I have over ...\n",
              "..                                                 ...  ...                                                ...\n",
              "979    Question on losses between years and then gains  ...  Question on losses between years and then gain...\n",
              "980  I am looking for a high risk and high reward s...  ...  I am looking for a high risk and high reward s...\n",
              "981                                     Small cap ETF?  ...  Small cap ETF?I was thinking of investing in a...\n",
              "982        $PRDS - an asymmetrical risk/reward profile  ...  $PRDS - an asymmetrical risk/reward profileBac...\n",
              "983   (12/28) Tuesday's Pre-Market Stock Movers & News  ...  (12/28) Tuesday's Pre-Market Stock Movers & Ne...\n",
              "\n",
              "[984 rows x 4 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "del df[\"Title\"]"
      ],
      "metadata": {
        "id": "KfGAgcCpEYZT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "del df[\"Post\"]"
      ],
      "metadata": {
        "id": "3HA0iW3pEbLr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "3boYrzPW_eTO",
        "outputId": "7c5e1bd8-2b9c-4e41-95d3-07b0f91925f2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-89305ef2-dcb1-4bb6-8816-cc0c5ff1c832\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>DateTime</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2022-01-15 12:30:33</td>\n",
              "      <td>Tip request: which market to buy a specific st...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2022-01-15 12:09:07</td>\n",
              "      <td>My first (concise) stock analysis, looking for...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2022-01-15 11:45:02</td>\n",
              "      <td>Insider trade alerts?I'm looking for a way to ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2022-01-15 07:53:53</td>\n",
              "      <td>Alphawave IP $AWE.L undervalued semiconductor ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2022-01-15 07:49:09</td>\n",
              "      <td>Loan from stocksSeeking guidance, I have over ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>979</th>\n",
              "      <td>2021-12-28 15:38:55</td>\n",
              "      <td>Question on losses between years and then gain...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>980</th>\n",
              "      <td>2021-12-28 15:27:08</td>\n",
              "      <td>I am looking for a high risk and high reward s...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>981</th>\n",
              "      <td>2021-12-28 14:58:51</td>\n",
              "      <td>Small cap ETF?I was thinking of investing in a...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>982</th>\n",
              "      <td>2021-12-28 14:49:16</td>\n",
              "      <td>$PRDS - an asymmetrical risk/reward profileBac...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>983</th>\n",
              "      <td>2021-12-28 13:02:39</td>\n",
              "      <td>(12/28) Tuesday's Pre-Market Stock Movers &amp; Ne...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>984 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-89305ef2-dcb1-4bb6-8816-cc0c5ff1c832')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-89305ef2-dcb1-4bb6-8816-cc0c5ff1c832 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-89305ef2-dcb1-4bb6-8816-cc0c5ff1c832');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "               DateTime                                               text\n",
              "0   2022-01-15 12:30:33  Tip request: which market to buy a specific st...\n",
              "1   2022-01-15 12:09:07  My first (concise) stock analysis, looking for...\n",
              "2   2022-01-15 11:45:02  Insider trade alerts?I'm looking for a way to ...\n",
              "3   2022-01-15 07:53:53  Alphawave IP $AWE.L undervalued semiconductor ...\n",
              "4   2022-01-15 07:49:09  Loan from stocksSeeking guidance, I have over ...\n",
              "..                  ...                                                ...\n",
              "979 2021-12-28 15:38:55  Question on losses between years and then gain...\n",
              "980 2021-12-28 15:27:08  I am looking for a high risk and high reward s...\n",
              "981 2021-12-28 14:58:51  Small cap ETF?I was thinking of investing in a...\n",
              "982 2021-12-28 14:49:16  $PRDS - an asymmetrical risk/reward profileBac...\n",
              "983 2021-12-28 13:02:39  (12/28) Tuesday's Pre-Market Stock Movers & Ne...\n",
              "\n",
              "[984 rows x 2 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#importing the textblob package for sentiment analysis\n",
        "from textblob import TextBlob\n",
        "\n",
        "# some text data pre-processing from TextBlob:\n",
        "def preprocess(text):\n",
        "    text = text.str.replace(\"(<br/>)\", \"\")\n",
        "    text = text.str.replace('(<a).*(>).*(</a>)', '')\n",
        "    text = text.str.replace('(&amp)', '')\n",
        "    text = text.str.replace('(&gt)', '')\n",
        "    text = text.str.replace('(&lt)', '')\n",
        "    text = text.str.replace('(\\xa0)', ' ')  \n",
        "    return text\n",
        "df['text'] = preprocess(df['text'])\n",
        "\n",
        "df['polarity'] = df['text'].map(lambda text: TextBlob(text).sentiment.polarity)\n",
        "df['review_len'] = df['text'].astype(str).apply(len)\n",
        "df['word_count'] = df['text'].apply(lambda x: len(str(x).split()))"
      ],
      "metadata": {
        "id": "oeHWhGc0_5mT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "R0_QOCLo_6Dj",
        "outputId": "1bddc27d-e77c-4b48-d1b8-d58af88884a1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-8fab06c4-27ba-44c9-bfac-e8e01998ef2e\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>DateTime</th>\n",
              "      <th>text</th>\n",
              "      <th>polarity</th>\n",
              "      <th>review_len</th>\n",
              "      <th>word_count</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2022-01-15 12:30:33</td>\n",
              "      <td>Tip request: which market to buy a specific st...</td>\n",
              "      <td>0.027778</td>\n",
              "      <td>566</td>\n",
              "      <td>101</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2022-01-15 12:09:07</td>\n",
              "      <td>My first (concise) stock analysis, looking for...</td>\n",
              "      <td>0.128492</td>\n",
              "      <td>6485</td>\n",
              "      <td>1121</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2022-01-15 11:45:02</td>\n",
              "      <td>Insider trade alerts?I'm looking for a way to ...</td>\n",
              "      <td>0.350970</td>\n",
              "      <td>470</td>\n",
              "      <td>86</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2022-01-15 07:53:53</td>\n",
              "      <td>Alphawave IP $AWE.L undervalued semiconductor ...</td>\n",
              "      <td>-0.100617</td>\n",
              "      <td>540</td>\n",
              "      <td>86</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2022-01-15 07:49:09</td>\n",
              "      <td>Loan from stocksSeeking guidance, I have over ...</td>\n",
              "      <td>-0.161111</td>\n",
              "      <td>243</td>\n",
              "      <td>43</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>979</th>\n",
              "      <td>2021-12-28 15:38:55</td>\n",
              "      <td>Question on losses between years and then gain...</td>\n",
              "      <td>0.233333</td>\n",
              "      <td>440</td>\n",
              "      <td>78</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>980</th>\n",
              "      <td>2021-12-28 15:27:08</td>\n",
              "      <td>I am looking for a high risk and high reward s...</td>\n",
              "      <td>0.293333</td>\n",
              "      <td>395</td>\n",
              "      <td>75</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>981</th>\n",
              "      <td>2021-12-28 14:58:51</td>\n",
              "      <td>Small cap ETF?I was thinking of investing in a...</td>\n",
              "      <td>-0.086508</td>\n",
              "      <td>235</td>\n",
              "      <td>48</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>982</th>\n",
              "      <td>2021-12-28 14:49:16</td>\n",
              "      <td>$PRDS - an asymmetrical risk/reward profileBac...</td>\n",
              "      <td>0.083468</td>\n",
              "      <td>2184</td>\n",
              "      <td>366</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>983</th>\n",
              "      <td>2021-12-28 13:02:39</td>\n",
              "      <td>(12/28) Tuesday's Pre-Market Stock Movers &amp; Ne...</td>\n",
              "      <td>0.150016</td>\n",
              "      <td>9227</td>\n",
              "      <td>1170</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>984 rows × 5 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8fab06c4-27ba-44c9-bfac-e8e01998ef2e')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-8fab06c4-27ba-44c9-bfac-e8e01998ef2e button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-8fab06c4-27ba-44c9-bfac-e8e01998ef2e');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "               DateTime  ... word_count\n",
              "0   2022-01-15 12:30:33  ...        101\n",
              "1   2022-01-15 12:09:07  ...       1121\n",
              "2   2022-01-15 11:45:02  ...         86\n",
              "3   2022-01-15 07:53:53  ...         86\n",
              "4   2022-01-15 07:49:09  ...         43\n",
              "..                  ...  ...        ...\n",
              "979 2021-12-28 15:38:55  ...         78\n",
              "980 2021-12-28 15:27:08  ...         75\n",
              "981 2021-12-28 14:58:51  ...         48\n",
              "982 2021-12-28 14:49:16  ...        366\n",
              "983 2021-12-28 13:02:39  ...       1170\n",
              "\n",
              "[984 rows x 5 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df = df.drop([\"review_len\", \"word_count\", \"text\"], axis = 1)\n",
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "V6Lpc7HfAGKt",
        "outputId": "105bd027-c73b-4dbd-9a67-23c10a52e1b7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-42715fc5-861d-409e-8aaa-ecb67df2bf58\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>DateTime</th>\n",
              "      <th>polarity</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2022-01-15 12:30:33</td>\n",
              "      <td>0.027778</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2022-01-15 12:09:07</td>\n",
              "      <td>0.128492</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2022-01-15 11:45:02</td>\n",
              "      <td>0.350970</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2022-01-15 07:53:53</td>\n",
              "      <td>-0.100617</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2022-01-15 07:49:09</td>\n",
              "      <td>-0.161111</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>979</th>\n",
              "      <td>2021-12-28 15:38:55</td>\n",
              "      <td>0.233333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>980</th>\n",
              "      <td>2021-12-28 15:27:08</td>\n",
              "      <td>0.293333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>981</th>\n",
              "      <td>2021-12-28 14:58:51</td>\n",
              "      <td>-0.086508</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>982</th>\n",
              "      <td>2021-12-28 14:49:16</td>\n",
              "      <td>0.083468</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>983</th>\n",
              "      <td>2021-12-28 13:02:39</td>\n",
              "      <td>0.150016</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>984 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-42715fc5-861d-409e-8aaa-ecb67df2bf58')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-42715fc5-861d-409e-8aaa-ecb67df2bf58 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-42715fc5-861d-409e-8aaa-ecb67df2bf58');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "               DateTime  polarity\n",
              "0   2022-01-15 12:30:33  0.027778\n",
              "1   2022-01-15 12:09:07  0.128492\n",
              "2   2022-01-15 11:45:02  0.350970\n",
              "3   2022-01-15 07:53:53 -0.100617\n",
              "4   2022-01-15 07:49:09 -0.161111\n",
              "..                  ...       ...\n",
              "979 2021-12-28 15:38:55  0.233333\n",
              "980 2021-12-28 15:27:08  0.293333\n",
              "981 2021-12-28 14:58:51 -0.086508\n",
              "982 2021-12-28 14:49:16  0.083468\n",
              "983 2021-12-28 13:02:39  0.150016\n",
              "\n",
              "[984 rows x 2 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.to_csv(\"r_stocks_polarity_08.csv\")"
      ],
      "metadata": {
        "id": "Z4aRtfHhALcu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "files.download(\"r_stocks_polarity_08.csv\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "cs413psyE51l",
        "outputId": "a07915e1-fd82-49c1-9f1d-3021b96ecf9a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "download(\"download_1db6055c-3931-4b12-8abd-ae1dbcee69ca\", \"r_stocks_polarity_08.csv\", 41253)"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {}
        }
      ]
    }
  ]
}