tradingview_stocks_webscrape.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports\n",
    "\n",
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "200"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Obtain URL and get request:\n",
    "# USA Large Cap Stocks\n",
    "\n",
    "url = \"https://www.tradingview.com/markets/stocks-usa/market-movers-large-cap/\"\n",
    "\n",
    "response = requests.get(url)\n",
    "\n",
    "# Check Status code:\n",
    "\n",
    "response.status_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup = BeautifulSoup(response.content, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = soup.find('tbody', {'class': 'tv-data-table__tbody'}).find_all('tr')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Company Name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Stock Name\n",
    "stock_name = [result.find('span', {'class': 'tv-screener__description'}).get_text().strip() for result in results]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ticker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'MSFT'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results[1].find('td').find('a').get_text()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ticker\n",
    "ticker = [result.find('td').find('a').get_text() for result in results]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stock Price"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'168.64'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results[0].find_all('td')[1].get_text()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Price\n",
    "price = [result.find_all('td')[1].get_text() for result in results]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Change %"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Change %\n",
    "change_pct = [result.find_all('td')[2].get_text() for result in results]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Change (Dollar Amounts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Change In Dollar Amount\n",
    "change_amt = [result.find_all('td')[3].get_text() for result in results]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Technical Rating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rating\n",
    "rating = [result.find_all('td')[4].get_text() for result in results]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Volume"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Volume\n",
    "volume = [result.find_all('td')[5].get_text() for result in results]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Volume*Price"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Volume times price\n",
    "volume_price = [result.find_all('td')[6].get_text() for result in results]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Market Cap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Market Cap\n",
    "mcap = [result.find_all('td')[7].get_text() for result in results]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Dataframe Of Raw Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Top 100 Stocks Tradingview\n",
    "\n",
    "tview_stocks_df = pd.DataFrame({'Stock': stock_name, 'Ticker': ticker,\n",
    "                                'Price': price, 'Change %': change_pct,\n",
    "                                'Change Amt.': change_amt, 'Rating': rating,\n",
    "                                'Volume': volume, 'Volume*Price': volume_price,\n",
    "                                'Market Cap': mcap})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Stock</th>\n",
       "      <th>Ticker</th>\n",
       "      <th>Price</th>\n",
       "      <th>Change %</th>\n",
       "      <th>Change Amt.</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Volume</th>\n",
       "      <th>Volume*Price</th>\n",
       "      <th>Market Cap</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Apple Inc</td>\n",
       "      <td>AAPL</td>\n",
       "      <td>168.64</td>\n",
       "      <td>-2.02%</td>\n",
       "      <td>-3.48</td>\n",
       "      <td>Sell</td>\n",
       "      <td>98.671M</td>\n",
       "      <td>16639815380.48</td>\n",
       "      <td>2752.111B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Microsoft Corp.</td>\n",
       "      <td>MSFT</td>\n",
       "      <td>295.04</td>\n",
       "      <td>-2.43%</td>\n",
       "      <td>-7.34</td>\n",
       "      <td>Strong Sell</td>\n",
       "      <td>39.175M</td>\n",
       "      <td>11558269005.44</td>\n",
       "      <td>2211.876B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Alphabet Inc (Google) Class C</td>\n",
       "      <td>GOOG</td>\n",
       "      <td>2682.60</td>\n",
       "      <td>-3.23%</td>\n",
       "      <td>-89.45</td>\n",
       "      <td>Sell</td>\n",
       "      <td>1.940M</td>\n",
       "      <td>5205408248.40</td>\n",
       "      <td>1774.411B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Alphabet Inc (Google) Class A</td>\n",
       "      <td>GOOGL</td>\n",
       "      <td>2685.65</td>\n",
       "      <td>-3.13%</td>\n",
       "      <td>-86.75</td>\n",
       "      <td>Sell</td>\n",
       "      <td>1.998M</td>\n",
       "      <td>5365351285.25</td>\n",
       "      <td>1774.411B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Amazon.com, Inc.</td>\n",
       "      <td>AMZN</td>\n",
       "      <td>3065.87</td>\n",
       "      <td>-3.59%</td>\n",
       "      <td>-114.20</td>\n",
       "      <td>Sell</td>\n",
       "      <td>3.855M</td>\n",
       "      <td>11817506286.32</td>\n",
       "      <td>1554.849B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Tesla, Inc.</td>\n",
       "      <td>TSLA</td>\n",
       "      <td>860.00</td>\n",
       "      <td>-4.93%</td>\n",
       "      <td>-44.55</td>\n",
       "      <td>Strong Sell</td>\n",
       "      <td>26.548M</td>\n",
       "      <td>22830879240.00</td>\n",
       "      <td>888.817B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Berkshire Hathaway Inc. New</td>\n",
       "      <td>BRK.B</td>\n",
       "      <td>319.14</td>\n",
       "      <td>-0.22%</td>\n",
       "      <td>-0.71</td>\n",
       "      <td>Buy</td>\n",
       "      <td>7.497M</td>\n",
       "      <td>2392707151.26</td>\n",
       "      <td>713.171B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Berkshire Hathaway Inc.</td>\n",
       "      <td>BRK.A</td>\n",
       "      <td>479370.00</td>\n",
       "      <td>-0.15%</td>\n",
       "      <td>-735.00</td>\n",
       "      <td>Buy</td>\n",
       "      <td>1.625K</td>\n",
       "      <td>778976250.00</td>\n",
       "      <td>713.171B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Taiwan Semiconductor Manufacturing Company Ltd.</td>\n",
       "      <td>TSM</td>\n",
       "      <td>121.01</td>\n",
       "      <td>-2.97%</td>\n",
       "      <td>-3.70</td>\n",
       "      <td>Sell</td>\n",
       "      <td>9.846M</td>\n",
       "      <td>1191423437.61</td>\n",
       "      <td>604.833B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>NVIDIA Corporation</td>\n",
       "      <td>NVDA</td>\n",
       "      <td>239.49</td>\n",
       "      <td>-7.26%</td>\n",
       "      <td>-18.75</td>\n",
       "      <td>Sell</td>\n",
       "      <td>60.546M</td>\n",
       "      <td>14500171598.58</td>\n",
       "      <td>598.725B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Meta Platforms, Inc.</td>\n",
       "      <td>FB</td>\n",
       "      <td>219.55</td>\n",
       "      <td>-3.74%</td>\n",
       "      <td>-8.52</td>\n",
       "      <td>Sell</td>\n",
       "      <td>46.156M</td>\n",
       "      <td>10133656501.30</td>\n",
       "      <td>597.602B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Visa Inc.</td>\n",
       "      <td>V</td>\n",
       "      <td>224.69</td>\n",
       "      <td>-0.40%</td>\n",
       "      <td>-0.90</td>\n",
       "      <td>Buy</td>\n",
       "      <td>7.535M</td>\n",
       "      <td>1693085436.14</td>\n",
       "      <td>471.134B</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              Stock Ticker      Price  \\\n",
       "0                                         Apple Inc   AAPL     168.64   \n",
       "1                                   Microsoft Corp.   MSFT     295.04   \n",
       "2                     Alphabet Inc (Google) Class C   GOOG    2682.60   \n",
       "3                     Alphabet Inc (Google) Class A  GOOGL    2685.65   \n",
       "4                                  Amazon.com, Inc.   AMZN    3065.87   \n",
       "5                                       Tesla, Inc.   TSLA     860.00   \n",
       "6                       Berkshire Hathaway Inc. New  BRK.B     319.14   \n",
       "7                           Berkshire Hathaway Inc.  BRK.A  479370.00   \n",
       "8   Taiwan Semiconductor Manufacturing Company Ltd.    TSM     121.01   \n",
       "9                                NVIDIA Corporation   NVDA     239.49   \n",
       "10                             Meta Platforms, Inc.     FB     219.55   \n",
       "11                                        Visa Inc.      V     224.69   \n",
       "\n",
       "   Change % Change Amt.       Rating   Volume    Volume*Price Market Cap  \n",
       "0    -2.02%       -3.48         Sell  98.671M  16639815380.48  2752.111B  \n",
       "1    -2.43%       -7.34  Strong Sell  39.175M  11558269005.44  2211.876B  \n",
       "2    -3.23%      -89.45         Sell   1.940M   5205408248.40  1774.411B  \n",
       "3    -3.13%      -86.75         Sell   1.998M   5365351285.25  1774.411B  \n",
       "4    -3.59%     -114.20         Sell   3.855M  11817506286.32  1554.849B  \n",
       "5    -4.93%      -44.55  Strong Sell  26.548M  22830879240.00   888.817B  \n",
       "6    -0.22%       -0.71          Buy   7.497M   2392707151.26   713.171B  \n",
       "7    -0.15%     -735.00          Buy   1.625K    778976250.00   713.171B  \n",
       "8    -2.97%       -3.70         Sell   9.846M   1191423437.61   604.833B  \n",
       "9    -7.26%      -18.75         Sell  60.546M  14500171598.58   598.725B  \n",
       "10   -3.74%       -8.52         Sell  46.156M  10133656501.30   597.602B  \n",
       "11   -0.40%       -0.90          Buy   7.535M   1693085436.14   471.134B  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tview_stocks_df.head(12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Stock</th>\n",
       "      <th>Ticker</th>\n",
       "      <th>Price</th>\n",
       "      <th>Change %</th>\n",
       "      <th>Change Amt.</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Volume</th>\n",
       "      <th>Volume*Price</th>\n",
       "      <th>Market Cap</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>Applied Materials, Inc.</td>\n",
       "      <td>AMAT</td>\n",
       "      <td>132.49</td>\n",
       "      <td>-5.21%</td>\n",
       "      <td>-7.28</td>\n",
       "      <td>Strong Sell</td>\n",
       "      <td>9.473M</td>\n",
       "      <td>1255091019.00</td>\n",
       "      <td>117.650B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>JD.com, Inc.</td>\n",
       "      <td>JD</td>\n",
       "      <td>73.98</td>\n",
       "      <td>-3.17%</td>\n",
       "      <td>-2.42</td>\n",
       "      <td>Sell</td>\n",
       "      <td>6.673M</td>\n",
       "      <td>493694728.92</td>\n",
       "      <td>117.480B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>BlackRock, Inc.</td>\n",
       "      <td>BLK</td>\n",
       "      <td>772.49</td>\n",
       "      <td>-0.73%</td>\n",
       "      <td>-5.70</td>\n",
       "      <td>Strong Sell</td>\n",
       "      <td>1.272M</td>\n",
       "      <td>982497586.42</td>\n",
       "      <td>117.175B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>ServiceNow, Inc.</td>\n",
       "      <td>NOW</td>\n",
       "      <td>583.72</td>\n",
       "      <td>-4.38%</td>\n",
       "      <td>-26.74</td>\n",
       "      <td>Sell</td>\n",
       "      <td>2.140M</td>\n",
       "      <td>1249157881.40</td>\n",
       "      <td>116.744B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>Diageo plc</td>\n",
       "      <td>DEO</td>\n",
       "      <td>201.75</td>\n",
       "      <td>-1.19%</td>\n",
       "      <td>-2.43</td>\n",
       "      <td>Sell</td>\n",
       "      <td>339.053K</td>\n",
       "      <td>68403942.75</td>\n",
       "      <td>116.690B</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      Stock Ticker   Price Change % Change Amt.       Rating  \\\n",
       "95  Applied Materials, Inc.   AMAT  132.49   -5.21%       -7.28  Strong Sell   \n",
       "96             JD.com, Inc.     JD   73.98   -3.17%       -2.42         Sell   \n",
       "97          BlackRock, Inc.    BLK  772.49   -0.73%       -5.70  Strong Sell   \n",
       "98         ServiceNow, Inc.    NOW  583.72   -4.38%      -26.74         Sell   \n",
       "99               Diageo plc    DEO  201.75   -1.19%       -2.43         Sell   \n",
       "\n",
       "      Volume   Volume*Price Market Cap  \n",
       "95    9.473M  1255091019.00   117.650B  \n",
       "96    6.673M   493694728.92   117.480B  \n",
       "97    1.272M   982497586.42   117.175B  \n",
       "98    2.140M  1249157881.40   116.744B  \n",
       "99  339.053K    68403942.75   116.690B  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tview_stocks_df.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Write To Excel File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write to excel file:\n",
    "from datetime import date\n",
    "\n",
    "tview_stocks_df.to_excel(\"trading_view_stocks_\" + str(date.today()) + \".xlsx\", index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Write as .csv file:\n",
    "tview_stocks_df.to_csv(\"trading_view_stocks_\" + str(date.today()) + \".csv\", index = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}