wikipedia_scraper.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f057325a-3bd8-4c7c-9565-a03683e860ca",
   "metadata": {},
   "source": [
    "# Scraping Webpage Data - Wikipedia"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6a136ea2-f663-40df-9799-c15cfcef92a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import Libraries\n",
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "482e054f-acb7-402e-8a5a-64076308cc09",
   "metadata": {},
   "outputs": [],
   "source": [
    "# User Agent - use: whatismyuseragent.com\n",
    "USER_AGENT = \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5.1 Safari/605.1.15\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2b742fc7-ab4f-46a4-9f3f-c0f6559087f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Headers for the request\n",
    "HEADERS = ({'User-Agent': USER_AGENT, 'Accept-Language': \"en-US, en;q=0.5\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "412fe5e6-f94a-478f-a35a-48571a88f577",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Wikipedia URL\n",
    "URL = \"https://en.wikipedia.org/wiki/List_of_largest_companies_in_the_United_States_by_revenue\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c5c03f86-4645-4af5-a6fe-6a556b482cf8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# HTTP request\n",
    "page = requests.get(URL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "426f956e-d683-4c82-a72b-7a63e7f7a3f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "200"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "page.status_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2fb9ebbb-17cb-43e4-a62e-17f650c4fce3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Emsamble the soup with the page information\n",
    "soup = BeautifulSoup(page.text, \"html\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "af4a2c75-7695-40f9-b6d6-709a5a69f079",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Uncomment next line to see results\n",
    "# print(soup)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ec9a1d5-045d-4e9e-8218-fffebe478957",
   "metadata": {},
   "source": [
    "#### table element with the attributes classes to easily identify it.\n",
    "- `<table class=\"wikitable sortable jquery-tablesorter\">`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c2052146-e97f-488e-af6d-4da1a0b3ad26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<table class=\"box-More_citations_needed plainlinks metadata ambox ambox-content ambox-Refimprove\" role=\"presentation\"><tbody><tr><td class=\"mbox-image\"><div class=\"mbox-image-div\"><span typeof=\"mw:File\"><a class=\"mw-file-description\" href=\"/wiki/File:Question_book-new.svg\"><img alt=\"\" class=\"mw-file-element\" data-file-height=\"399\" data-file-width=\"512\" decoding=\"async\" height=\"39\" src=\"//upload.wikimedia.org/wikipedia/en/thumb/9/99/Question_book-new.svg/50px-Question_book-new.svg.png\" srcset=\"//upload.wikimedia.org/wikipedia/en/thumb/9/99/Question_book-new.svg/75px-Question_book-new.svg.png 1.5x, //upload.wikimedia.org/wikipedia/en/thumb/9/99/Question_book-new.svg/100px-Question_book-new.svg.png 2x\" width=\"50\"/></a></span></div></td><td class=\"mbox-text\"><div class=\"mbox-text-span\">This article <b>needs additional citations for <a href=\"/wiki/Wikipedia:Verifiability\" title=\"Wikipedia:Verifiability\">verification</a></b>.<span class=\"hide-when-compact\"> Please help <a href=\"/wiki/Special:EditPage/List_of_largest_companies_in_the_United_States_by_revenue\" title=\"Special:EditPage/List of largest companies in the United States by revenue\">improve this article</a> by <a href=\"/wiki/Help:Referencing_for_beginners\" title=\"Help:Referencing for beginners\">adding citations to reliable sources</a>. Unsourced material may be challenged and removed.<br/><small><span class=\"plainlinks\"><i>Find sources:</i> <a class=\"external text\" href=\"https://www.google.com/search?as_eq=wikipedia&amp;q=%22List+of+largest+companies+in+the+United+States+by+revenue%22\" rel=\"nofollow\">\"List of largest companies in the United States by revenue\"</a> – <a class=\"external text\" href=\"https://www.google.com/search?tbm=nws&amp;q=%22List+of+largest+companies+in+the+United+States+by+revenue%22+-wikipedia&amp;tbs=ar:1\" rel=\"nofollow\">news</a> <b>·</b> <a class=\"external text\" href=\"https://www.google.com/search?&amp;q=%22List+of+largest+companies+in+the+United+States+by+revenue%22&amp;tbs=bkt:s&amp;tbm=bks\" rel=\"nofollow\">newspapers</a> <b>·</b> <a class=\"external text\" href=\"https://www.google.com/search?tbs=bks:1&amp;q=%22List+of+largest+companies+in+the+United+States+by+revenue%22+-wikipedia\" rel=\"nofollow\">books</a> <b>·</b> <a class=\"external text\" href=\"https://scholar.google.com/scholar?q=%22List+of+largest+companies+in+the+United+States+by+revenue%22\" rel=\"nofollow\">scholar</a> <b>·</b> <a class=\"external text\" href=\"https://www.jstor.org/action/doBasicSearch?Query=%22List+of+largest+companies+in+the+United+States+by+revenue%22&amp;acc=on&amp;wc=on\" rel=\"nofollow\">JSTOR</a></span></small></span> <span class=\"date-container\"><i>(<span class=\"date\">June 2020</span>)</i></span><span class=\"hide-when-compact\"><i> (<small><a href=\"/wiki/Help:Maintenance_template_removal\" title=\"Help:Maintenance template removal\">Learn how and when to remove this template message</a></small>)</i></span></div></td></tr></tbody></table>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.find('table')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2d2f19f3-febf-47ca-8fe8-2bfb97dd31cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# From all tables select the first one - There are three tables on this page.\n",
    "table = soup.find_all('table')[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5e8824ce-40ed-47a9-93b1-2eafae8a45cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# uncomment next line to see results\n",
    "# table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "c2aac967-b17a-4fed-9ead-a2e4865028d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Find all `th` elements in the table = `headers` for the table\n",
    "th_tags = table.find_all('th')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5504361d-11b4-4bf3-b2ea-e4b960bb656c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Rank',\n",
       " 'Name',\n",
       " 'Industry',\n",
       " 'Revenue (USD millions)',\n",
       " 'Revenue growth',\n",
       " 'Employees',\n",
       " 'Headquarters']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Collect table headers to then add them as df columns\n",
    "th_text = []\n",
    "for th_tag in th_tags:\n",
    "    th_text.append(th_tag.text.strip())\n",
    "\n",
    "th_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "dd2498d4-74b2-4574-b657-9b78accb410f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Find all table rows. Discard the first row because `headers` are the first one.\n",
    "rows = table.find_all('tr')[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "899f6173-dc52-4d25-a1bb-e9aaba5f6578",
   "metadata": {},
   "outputs": [],
   "source": [
    "d = {'Rank': [], 'Name': [], 'Industry': [], 'Revenue (USD millions)': [], 'Revenue growth': [], 'Employees': [], 'Headquarters': []}\n",
    "\n",
    "# Collect the information to fill up the dictionary\n",
    "for row in rows:\n",
    "    elems = row.find_all('td')\n",
    "    d['Rank'].append(elems[0].text.strip())\n",
    "    d['Name'].append(elems[1].text.strip())\n",
    "    d['Industry'].append(elems[2].text.strip())\n",
    "    d['Revenue (USD millions)'].append(elems[3].text.strip())\n",
    "    d['Revenue growth'].append(elems[4].text.strip())\n",
    "    d['Employees'].append(elems[5].text.strip())\n",
    "    d['Headquarters'].append(elems[6].text.strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "9bbceaf8-667e-4aaa-9d06-2eb1b88b45fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the `DataFrame` from the dictionary\n",
    "companies_df = pd.DataFrame.from_dict(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "916e7af9-4ed7-45d1-89df-dac5a3a2f469",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Rank</th>\n",
       "      <th>Name</th>\n",
       "      <th>Industry</th>\n",
       "      <th>Revenue (USD millions)</th>\n",
       "      <th>Revenue growth</th>\n",
       "      <th>Employees</th>\n",
       "      <th>Headquarters</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Walmart</td>\n",
       "      <td>Retail</td>\n",
       "      <td>611,289</td>\n",
       "      <td>6.7%</td>\n",
       "      <td>2,100,000</td>\n",
       "      <td>Bentonville, Arkansas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Amazon</td>\n",
       "      <td>Retail and Cloud Computing</td>\n",
       "      <td>513,983</td>\n",
       "      <td>9.4%</td>\n",
       "      <td>1,540,000</td>\n",
       "      <td>Seattle, Washington</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Exxon Mobil</td>\n",
       "      <td>Petroleum industry</td>\n",
       "      <td>413,680</td>\n",
       "      <td>44.8%</td>\n",
       "      <td>62,000</td>\n",
       "      <td>Spring, Texas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Apple</td>\n",
       "      <td>Electronics industry</td>\n",
       "      <td>394,328</td>\n",
       "      <td>7.8%</td>\n",
       "      <td>164,000</td>\n",
       "      <td>Cupertino, California</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>UnitedHealth Group</td>\n",
       "      <td>Healthcare</td>\n",
       "      <td>324,162</td>\n",
       "      <td>12.7%</td>\n",
       "      <td>400,000</td>\n",
       "      <td>Minnetonka, Minnesota</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>96</td>\n",
       "      <td>Best Buy</td>\n",
       "      <td>Retail</td>\n",
       "      <td>46,298</td>\n",
       "      <td>10.6%</td>\n",
       "      <td>71,100</td>\n",
       "      <td>Richfield, Minnesota</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>97</td>\n",
       "      <td>Bristol-Myers Squibb</td>\n",
       "      <td>Pharmaceutical industry</td>\n",
       "      <td>46,159</td>\n",
       "      <td>0.5%</td>\n",
       "      <td>34,300</td>\n",
       "      <td>New York City, New York</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>98</td>\n",
       "      <td>United Airlines</td>\n",
       "      <td>Airline</td>\n",
       "      <td>44,955</td>\n",
       "      <td>82.5%</td>\n",
       "      <td>92,795</td>\n",
       "      <td>Chicago, Illinois</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>99</td>\n",
       "      <td>Thermo Fisher Scientific</td>\n",
       "      <td>Laboratory instruments</td>\n",
       "      <td>44,915</td>\n",
       "      <td>14.5%</td>\n",
       "      <td>130,000</td>\n",
       "      <td>Waltham, Massachusetts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>100</td>\n",
       "      <td>Qualcomm</td>\n",
       "      <td>Technology</td>\n",
       "      <td>44,200</td>\n",
       "      <td>31.7%</td>\n",
       "      <td>51,000</td>\n",
       "      <td>San Diego, California</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Rank                      Name                    Industry  \\\n",
       "0     1                   Walmart                      Retail   \n",
       "1     2                    Amazon  Retail and Cloud Computing   \n",
       "2     3               Exxon Mobil          Petroleum industry   \n",
       "3     4                     Apple        Electronics industry   \n",
       "4     5        UnitedHealth Group                  Healthcare   \n",
       "..  ...                       ...                         ...   \n",
       "95   96                  Best Buy                      Retail   \n",
       "96   97      Bristol-Myers Squibb     Pharmaceutical industry   \n",
       "97   98           United Airlines                     Airline   \n",
       "98   99  Thermo Fisher Scientific      Laboratory instruments   \n",
       "99  100                  Qualcomm                  Technology   \n",
       "\n",
       "   Revenue (USD millions) Revenue growth  Employees             Headquarters  \n",
       "0                 611,289           6.7%  2,100,000    Bentonville, Arkansas  \n",
       "1                 513,983           9.4%  1,540,000      Seattle, Washington  \n",
       "2                 413,680          44.8%     62,000            Spring, Texas  \n",
       "3                 394,328           7.8%    164,000    Cupertino, California  \n",
       "4                 324,162          12.7%    400,000    Minnetonka, Minnesota  \n",
       "..                    ...            ...        ...                      ...  \n",
       "95                 46,298          10.6%     71,100     Richfield, Minnesota  \n",
       "96                 46,159           0.5%     34,300  New York City, New York  \n",
       "97                 44,955          82.5%     92,795        Chicago, Illinois  \n",
       "98                 44,915          14.5%    130,000   Waltham, Massachusetts  \n",
       "99                 44,200          31.7%     51,000    San Diego, California  \n",
       "\n",
       "[100 rows x 7 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "companies_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "fe631770-b138-4cdd-b0d8-0f613a0623b5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 100 entries, 0 to 99\n",
      "Data columns (total 7 columns):\n",
      " #   Column                  Non-Null Count  Dtype \n",
      "---  ------                  --------------  ----- \n",
      " 0   Rank                    100 non-null    object\n",
      " 1   Name                    100 non-null    object\n",
      " 2   Industry                100 non-null    object\n",
      " 3   Revenue (USD millions)  100 non-null    object\n",
      " 4   Revenue growth          100 non-null    object\n",
      " 5   Employees               100 non-null    object\n",
      " 6   Headquarters            100 non-null    object\n",
      "dtypes: object(7)\n",
      "memory usage: 5.6+ KB\n"
     ]
    }
   ],
   "source": [
    "# Check if there are missing values\n",
    "companies_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "1cb98dd5-04f8-4cf2-b747-5f83f3a89f56",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save `df` as a `csv` file\n",
    "companies_df.to_csv('largest_companies_in_the_us_by_revenue.csv', header=True, index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}