Monish_K_Data_Scraping_2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Importing libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### web-scraping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "src = requests.get(\"https://www.worldometers.info/coronavirus/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [200]>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# successful hit of the website\n",
    "src"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# converting to bs4 object\n",
    "soup = BeautifulSoup(src.text, 'lxml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# finding the table\n",
    "table = soup.find('table', id = 'main_table_countries_today')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# headers for the table\n",
    "headers = [ele.text for ele in table.find_all('th')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# selecting only the countries list\n",
    "countries_list = table.find_all('tr')[9:-8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extracting the rows and storing it in a list\n",
    "countries_df = pd.DataFrame(columns=headers)\n",
    "for country in countries_list:\n",
    "    countries_df.loc[len(countries_df)] = [ele.text for ele in country.find_all('td')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# keeping only the required columns\n",
    "countries_df = countries_df[['Country,Other', 'TotalCases', 'TotalDeaths', 'TotalTests', 'Population']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# renaming columns\n",
    "countries_df.rename(columns={'Country,Other' : 'country',\n",
    "                             'TotalCases' : 'cases',\n",
    "                             'TotalDeaths' : 'deaths',\n",
    "                             'TotalTests' : 'tests',\n",
    "                             'Population' : 'population'}, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>There are two columns that have null values in it. Deaths and population</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# removing the blank spaces\n",
    "countries_df = countries_df[countries_df[\"deaths\"] != \" \"]\n",
    "countries_df = countries_df[countries_df[\"population\"] != \" \"]\n",
    "countries_df = countries_df[countries_df[\"tests\"] != \"\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# converting object to int for the required columns\n",
    "for col in countries_df.columns.tolist()[1:]:\n",
    "    \n",
    "    countries_df[col] = countries_df[col].apply(lambda x: int(x.replace(\",\", \"\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "countries_df.reset_index(inplace=True, drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>country</th>\n",
       "      <th>cases</th>\n",
       "      <th>deaths</th>\n",
       "      <th>tests</th>\n",
       "      <th>population</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>USA</td>\n",
       "      <td>104996288</td>\n",
       "      <td>1142704</td>\n",
       "      <td>1163294129</td>\n",
       "      <td>334805269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>India</td>\n",
       "      <td>44685132</td>\n",
       "      <td>530761</td>\n",
       "      <td>917870730</td>\n",
       "      <td>1406631776</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>France</td>\n",
       "      <td>39590190</td>\n",
       "      <td>164759</td>\n",
       "      <td>271490188</td>\n",
       "      <td>65584518</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  country      cases   deaths       tests  population\n",
       "0     USA  104996288  1142704  1163294129   334805269\n",
       "1   India   44685132   530761   917870730  1406631776\n",
       "2  France   39590190   164759   271490188    65584518"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "countries_df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculating the tests_per_case\n",
    "countries_df[\"tests_per_case\"] = countries_df[\"tests\"] / countries_df[\"cases\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# rounding tests_per_case to four digits\n",
    "countries_df[\"tests_per_case\"] = countries_df[\"tests_per_case\"].apply(lambda x: round(x, 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# setting country as the index\n",
    "countries_df.set_index('country', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cases</th>\n",
       "      <th>deaths</th>\n",
       "      <th>tests</th>\n",
       "      <th>population</th>\n",
       "      <th>tests_per_case</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>country</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>China</th>\n",
       "      <td>503302</td>\n",
       "      <td>5272</td>\n",
       "      <td>160000000</td>\n",
       "      <td>1448471400</td>\n",
       "      <td>317.9006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UAE</th>\n",
       "      <td>1051241</td>\n",
       "      <td>2348</td>\n",
       "      <td>199077922</td>\n",
       "      <td>10081785</td>\n",
       "      <td>189.3742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Turks and Caicos</th>\n",
       "      <td>6551</td>\n",
       "      <td>38</td>\n",
       "      <td>611527</td>\n",
       "      <td>39741</td>\n",
       "      <td>93.3486</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Oman</th>\n",
       "      <td>399449</td>\n",
       "      <td>4628</td>\n",
       "      <td>25000000</td>\n",
       "      <td>5323993</td>\n",
       "      <td>62.5862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Bermuda</th>\n",
       "      <td>18791</td>\n",
       "      <td>159</td>\n",
       "      <td>1026742</td>\n",
       "      <td>61939</td>\n",
       "      <td>54.6401</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Saudi Arabia</th>\n",
       "      <td>829041</td>\n",
       "      <td>9606</td>\n",
       "      <td>45121063</td>\n",
       "      <td>35844909</td>\n",
       "      <td>54.4256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Rwanda</th>\n",
       "      <td>133170</td>\n",
       "      <td>1468</td>\n",
       "      <td>5959042</td>\n",
       "      <td>13600464</td>\n",
       "      <td>44.7476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Denmark</th>\n",
       "      <td>3174758</td>\n",
       "      <td>8237</td>\n",
       "      <td>129185732</td>\n",
       "      <td>5834950</td>\n",
       "      <td>40.6915</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Bhutan</th>\n",
       "      <td>62611</td>\n",
       "      <td>21</td>\n",
       "      <td>2303734</td>\n",
       "      <td>787941</td>\n",
       "      <td>36.7944</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Austria</th>\n",
       "      <td>5871234</td>\n",
       "      <td>21825</td>\n",
       "      <td>211273524</td>\n",
       "      <td>9066710</td>\n",
       "      <td>35.9845</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Spain</th>\n",
       "      <td>13755956</td>\n",
       "      <td>119186</td>\n",
       "      <td>471036328</td>\n",
       "      <td>46719142</td>\n",
       "      <td>34.2424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sierra Leone</th>\n",
       "      <td>7760</td>\n",
       "      <td>126</td>\n",
       "      <td>259958</td>\n",
       "      <td>8306436</td>\n",
       "      <td>33.4997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Gabon</th>\n",
       "      <td>48981</td>\n",
       "      <td>306</td>\n",
       "      <td>1621909</td>\n",
       "      <td>2331533</td>\n",
       "      <td>33.1130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Tonga</th>\n",
       "      <td>16801</td>\n",
       "      <td>13</td>\n",
       "      <td>535009</td>\n",
       "      <td>107749</td>\n",
       "      <td>31.8439</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Yemen</th>\n",
       "      <td>11945</td>\n",
       "      <td>2159</td>\n",
       "      <td>329592</td>\n",
       "      <td>31154867</td>\n",
       "      <td>27.5925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Hong Kong</th>\n",
       "      <td>2882553</td>\n",
       "      <td>13445</td>\n",
       "      <td>76127725</td>\n",
       "      <td>7604299</td>\n",
       "      <td>26.4098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Gibraltar</th>\n",
       "      <td>20414</td>\n",
       "      <td>111</td>\n",
       "      <td>534283</td>\n",
       "      <td>33704</td>\n",
       "      <td>26.1724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Niger</th>\n",
       "      <td>9931</td>\n",
       "      <td>312</td>\n",
       "      <td>254538</td>\n",
       "      <td>26083660</td>\n",
       "      <td>25.6307</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Chad</th>\n",
       "      <td>7675</td>\n",
       "      <td>194</td>\n",
       "      <td>191341</td>\n",
       "      <td>17413580</td>\n",
       "      <td>24.9304</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mali</th>\n",
       "      <td>32918</td>\n",
       "      <td>743</td>\n",
       "      <td>793891</td>\n",
       "      <td>21473764</td>\n",
       "      <td>24.1172</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     cases  deaths      tests  population  tests_per_case\n",
       "country                                                                  \n",
       "China               503302    5272  160000000  1448471400        317.9006\n",
       "UAE                1051241    2348  199077922    10081785        189.3742\n",
       "Turks and Caicos      6551      38     611527       39741         93.3486\n",
       "Oman                399449    4628   25000000     5323993         62.5862\n",
       "Bermuda              18791     159    1026742       61939         54.6401\n",
       "Saudi Arabia        829041    9606   45121063    35844909         54.4256\n",
       "Rwanda              133170    1468    5959042    13600464         44.7476\n",
       "Denmark            3174758    8237  129185732     5834950         40.6915\n",
       "Bhutan               62611      21    2303734      787941         36.7944\n",
       "Austria            5871234   21825  211273524     9066710         35.9845\n",
       "Spain             13755956  119186  471036328    46719142         34.2424\n",
       "Sierra Leone          7760     126     259958     8306436         33.4997\n",
       "Gabon                48981     306    1621909     2331533         33.1130\n",
       "Tonga                16801      13     535009      107749         31.8439\n",
       "Yemen                11945    2159     329592    31154867         27.5925\n",
       "Hong Kong          2882553   13445   76127725     7604299         26.4098\n",
       "Gibraltar            20414     111     534283       33704         26.1724\n",
       "Niger                 9931     312     254538    26083660         25.6307\n",
       "Chad                  7675     194     191341    17413580         24.9304\n",
       "Mali                 32918     743     793891    21473764         24.1172"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# displaying top 20 rows\n",
    "countries_df.sort_values(by=[\"tests_per_case\"], ascending=False).head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>total tests per case is a feature that helps in identifying the number of tests required to identify a positive case. For example if 100 tests are done to identify 2 cases then the tests per case would be 50. Generally the assumption would be to suggest that a low value means that the infection is wide spread across the country and a high value means that the spread is not high. But this would not be a very good indicator in understanding the spread of the infection because countries like China, despite facing worst covid outbreaks in recent months, does not comply to the interpretation.</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}