notebooks/Scrape wiki tables with pandas and python.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use only pandas\n",
    "\n",
    "source: https://gist.github.com/aculich/b34868c098d94d614515\n",
    "\n",
    "\n",
    "Installation requirements\n",
    "\n",
    "```\n",
    "pip install pandas\n",
    "pip install lxml\n",
    "pip install html5lib\n",
    "pip install BeautifulSoup4```\n",
    "\n",
    "or \n",
    "\n",
    "```conda install html5lib ```\n",
    "\n",
    "or \n",
    "\n",
    "```easy_install html5lib```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read Wiki Tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracted 1 wikitables\n"
     ]
    }
   ],
   "source": [
    "# extract tables from wikipedia\n",
    "from pandas.io.html import read_html\n",
    "page = 'https://en.wikipedia.org/wiki/List_of_Asian_countries_by_area'\n",
    "\n",
    "wikitables = read_html(page,  attrs={\"class\":\"wikitable\"})\n",
    "\n",
    "print (\"Extracted {num} wikitables\".format(num=len(wikitables)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Rank</td>\n",
       "      <td>Country</td>\n",
       "      <td>Area (km²)</td>\n",
       "      <td>Notes</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Russia*</td>\n",
       "      <td>13000000</td>\n",
       "      <td>17,125,191 km² including European Russia[1]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>China</td>\n",
       "      <td>9596961</td>\n",
       "      <td>excludes Hong Kong, Macau, Taiwan and disputed...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>India[2]</td>\n",
       "      <td>3287263</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Kazakhstan*</td>\n",
       "      <td>2544900</td>\n",
       "      <td>2,724,900 km² including European part</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>Saudi Arabia</td>\n",
       "      <td>2149690</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>Iran</td>\n",
       "      <td>1648195</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>Mongolia</td>\n",
       "      <td>1564110</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>Indonesia*</td>\n",
       "      <td>1472639</td>\n",
       "      <td>1,904,569 km² including Oceanian part</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>Pakistan</td>\n",
       "      <td>881913</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>10</td>\n",
       "      <td>Turkey*</td>\n",
       "      <td>759592</td>\n",
       "      <td>783,356 km² including East Thrace</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>11</td>\n",
       "      <td>Myanmar</td>\n",
       "      <td>676578</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>12</td>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>652230</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>13</td>\n",
       "      <td>Yemen</td>\n",
       "      <td>527968</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>14</td>\n",
       "      <td>Thailand</td>\n",
       "      <td>513120</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>15</td>\n",
       "      <td>Turkmenistan</td>\n",
       "      <td>488100</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>16</td>\n",
       "      <td>Uzbekistan</td>\n",
       "      <td>447400</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>17</td>\n",
       "      <td>Iraq</td>\n",
       "      <td>438317</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>18</td>\n",
       "      <td>Japan</td>\n",
       "      <td>377930</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>19</td>\n",
       "      <td>Vietnam</td>\n",
       "      <td>331212</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>20</td>\n",
       "      <td>Malaysia</td>\n",
       "      <td>330803</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>21</td>\n",
       "      <td>Oman</td>\n",
       "      <td>309500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>22</td>\n",
       "      <td>Philippines</td>\n",
       "      <td>300000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>23</td>\n",
       "      <td>Laos</td>\n",
       "      <td>236800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>24</td>\n",
       "      <td>Kyrgyzstan</td>\n",
       "      <td>199951</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>25</td>\n",
       "      <td>Syria</td>\n",
       "      <td>185180</td>\n",
       "      <td>Includes the parts of the Golan Heights</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>26</td>\n",
       "      <td>Cambodia</td>\n",
       "      <td>181035</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>27</td>\n",
       "      <td>Bangladesh</td>\n",
       "      <td>147570</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>28</td>\n",
       "      <td>Nepal</td>\n",
       "      <td>147181</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>29</td>\n",
       "      <td>Tajikistan</td>\n",
       "      <td>143100</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>30</td>\n",
       "      <td>North Korea</td>\n",
       "      <td>120538</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>31</td>\n",
       "      <td>South Korea</td>\n",
       "      <td>100210</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>32</td>\n",
       "      <td>Jordan</td>\n",
       "      <td>89342</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>33</td>\n",
       "      <td>Azerbaijan*</td>\n",
       "      <td>86000</td>\n",
       "      <td>Located in the Caucasus, between Europe and Asia</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>34</td>\n",
       "      <td>United Arab Emirates</td>\n",
       "      <td>83600</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>35</td>\n",
       "      <td>Georgia*</td>\n",
       "      <td>69000</td>\n",
       "      <td>Located in the Caucasus, between Europe and Asia</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>36</td>\n",
       "      <td>Sri Lanka</td>\n",
       "      <td>65610</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>37</td>\n",
       "      <td>Bhutan</td>\n",
       "      <td>38394</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>38</td>\n",
       "      <td>Taiwan</td>\n",
       "      <td>36193</td>\n",
       "      <td>partially recognized state/not a UN member</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>39</td>\n",
       "      <td>Armenia</td>\n",
       "      <td>29843</td>\n",
       "      <td>Located in the Caucasus, between Europe and Asia</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>40</td>\n",
       "      <td>Israel</td>\n",
       "      <td>22072</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>41</td>\n",
       "      <td>Kuwait</td>\n",
       "      <td>17818</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>42</td>\n",
       "      <td>Timor-Leste</td>\n",
       "      <td>14874</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>43</td>\n",
       "      <td>Qatar</td>\n",
       "      <td>11586</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>44</td>\n",
       "      <td>Lebanon</td>\n",
       "      <td>10452</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>45</td>\n",
       "      <td>Cyprus</td>\n",
       "      <td>9251</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>46</td>\n",
       "      <td>Palestine</td>\n",
       "      <td>6220</td>\n",
       "      <td>partially recognized state/non-member observer...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>47</td>\n",
       "      <td>Brunei</td>\n",
       "      <td>5765</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>48</td>\n",
       "      <td>Bahrain</td>\n",
       "      <td>760</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>49</td>\n",
       "      <td>Singapore</td>\n",
       "      <td>697</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>50</td>\n",
       "      <td>Maldives</td>\n",
       "      <td>300</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Total</td>\n",
       "      <td>44528251</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       0                     1           2  \\\n",
       "0   Rank               Country  Area (km²)   \n",
       "1      1               Russia*    13000000   \n",
       "2      2                 China     9596961   \n",
       "3      3              India[2]     3287263   \n",
       "4      4           Kazakhstan*     2544900   \n",
       "5      5          Saudi Arabia     2149690   \n",
       "6      6                  Iran     1648195   \n",
       "7      7              Mongolia     1564110   \n",
       "8      8            Indonesia*     1472639   \n",
       "9      9              Pakistan      881913   \n",
       "10    10               Turkey*      759592   \n",
       "11    11               Myanmar      676578   \n",
       "12    12           Afghanistan      652230   \n",
       "13    13                 Yemen      527968   \n",
       "14    14              Thailand      513120   \n",
       "15    15          Turkmenistan      488100   \n",
       "16    16            Uzbekistan      447400   \n",
       "17    17                  Iraq      438317   \n",
       "18    18                 Japan      377930   \n",
       "19    19               Vietnam      331212   \n",
       "20    20              Malaysia      330803   \n",
       "21    21                  Oman      309500   \n",
       "22    22           Philippines      300000   \n",
       "23    23                  Laos      236800   \n",
       "24    24            Kyrgyzstan      199951   \n",
       "25    25                 Syria      185180   \n",
       "26    26              Cambodia      181035   \n",
       "27    27            Bangladesh      147570   \n",
       "28    28                 Nepal      147181   \n",
       "29    29            Tajikistan      143100   \n",
       "30    30           North Korea      120538   \n",
       "31    31           South Korea      100210   \n",
       "32    32                Jordan       89342   \n",
       "33    33           Azerbaijan*       86000   \n",
       "34    34  United Arab Emirates       83600   \n",
       "35    35              Georgia*       69000   \n",
       "36    36             Sri Lanka       65610   \n",
       "37    37                Bhutan       38394   \n",
       "38    38                Taiwan       36193   \n",
       "39    39               Armenia       29843   \n",
       "40    40                Israel       22072   \n",
       "41    41                Kuwait       17818   \n",
       "42    42           Timor-Leste       14874   \n",
       "43    43                 Qatar       11586   \n",
       "44    44               Lebanon       10452   \n",
       "45    45                Cyprus        9251   \n",
       "46    46             Palestine        6220   \n",
       "47    47                Brunei        5765   \n",
       "48    48               Bahrain         760   \n",
       "49    49             Singapore         697   \n",
       "50    50              Maldives         300   \n",
       "51   NaN                 Total    44528251   \n",
       "\n",
       "                                                    3   4  \n",
       "0                                               Notes NaN  \n",
       "1         17,125,191 km² including European Russia[1] NaN  \n",
       "2   excludes Hong Kong, Macau, Taiwan and disputed... NaN  \n",
       "3                                                 NaN NaN  \n",
       "4               2,724,900 km² including European part NaN  \n",
       "5                                                 NaN NaN  \n",
       "6                                                 NaN NaN  \n",
       "7                                                 NaN NaN  \n",
       "8               1,904,569 km² including Oceanian part NaN  \n",
       "9                                                 NaN NaN  \n",
       "10                  783,356 km² including East Thrace NaN  \n",
       "11                                                NaN NaN  \n",
       "12                                                NaN NaN  \n",
       "13                                                NaN NaN  \n",
       "14                                                NaN NaN  \n",
       "15                                                NaN NaN  \n",
       "16                                                NaN NaN  \n",
       "17                                                NaN NaN  \n",
       "18                                                NaN NaN  \n",
       "19                                                NaN NaN  \n",
       "20                                                NaN NaN  \n",
       "21                                                NaN NaN  \n",
       "22                                                NaN NaN  \n",
       "23                                                NaN NaN  \n",
       "24                                                NaN NaN  \n",
       "25            Includes the parts of the Golan Heights NaN  \n",
       "26                                                NaN NaN  \n",
       "27                                                NaN NaN  \n",
       "28                                                NaN NaN  \n",
       "29                                                NaN NaN  \n",
       "30                                                NaN NaN  \n",
       "31                                                NaN NaN  \n",
       "32                                                NaN NaN  \n",
       "33   Located in the Caucasus, between Europe and Asia NaN  \n",
       "34                                                NaN NaN  \n",
       "35   Located in the Caucasus, between Europe and Asia NaN  \n",
       "36                                                NaN NaN  \n",
       "37                                                NaN NaN  \n",
       "38         partially recognized state/not a UN member NaN  \n",
       "39   Located in the Caucasus, between Europe and Asia NaN  \n",
       "40                                                NaN NaN  \n",
       "41                                                NaN NaN  \n",
       "42                                                NaN NaN  \n",
       "43                                                NaN NaN  \n",
       "44                                                NaN NaN  \n",
       "45                                                NaN NaN  \n",
       "46  partially recognized state/non-member observer... NaN  \n",
       "47                                                NaN NaN  \n",
       "48                                                NaN NaN  \n",
       "49                                                NaN NaN  \n",
       "50                                                NaN NaN  \n",
       "51                                                NaN NaN  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wikitables[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracted 7 wikitables\n"
     ]
    }
   ],
   "source": [
    "# extract several tables from wikipedia from a single page\n",
    "from pandas.io.html import read_html\n",
    "page = 'https://en.wikipedia.org/wiki/New_York_City'\n",
    "\n",
    "wikitables = read_html(page, index_col=0, attrs={\"class\":\"wikitable\"}, header=None)\n",
    "\n",
    "print (\"Extracted {num} wikitables\".format(num=len(wikitables)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th>New York City's five boroughsvte</th>\n",
       "      <th colspan=\"8\" halign=\"left\">New York City's five boroughsvte</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Jurisdiction</th>\n",
       "      <th>Jurisdiction</th>\n",
       "      <th>Population</th>\n",
       "      <th colspan=\"2\" halign=\"left\">Gross Domestic Product</th>\n",
       "      <th colspan=\"2\" halign=\"left\">Land area</th>\n",
       "      <th colspan=\"2\" halign=\"left\">Density</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Borough</th>\n",
       "      <th>County</th>\n",
       "      <th>Estimate (2018)[150]</th>\n",
       "      <th>billions(US$)[151]</th>\n",
       "      <th>per capita(US$)</th>\n",
       "      <th>square miles</th>\n",
       "      <th>squarekm</th>\n",
       "      <th>persons / sq. mi</th>\n",
       "      <th>persons /km2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>The Bronx</th>\n",
       "      <td>Bronx</td>\n",
       "      <td>1432132</td>\n",
       "      <td>42.695</td>\n",
       "      <td>29200</td>\n",
       "      <td>42.10</td>\n",
       "      <td>109.04</td>\n",
       "      <td>34653</td>\n",
       "      <td>13231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Brooklyn</th>\n",
       "      <td>Kings</td>\n",
       "      <td>2582830</td>\n",
       "      <td>91.559</td>\n",
       "      <td>34600</td>\n",
       "      <td>70.82</td>\n",
       "      <td>183.42</td>\n",
       "      <td>37137</td>\n",
       "      <td>14649</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Manhattan</th>\n",
       "      <td>New York</td>\n",
       "      <td>1628701</td>\n",
       "      <td>600.244</td>\n",
       "      <td>360900</td>\n",
       "      <td>22.83</td>\n",
       "      <td>59.13</td>\n",
       "      <td>72033</td>\n",
       "      <td>27826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Queens</th>\n",
       "      <td>Queens</td>\n",
       "      <td>2278906</td>\n",
       "      <td>93.310</td>\n",
       "      <td>39600</td>\n",
       "      <td>108.53</td>\n",
       "      <td>281.09</td>\n",
       "      <td>21460</td>\n",
       "      <td>8354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Staten Island</th>\n",
       "      <td>Richmond</td>\n",
       "      <td>476179</td>\n",
       "      <td>14.514</td>\n",
       "      <td>30300</td>\n",
       "      <td>58.37</td>\n",
       "      <td>151.18</td>\n",
       "      <td>8112</td>\n",
       "      <td>3132</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "New York City's five boroughsvte New York City's five boroughsvte  \\\n",
       "Jurisdiction                                         Jurisdiction   \n",
       "Borough                                                    County   \n",
       "The Bronx                                                   Bronx   \n",
       "Brooklyn                                                    Kings   \n",
       "Manhattan                                                New York   \n",
       "Queens                                                     Queens   \n",
       "Staten Island                                            Richmond   \n",
       "\n",
       "New York City's five boroughsvte                                              \\\n",
       "Jurisdiction                               Population Gross Domestic Product   \n",
       "Borough                          Estimate (2018)[150]     billions(US$)[151]   \n",
       "The Bronx                                     1432132                 42.695   \n",
       "Brooklyn                                      2582830                 91.559   \n",
       "Manhattan                                     1628701                600.244   \n",
       "Queens                                        2278906                 93.310   \n",
       "Staten Island                                  476179                 14.514   \n",
       "\n",
       "New York City's five boroughsvte                                        \\\n",
       "Jurisdiction                                        Land area            \n",
       "Borough                          per capita(US$) square miles squarekm   \n",
       "The Bronx                                  29200        42.10   109.04   \n",
       "Brooklyn                                   34600        70.82   183.42   \n",
       "Manhattan                                 360900        22.83    59.13   \n",
       "Queens                                     39600       108.53   281.09   \n",
       "Staten Island                              30300        58.37   151.18   \n",
       "\n",
       "New York City's five boroughsvte                                \n",
       "Jurisdiction                              Density               \n",
       "Borough                          persons / sq. mi persons /km2  \n",
       "The Bronx                                   34653        13231  \n",
       "Brooklyn                                    37137        14649  \n",
       "Manhattan                                   72033        27826  \n",
       "Queens                                      21460         8354  \n",
       "Staten Island                                8112         3132  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wikitables[0].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(17, 13)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wikitables[1].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th>vteClimate data for New York (Belvedere Castle, Central Park), 1981–2010 normals,[a] extremes 1869–present[b]</th>\n",
       "      <th colspan=\"13\" halign=\"left\">vteClimate data for New York (Belvedere Castle, Central Park), 1981–2010 normals,[a] extremes 1869–present[b]</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Month</th>\n",
       "      <th>Jan</th>\n",
       "      <th>Feb</th>\n",
       "      <th>Mar</th>\n",
       "      <th>Apr</th>\n",
       "      <th>May</th>\n",
       "      <th>Jun</th>\n",
       "      <th>Jul</th>\n",
       "      <th>Aug</th>\n",
       "      <th>Sep</th>\n",
       "      <th>Oct</th>\n",
       "      <th>Nov</th>\n",
       "      <th>Dec</th>\n",
       "      <th>Year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Record high °F (°C)</th>\n",
       "      <td>72(22)</td>\n",
       "      <td>78(26)</td>\n",
       "      <td>86(30)</td>\n",
       "      <td>96(36)</td>\n",
       "      <td>99(37)</td>\n",
       "      <td>101(38)</td>\n",
       "      <td>106(41)</td>\n",
       "      <td>104(40)</td>\n",
       "      <td>102(39)</td>\n",
       "      <td>94(34)</td>\n",
       "      <td>84(29)</td>\n",
       "      <td>75(24)</td>\n",
       "      <td>106(41)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mean maximum °F (°C)</th>\n",
       "      <td>59.6(15.3)</td>\n",
       "      <td>60.7(15.9)</td>\n",
       "      <td>71.5(21.9)</td>\n",
       "      <td>83.0(28.3)</td>\n",
       "      <td>88.0(31.1)</td>\n",
       "      <td>92.3(33.5)</td>\n",
       "      <td>95.4(35.2)</td>\n",
       "      <td>93.7(34.3)</td>\n",
       "      <td>88.5(31.4)</td>\n",
       "      <td>78.8(26.0)</td>\n",
       "      <td>71.3(21.8)</td>\n",
       "      <td>62.2(16.8)</td>\n",
       "      <td>97.0(36.1)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Average high °F (°C)</th>\n",
       "      <td>38.3(3.5)</td>\n",
       "      <td>41.6(5.3)</td>\n",
       "      <td>49.7(9.8)</td>\n",
       "      <td>61.2(16.2)</td>\n",
       "      <td>70.8(21.6)</td>\n",
       "      <td>79.3(26.3)</td>\n",
       "      <td>84.1(28.9)</td>\n",
       "      <td>82.6(28.1)</td>\n",
       "      <td>75.2(24.0)</td>\n",
       "      <td>63.8(17.7)</td>\n",
       "      <td>53.8(12.1)</td>\n",
       "      <td>43.0(6.1)</td>\n",
       "      <td>62.0(16.7)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Daily mean °F (°C)</th>\n",
       "      <td>32.6(0.3)</td>\n",
       "      <td>35.3(1.8)</td>\n",
       "      <td>42.5(5.8)</td>\n",
       "      <td>53.0(11.7)</td>\n",
       "      <td>62.4(16.9)</td>\n",
       "      <td>71.5(21.9)</td>\n",
       "      <td>76.5(24.7)</td>\n",
       "      <td>75.2(24.0)</td>\n",
       "      <td>68.0(20.0)</td>\n",
       "      <td>56.9(13.8)</td>\n",
       "      <td>47.7(8.7)</td>\n",
       "      <td>37.5(3.1)</td>\n",
       "      <td>55.0(12.8)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Average low °F (°C)</th>\n",
       "      <td>26.9(−2.8)</td>\n",
       "      <td>28.9(−1.7)</td>\n",
       "      <td>35.2(1.8)</td>\n",
       "      <td>44.8(7.1)</td>\n",
       "      <td>54.0(12.2)</td>\n",
       "      <td>63.6(17.6)</td>\n",
       "      <td>68.8(20.4)</td>\n",
       "      <td>67.8(19.9)</td>\n",
       "      <td>60.8(16.0)</td>\n",
       "      <td>50.0(10.0)</td>\n",
       "      <td>41.6(5.3)</td>\n",
       "      <td>32.0(0.0)</td>\n",
       "      <td>48.0(8.9)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mean minimum °F (°C)</th>\n",
       "      <td>9.2(−12.7)</td>\n",
       "      <td>12.8(−10.7)</td>\n",
       "      <td>18.5(−7.5)</td>\n",
       "      <td>32.3(0.2)</td>\n",
       "      <td>43.5(6.4)</td>\n",
       "      <td>52.9(11.6)</td>\n",
       "      <td>60.3(15.7)</td>\n",
       "      <td>58.8(14.9)</td>\n",
       "      <td>48.6(9.2)</td>\n",
       "      <td>38.0(3.3)</td>\n",
       "      <td>27.7(−2.4)</td>\n",
       "      <td>15.6(−9.1)</td>\n",
       "      <td>7.0(−13.9)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Record low °F (°C)</th>\n",
       "      <td>−6(−21)</td>\n",
       "      <td>−15(−26)</td>\n",
       "      <td>3(−16)</td>\n",
       "      <td>12(−11)</td>\n",
       "      <td>32(0)</td>\n",
       "      <td>44(7)</td>\n",
       "      <td>52(11)</td>\n",
       "      <td>50(10)</td>\n",
       "      <td>39(4)</td>\n",
       "      <td>28(−2)</td>\n",
       "      <td>5(−15)</td>\n",
       "      <td>−13(−25)</td>\n",
       "      <td>−15(−26)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Average precipitation inches (mm)</th>\n",
       "      <td>3.65(93)</td>\n",
       "      <td>3.09(78)</td>\n",
       "      <td>4.36(111)</td>\n",
       "      <td>4.50(114)</td>\n",
       "      <td>4.19(106)</td>\n",
       "      <td>4.41(112)</td>\n",
       "      <td>4.60(117)</td>\n",
       "      <td>4.44(113)</td>\n",
       "      <td>4.28(109)</td>\n",
       "      <td>4.40(112)</td>\n",
       "      <td>4.02(102)</td>\n",
       "      <td>4.00(102)</td>\n",
       "      <td>49.94(1,268)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Average snowfall inches (cm)</th>\n",
       "      <td>7.0(18)</td>\n",
       "      <td>9.2(23)</td>\n",
       "      <td>3.9(9.9)</td>\n",
       "      <td>0.6(1.5)</td>\n",
       "      <td>0(0)</td>\n",
       "      <td>0(0)</td>\n",
       "      <td>0(0)</td>\n",
       "      <td>0(0)</td>\n",
       "      <td>0(0)</td>\n",
       "      <td>0(0)</td>\n",
       "      <td>0.3(0.76)</td>\n",
       "      <td>4.8(12)</td>\n",
       "      <td>25.8(66)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Average precipitation days (≥ 0.01 in)</th>\n",
       "      <td>10.4</td>\n",
       "      <td>9.2</td>\n",
       "      <td>10.9</td>\n",
       "      <td>11.5</td>\n",
       "      <td>11.1</td>\n",
       "      <td>11.2</td>\n",
       "      <td>10.4</td>\n",
       "      <td>9.5</td>\n",
       "      <td>8.7</td>\n",
       "      <td>8.9</td>\n",
       "      <td>9.6</td>\n",
       "      <td>10.6</td>\n",
       "      <td>122.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Average snowy days (≥ 0.1 in)</th>\n",
       "      <td>4.0</td>\n",
       "      <td>2.8</td>\n",
       "      <td>1.8</td>\n",
       "      <td>0.3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2</td>\n",
       "      <td>2.3</td>\n",
       "      <td>11.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Average relative humidity (%)</th>\n",
       "      <td>61.5</td>\n",
       "      <td>60.2</td>\n",
       "      <td>58.5</td>\n",
       "      <td>55.3</td>\n",
       "      <td>62.7</td>\n",
       "      <td>65.2</td>\n",
       "      <td>64.2</td>\n",
       "      <td>66.0</td>\n",
       "      <td>67.8</td>\n",
       "      <td>65.6</td>\n",
       "      <td>64.6</td>\n",
       "      <td>64.1</td>\n",
       "      <td>63.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mean monthly sunshine hours</th>\n",
       "      <td>162.7</td>\n",
       "      <td>163.1</td>\n",
       "      <td>212.5</td>\n",
       "      <td>225.6</td>\n",
       "      <td>256.6</td>\n",
       "      <td>257.3</td>\n",
       "      <td>268.2</td>\n",
       "      <td>268.2</td>\n",
       "      <td>219.3</td>\n",
       "      <td>211.2</td>\n",
       "      <td>151.0</td>\n",
       "      <td>139.0</td>\n",
       "      <td>2534.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Percent possible sunshine</th>\n",
       "      <td>54</td>\n",
       "      <td>55</td>\n",
       "      <td>57</td>\n",
       "      <td>57</td>\n",
       "      <td>57</td>\n",
       "      <td>57</td>\n",
       "      <td>59</td>\n",
       "      <td>63</td>\n",
       "      <td>59</td>\n",
       "      <td>61</td>\n",
       "      <td>51</td>\n",
       "      <td>48</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Average ultraviolet index</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Source #1: NOAA (relative humidity and sun 1961–1990)[196][210][192][211]</th>\n",
       "      <td>Source #1: NOAA (relative humidity and sun 196...</td>\n",
       "      <td>Source #1: NOAA (relative humidity and sun 196...</td>\n",
       "      <td>Source #1: NOAA (relative humidity and sun 196...</td>\n",
       "      <td>Source #1: NOAA (relative humidity and sun 196...</td>\n",
       "      <td>Source #1: NOAA (relative humidity and sun 196...</td>\n",
       "      <td>Source #1: NOAA (relative humidity and sun 196...</td>\n",
       "      <td>Source #1: NOAA (relative humidity and sun 196...</td>\n",
       "      <td>Source #1: NOAA (relative humidity and sun 196...</td>\n",
       "      <td>Source #1: NOAA (relative humidity and sun 196...</td>\n",
       "      <td>Source #1: NOAA (relative humidity and sun 196...</td>\n",
       "      <td>Source #1: NOAA (relative humidity and sun 196...</td>\n",
       "      <td>Source #1: NOAA (relative humidity and sun 196...</td>\n",
       "      <td>Source #1: NOAA (relative humidity and sun 196...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Source #2: Weather Atlas[212] See Geography of New York City for additional climate information from the outer boroughs.</th>\n",
       "      <td>Source #2: Weather Atlas[212] See Geography of...</td>\n",
       "      <td>Source #2: Weather Atlas[212] See Geography of...</td>\n",
       "      <td>Source #2: Weather Atlas[212] See Geography of...</td>\n",
       "      <td>Source #2: Weather Atlas[212] See Geography of...</td>\n",
       "      <td>Source #2: Weather Atlas[212] See Geography of...</td>\n",
       "      <td>Source #2: Weather Atlas[212] See Geography of...</td>\n",
       "      <td>Source #2: Weather Atlas[212] See Geography of...</td>\n",
       "      <td>Source #2: Weather Atlas[212] See Geography of...</td>\n",
       "      <td>Source #2: Weather Atlas[212] See Geography of...</td>\n",
       "      <td>Source #2: Weather Atlas[212] See Geography of...</td>\n",
       "      <td>Source #2: Weather Atlas[212] See Geography of...</td>\n",
       "      <td>Source #2: Weather Atlas[212] See Geography of...</td>\n",
       "      <td>Source #2: Weather Atlas[212] See Geography of...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "vteClimate data for New York (Belvedere Castle, Central Park), 1981–2010 normals,[a] extremes 1869–present[b] vteClimate data for New York (Belvedere Castle, Central Park), 1981–2010 normals,[a] extremes 1869–present[b]  \\\n",
       "Month                                                                                                                                                                                                                   Jan   \n",
       "Record high °F (°C)                                                                                                                                       72(22)                                                              \n",
       "Mean maximum °F (°C)                                                                                                                                  59.6(15.3)                                                              \n",
       "Average high °F (°C)                                                                                                                                   38.3(3.5)                                                              \n",
       "Daily mean °F (°C)                                                                                                                                     32.6(0.3)                                                              \n",
       "Average low °F (°C)                                                                                                                                   26.9(−2.8)                                                              \n",
       "Mean minimum °F (°C)                                                                                                                                  9.2(−12.7)                                                              \n",
       "Record low °F (°C)                                                                                                                                       −6(−21)                                                              \n",
       "Average precipitation inches (mm)                                                                                                                       3.65(93)                                                              \n",
       "Average snowfall inches (cm)                                                                                                                             7.0(18)                                                              \n",
       "Average precipitation days (≥ 0.01 in)                                                                                                                      10.4                                                              \n",
       "Average snowy days (≥ 0.1 in)                                                                                                                                4.0                                                              \n",
       "Average relative humidity (%)                                                                                                                               61.5                                                              \n",
       "Mean monthly sunshine hours                                                                                                                                162.7                                                              \n",
       "Percent possible sunshine                                                                                                                                     54                                                              \n",
       "Average ultraviolet index                                                                                                                                      2                                                              \n",
       "Source #1: NOAA (relative humidity and sun 1961...                                                             Source #1: NOAA (relative humidity and sun 196...                                                              \n",
       "Source #2: Weather Atlas[212] See Geography of ...                                                             Source #2: Weather Atlas[212] See Geography of...                                                              \n",
       "\n",
       "vteClimate data for New York (Belvedere Castle, Central Park), 1981–2010 normals,[a] extremes 1869–present[b]                                                     \\\n",
       "Month                                                                                                                                                        Feb   \n",
       "Record high °F (°C)                                                                                                                                       78(26)   \n",
       "Mean maximum °F (°C)                                                                                                                                  60.7(15.9)   \n",
       "Average high °F (°C)                                                                                                                                   41.6(5.3)   \n",
       "Daily mean °F (°C)                                                                                                                                     35.3(1.8)   \n",
       "Average low °F (°C)                                                                                                                                   28.9(−1.7)   \n",
       "Mean minimum °F (°C)                                                                                                                                 12.8(−10.7)   \n",
       "Record low °F (°C)                                                                                                                                      −15(−26)   \n",
       "Average precipitation inches (mm)                                                                                                                       3.09(78)   \n",
       "Average snowfall inches (cm)                                                                                                                             9.2(23)   \n",
       "Average precipitation days (≥ 0.01 in)                                                                                                                       9.2   \n",
       "Average snowy days (≥ 0.1 in)                                                                                                                                2.8   \n",
       "Average relative humidity (%)                                                                                                                               60.2   \n",
       "Mean monthly sunshine hours                                                                                                                                163.1   \n",
       "Percent possible sunshine                                                                                                                                     55   \n",
       "Average ultraviolet index                                                                                                                                      3   \n",
       "Source #1: NOAA (relative humidity and sun 1961...                                                             Source #1: NOAA (relative humidity and sun 196...   \n",
       "Source #2: Weather Atlas[212] See Geography of ...                                                             Source #2: Weather Atlas[212] See Geography of...   \n",
       "\n",
       "vteClimate data for New York (Belvedere Castle, Central Park), 1981–2010 normals,[a] extremes 1869–present[b]                                                     \\\n",
       "Month                                                                                                                                                        Mar   \n",
       "Record high °F (°C)                                                                                                                                       86(30)   \n",
       "Mean maximum °F (°C)                                                                                                                                  71.5(21.9)   \n",
       "Average high °F (°C)                                                                                                                                   49.7(9.8)   \n",
       "Daily mean °F (°C)                                                                                                                                     42.5(5.8)   \n",
       "Average low °F (°C)                                                                                                                                    35.2(1.8)   \n",
       "Mean minimum °F (°C)                                                                                                                                  18.5(−7.5)   \n",
       "Record low °F (°C)                                                                                                                                        3(−16)   \n",
       "Average precipitation inches (mm)                                                                                                                      4.36(111)   \n",
       "Average snowfall inches (cm)                                                                                                                            3.9(9.9)   \n",
       "Average precipitation days (≥ 0.01 in)                                                                                                                      10.9   \n",
       "Average snowy days (≥ 0.1 in)                                                                                                                                1.8   \n",
       "Average relative humidity (%)                                                                                                                               58.5   \n",
       "Mean monthly sunshine hours                                                                                                                                212.5   \n",
       "Percent possible sunshine                                                                                                                                     57   \n",
       "Average ultraviolet index                                                                                                                                      4   \n",
       "Source #1: NOAA (relative humidity and sun 1961...                                                             Source #1: NOAA (relative humidity and sun 196...   \n",
       "Source #2: Weather Atlas[212] See Geography of ...                                                             Source #2: Weather Atlas[212] See Geography of...   \n",
       "\n",
       "vteClimate data for New York (Belvedere Castle, Central Park), 1981–2010 normals,[a] extremes 1869–present[b]                                                     \\\n",
       "Month                                                                                                                                                        Apr   \n",
       "Record high °F (°C)                                                                                                                                       96(36)   \n",
       "Mean maximum °F (°C)                                                                                                                                  83.0(28.3)   \n",
       "Average high °F (°C)                                                                                                                                  61.2(16.2)   \n",
       "Daily mean °F (°C)                                                                                                                                    53.0(11.7)   \n",
       "Average low °F (°C)                                                                                                                                    44.8(7.1)   \n",
       "Mean minimum °F (°C)                                                                                                                                   32.3(0.2)   \n",
       "Record low °F (°C)                                                                                                                                       12(−11)   \n",
       "Average precipitation inches (mm)                                                                                                                      4.50(114)   \n",
       "Average snowfall inches (cm)                                                                                                                            0.6(1.5)   \n",
       "Average precipitation days (≥ 0.01 in)                                                                                                                      11.5   \n",
       "Average snowy days (≥ 0.1 in)                                                                                                                                0.3   \n",
       "Average relative humidity (%)                                                                                                                               55.3   \n",
       "Mean monthly sunshine hours                                                                                                                                225.6   \n",
       "Percent possible sunshine                                                                                                                                     57   \n",
       "Average ultraviolet index                                                                                                                                      6   \n",
       "Source #1: NOAA (relative humidity and sun 1961...                                                             Source #1: NOAA (relative humidity and sun 196...   \n",
       "Source #2: Weather Atlas[212] See Geography of ...                                                             Source #2: Weather Atlas[212] See Geography of...   \n",
       "\n",
       "vteClimate data for New York (Belvedere Castle, Central Park), 1981–2010 normals,[a] extremes 1869–present[b]                                                     \\\n",
       "Month                                                                                                                                                        May   \n",
       "Record high °F (°C)                                                                                                                                       99(37)   \n",
       "Mean maximum °F (°C)                                                                                                                                  88.0(31.1)   \n",
       "Average high °F (°C)                                                                                                                                  70.8(21.6)   \n",
       "Daily mean °F (°C)                                                                                                                                    62.4(16.9)   \n",
       "Average low °F (°C)                                                                                                                                   54.0(12.2)   \n",
       "Mean minimum °F (°C)                                                                                                                                   43.5(6.4)   \n",
       "Record low °F (°C)                                                                                                                                         32(0)   \n",
       "Average precipitation inches (mm)                                                                                                                      4.19(106)   \n",
       "Average snowfall inches (cm)                                                                                                                                0(0)   \n",
       "Average precipitation days (≥ 0.01 in)                                                                                                                      11.1   \n",
       "Average snowy days (≥ 0.1 in)                                                                                                                                  0   \n",
       "Average relative humidity (%)                                                                                                                               62.7   \n",
       "Mean monthly sunshine hours                                                                                                                                256.6   \n",
       "Percent possible sunshine                                                                                                                                     57   \n",
       "Average ultraviolet index                                                                                                                                      7   \n",
       "Source #1: NOAA (relative humidity and sun 1961...                                                             Source #1: NOAA (relative humidity and sun 196...   \n",
       "Source #2: Weather Atlas[212] See Geography of ...                                                             Source #2: Weather Atlas[212] See Geography of...   \n",
       "\n",
       "vteClimate data for New York (Belvedere Castle, Central Park), 1981–2010 normals,[a] extremes 1869–present[b]                                                     \\\n",
       "Month                                                                                                                                                        Jun   \n",
       "Record high °F (°C)                                                                                                                                      101(38)   \n",
       "Mean maximum °F (°C)                                                                                                                                  92.3(33.5)   \n",
       "Average high °F (°C)                                                                                                                                  79.3(26.3)   \n",
       "Daily mean °F (°C)                                                                                                                                    71.5(21.9)   \n",
       "Average low °F (°C)                                                                                                                                   63.6(17.6)   \n",
       "Mean minimum °F (°C)                                                                                                                                  52.9(11.6)   \n",
       "Record low °F (°C)                                                                                                                                         44(7)   \n",
       "Average precipitation inches (mm)                                                                                                                      4.41(112)   \n",
       "Average snowfall inches (cm)                                                                                                                                0(0)   \n",
       "Average precipitation days (≥ 0.01 in)                                                                                                                      11.2   \n",
       "Average snowy days (≥ 0.1 in)                                                                                                                                  0   \n",
       "Average relative humidity (%)                                                                                                                               65.2   \n",
       "Mean monthly sunshine hours                                                                                                                                257.3   \n",
       "Percent possible sunshine                                                                                                                                     57   \n",
       "Average ultraviolet index                                                                                                                                      8   \n",
       "Source #1: NOAA (relative humidity and sun 1961...                                                             Source #1: NOAA (relative humidity and sun 196...   \n",
       "Source #2: Weather Atlas[212] See Geography of ...                                                             Source #2: Weather Atlas[212] See Geography of...   \n",
       "\n",
       "vteClimate data for New York (Belvedere Castle, Central Park), 1981–2010 normals,[a] extremes 1869–present[b]                                                     \\\n",
       "Month                                                                                                                                                        Jul   \n",
       "Record high °F (°C)                                                                                                                                      106(41)   \n",
       "Mean maximum °F (°C)                                                                                                                                  95.4(35.2)   \n",
       "Average high °F (°C)                                                                                                                                  84.1(28.9)   \n",
       "Daily mean °F (°C)                                                                                                                                    76.5(24.7)   \n",
       "Average low °F (°C)                                                                                                                                   68.8(20.4)   \n",
       "Mean minimum °F (°C)                                                                                                                                  60.3(15.7)   \n",
       "Record low °F (°C)                                                                                                                                        52(11)   \n",
       "Average precipitation inches (mm)                                                                                                                      4.60(117)   \n",
       "Average snowfall inches (cm)                                                                                                                                0(0)   \n",
       "Average precipitation days (≥ 0.01 in)                                                                                                                      10.4   \n",
       "Average snowy days (≥ 0.1 in)                                                                                                                                  0   \n",
       "Average relative humidity (%)                                                                                                                               64.2   \n",
       "Mean monthly sunshine hours                                                                                                                                268.2   \n",
       "Percent possible sunshine                                                                                                                                     59   \n",
       "Average ultraviolet index                                                                                                                                      8   \n",
       "Source #1: NOAA (relative humidity and sun 1961...                                                             Source #1: NOAA (relative humidity and sun 196...   \n",
       "Source #2: Weather Atlas[212] See Geography of ...                                                             Source #2: Weather Atlas[212] See Geography of...   \n",
       "\n",
       "vteClimate data for New York (Belvedere Castle, Central Park), 1981–2010 normals,[a] extremes 1869–present[b]                                                     \\\n",
       "Month                                                                                                                                                        Aug   \n",
       "Record high °F (°C)                                                                                                                                      104(40)   \n",
       "Mean maximum °F (°C)                                                                                                                                  93.7(34.3)   \n",
       "Average high °F (°C)                                                                                                                                  82.6(28.1)   \n",
       "Daily mean °F (°C)                                                                                                                                    75.2(24.0)   \n",
       "Average low °F (°C)                                                                                                                                   67.8(19.9)   \n",
       "Mean minimum °F (°C)                                                                                                                                  58.8(14.9)   \n",
       "Record low °F (°C)                                                                                                                                        50(10)   \n",
       "Average precipitation inches (mm)                                                                                                                      4.44(113)   \n",
       "Average snowfall inches (cm)                                                                                                                                0(0)   \n",
       "Average precipitation days (≥ 0.01 in)                                                                                                                       9.5   \n",
       "Average snowy days (≥ 0.1 in)                                                                                                                                  0   \n",
       "Average relative humidity (%)                                                                                                                               66.0   \n",
       "Mean monthly sunshine hours                                                                                                                                268.2   \n",
       "Percent possible sunshine                                                                                                                                     63   \n",
       "Average ultraviolet index                                                                                                                                      8   \n",
       "Source #1: NOAA (relative humidity and sun 1961...                                                             Source #1: NOAA (relative humidity and sun 196...   \n",
       "Source #2: Weather Atlas[212] See Geography of ...                                                             Source #2: Weather Atlas[212] See Geography of...   \n",
       "\n",
       "vteClimate data for New York (Belvedere Castle, Central Park), 1981–2010 normals,[a] extremes 1869–present[b]                                                     \\\n",
       "Month                                                                                                                                                        Sep   \n",
       "Record high °F (°C)                                                                                                                                      102(39)   \n",
       "Mean maximum °F (°C)                                                                                                                                  88.5(31.4)   \n",
       "Average high °F (°C)                                                                                                                                  75.2(24.0)   \n",
       "Daily mean °F (°C)                                                                                                                                    68.0(20.0)   \n",
       "Average low °F (°C)                                                                                                                                   60.8(16.0)   \n",
       "Mean minimum °F (°C)                                                                                                                                   48.6(9.2)   \n",
       "Record low °F (°C)                                                                                                                                         39(4)   \n",
       "Average precipitation inches (mm)                                                                                                                      4.28(109)   \n",
       "Average snowfall inches (cm)                                                                                                                                0(0)   \n",
       "Average precipitation days (≥ 0.01 in)                                                                                                                       8.7   \n",
       "Average snowy days (≥ 0.1 in)                                                                                                                                  0   \n",
       "Average relative humidity (%)                                                                                                                               67.8   \n",
       "Mean monthly sunshine hours                                                                                                                                219.3   \n",
       "Percent possible sunshine                                                                                                                                     59   \n",
       "Average ultraviolet index                                                                                                                                      6   \n",
       "Source #1: NOAA (relative humidity and sun 1961...                                                             Source #1: NOAA (relative humidity and sun 196...   \n",
       "Source #2: Weather Atlas[212] See Geography of ...                                                             Source #2: Weather Atlas[212] See Geography of...   \n",
       "\n",
       "vteClimate data for New York (Belvedere Castle, Central Park), 1981–2010 normals,[a] extremes 1869–present[b]                                                     \\\n",
       "Month                                                                                                                                                        Oct   \n",
       "Record high °F (°C)                                                                                                                                       94(34)   \n",
       "Mean maximum °F (°C)                                                                                                                                  78.8(26.0)   \n",
       "Average high °F (°C)                                                                                                                                  63.8(17.7)   \n",
       "Daily mean °F (°C)                                                                                                                                    56.9(13.8)   \n",
       "Average low °F (°C)                                                                                                                                   50.0(10.0)   \n",
       "Mean minimum °F (°C)                                                                                                                                   38.0(3.3)   \n",
       "Record low °F (°C)                                                                                                                                        28(−2)   \n",
       "Average precipitation inches (mm)                                                                                                                      4.40(112)   \n",
       "Average snowfall inches (cm)                                                                                                                                0(0)   \n",
       "Average precipitation days (≥ 0.01 in)                                                                                                                       8.9   \n",
       "Average snowy days (≥ 0.1 in)                                                                                                                                  0   \n",
       "Average relative humidity (%)                                                                                                                               65.6   \n",
       "Mean monthly sunshine hours                                                                                                                                211.2   \n",
       "Percent possible sunshine                                                                                                                                     61   \n",
       "Average ultraviolet index                                                                                                                                      4   \n",
       "Source #1: NOAA (relative humidity and sun 1961...                                                             Source #1: NOAA (relative humidity and sun 196...   \n",
       "Source #2: Weather Atlas[212] See Geography of ...                                                             Source #2: Weather Atlas[212] See Geography of...   \n",
       "\n",
       "vteClimate data for New York (Belvedere Castle, Central Park), 1981–2010 normals,[a] extremes 1869–present[b]                                                     \\\n",
       "Month                                                                                                                                                        Nov   \n",
       "Record high °F (°C)                                                                                                                                       84(29)   \n",
       "Mean maximum °F (°C)                                                                                                                                  71.3(21.8)   \n",
       "Average high °F (°C)                                                                                                                                  53.8(12.1)   \n",
       "Daily mean °F (°C)                                                                                                                                     47.7(8.7)   \n",
       "Average low °F (°C)                                                                                                                                    41.6(5.3)   \n",
       "Mean minimum °F (°C)                                                                                                                                  27.7(−2.4)   \n",
       "Record low °F (°C)                                                                                                                                        5(−15)   \n",
       "Average precipitation inches (mm)                                                                                                                      4.02(102)   \n",
       "Average snowfall inches (cm)                                                                                                                           0.3(0.76)   \n",
       "Average precipitation days (≥ 0.01 in)                                                                                                                       9.6   \n",
       "Average snowy days (≥ 0.1 in)                                                                                                                                0.2   \n",
       "Average relative humidity (%)                                                                                                                               64.6   \n",
       "Mean monthly sunshine hours                                                                                                                                151.0   \n",
       "Percent possible sunshine                                                                                                                                     51   \n",
       "Average ultraviolet index                                                                                                                                      2   \n",
       "Source #1: NOAA (relative humidity and sun 1961...                                                             Source #1: NOAA (relative humidity and sun 196...   \n",
       "Source #2: Weather Atlas[212] See Geography of ...                                                             Source #2: Weather Atlas[212] See Geography of...   \n",
       "\n",
       "vteClimate data for New York (Belvedere Castle, Central Park), 1981–2010 normals,[a] extremes 1869–present[b]                                                     \\\n",
       "Month                                                                                                                                                        Dec   \n",
       "Record high °F (°C)                                                                                                                                       75(24)   \n",
       "Mean maximum °F (°C)                                                                                                                                  62.2(16.8)   \n",
       "Average high °F (°C)                                                                                                                                   43.0(6.1)   \n",
       "Daily mean °F (°C)                                                                                                                                     37.5(3.1)   \n",
       "Average low °F (°C)                                                                                                                                    32.0(0.0)   \n",
       "Mean minimum °F (°C)                                                                                                                                  15.6(−9.1)   \n",
       "Record low °F (°C)                                                                                                                                      −13(−25)   \n",
       "Average precipitation inches (mm)                                                                                                                      4.00(102)   \n",
       "Average snowfall inches (cm)                                                                                                                             4.8(12)   \n",
       "Average precipitation days (≥ 0.01 in)                                                                                                                      10.6   \n",
       "Average snowy days (≥ 0.1 in)                                                                                                                                2.3   \n",
       "Average relative humidity (%)                                                                                                                               64.1   \n",
       "Mean monthly sunshine hours                                                                                                                                139.0   \n",
       "Percent possible sunshine                                                                                                                                     48   \n",
       "Average ultraviolet index                                                                                                                                      1   \n",
       "Source #1: NOAA (relative humidity and sun 1961...                                                             Source #1: NOAA (relative humidity and sun 196...   \n",
       "Source #2: Weather Atlas[212] See Geography of ...                                                             Source #2: Weather Atlas[212] See Geography of...   \n",
       "\n",
       "vteClimate data for New York (Belvedere Castle, Central Park), 1981–2010 normals,[a] extremes 1869–present[b]                                                     \n",
       "Month                                                                                                                                                       Year  \n",
       "Record high °F (°C)                                                                                                                                      106(41)  \n",
       "Mean maximum °F (°C)                                                                                                                                  97.0(36.1)  \n",
       "Average high °F (°C)                                                                                                                                  62.0(16.7)  \n",
       "Daily mean °F (°C)                                                                                                                                    55.0(12.8)  \n",
       "Average low °F (°C)                                                                                                                                    48.0(8.9)  \n",
       "Mean minimum °F (°C)                                                                                                                                  7.0(−13.9)  \n",
       "Record low °F (°C)                                                                                                                                      −15(−26)  \n",
       "Average precipitation inches (mm)                                                                                                                   49.94(1,268)  \n",
       "Average snowfall inches (cm)                                                                                                                            25.8(66)  \n",
       "Average precipitation days (≥ 0.01 in)                                                                                                                     122.0  \n",
       "Average snowy days (≥ 0.1 in)                                                                                                                               11.4  \n",
       "Average relative humidity (%)                                                                                                                               63.0  \n",
       "Mean monthly sunshine hours                                                                                                                               2534.7  \n",
       "Percent possible sunshine                                                                                                                                     57  \n",
       "Average ultraviolet index                                                                                                                                      5  \n",
       "Source #1: NOAA (relative humidity and sun 1961...                                                             Source #1: NOAA (relative humidity and sun 196...  \n",
       "Source #2: Weather Atlas[212] See Geography of ...                                                             Source #2: Weather Atlas[212] See Geography of...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wikitables[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th>New York City's five boroughsvte</th>\n",
       "      <th colspan=\"8\" halign=\"left\">New York City's five boroughsvte</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Jurisdiction</th>\n",
       "      <th>Jurisdiction</th>\n",
       "      <th>Population</th>\n",
       "      <th colspan=\"2\" halign=\"left\">Gross Domestic Product</th>\n",
       "      <th colspan=\"2\" halign=\"left\">Land area</th>\n",
       "      <th colspan=\"2\" halign=\"left\">Density</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Borough</th>\n",
       "      <th>County</th>\n",
       "      <th>Estimate (2018)[150]</th>\n",
       "      <th>billions(US$)[151]</th>\n",
       "      <th>per capita(US$)</th>\n",
       "      <th>square miles</th>\n",
       "      <th>squarekm</th>\n",
       "      <th>persons / sq. mi</th>\n",
       "      <th>persons /km2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>The Bronx</th>\n",
       "      <td>Bronx</td>\n",
       "      <td>1432132</td>\n",
       "      <td>42.695</td>\n",
       "      <td>29200</td>\n",
       "      <td>42.10</td>\n",
       "      <td>109.04</td>\n",
       "      <td>34653</td>\n",
       "      <td>13231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Brooklyn</th>\n",
       "      <td>Kings</td>\n",
       "      <td>2582830</td>\n",
       "      <td>91.559</td>\n",
       "      <td>34600</td>\n",
       "      <td>70.82</td>\n",
       "      <td>183.42</td>\n",
       "      <td>37137</td>\n",
       "      <td>14649</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Manhattan</th>\n",
       "      <td>New York</td>\n",
       "      <td>1628701</td>\n",
       "      <td>600.244</td>\n",
       "      <td>360900</td>\n",
       "      <td>22.83</td>\n",
       "      <td>59.13</td>\n",
       "      <td>72033</td>\n",
       "      <td>27826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Queens</th>\n",
       "      <td>Queens</td>\n",
       "      <td>2278906</td>\n",
       "      <td>93.310</td>\n",
       "      <td>39600</td>\n",
       "      <td>108.53</td>\n",
       "      <td>281.09</td>\n",
       "      <td>21460</td>\n",
       "      <td>8354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Staten Island</th>\n",
       "      <td>Richmond</td>\n",
       "      <td>476179</td>\n",
       "      <td>14.514</td>\n",
       "      <td>30300</td>\n",
       "      <td>58.37</td>\n",
       "      <td>151.18</td>\n",
       "      <td>8112</td>\n",
       "      <td>3132</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "New York City's five boroughsvte New York City's five boroughsvte  \\\n",
       "Jurisdiction                                         Jurisdiction   \n",
       "Borough                                                    County   \n",
       "The Bronx                                                   Bronx   \n",
       "Brooklyn                                                    Kings   \n",
       "Manhattan                                                New York   \n",
       "Queens                                                     Queens   \n",
       "Staten Island                                            Richmond   \n",
       "\n",
       "New York City's five boroughsvte                                              \\\n",
       "Jurisdiction                               Population Gross Domestic Product   \n",
       "Borough                          Estimate (2018)[150]     billions(US$)[151]   \n",
       "The Bronx                                     1432132                 42.695   \n",
       "Brooklyn                                      2582830                 91.559   \n",
       "Manhattan                                     1628701                600.244   \n",
       "Queens                                        2278906                 93.310   \n",
       "Staten Island                                  476179                 14.514   \n",
       "\n",
       "New York City's five boroughsvte                                        \\\n",
       "Jurisdiction                                        Land area            \n",
       "Borough                          per capita(US$) square miles squarekm   \n",
       "The Bronx                                  29200        42.10   109.04   \n",
       "Brooklyn                                   34600        70.82   183.42   \n",
       "Manhattan                                 360900        22.83    59.13   \n",
       "Queens                                     39600       108.53   281.09   \n",
       "Staten Island                              30300        58.37   151.18   \n",
       "\n",
       "New York City's five boroughsvte                                \n",
       "Jurisdiction                              Density               \n",
       "Borough                          persons / sq. mi persons /km2  \n",
       "The Bronx                                   34653        13231  \n",
       "Brooklyn                                    37137        14649  \n",
       "Manhattan                                   72033        27826  \n",
       "Queens                                      21460         8354  \n",
       "Staten Island                                8112         3132  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wikitables[0].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracted 1 wikitables\n"
     ]
    }
   ],
   "source": [
    "# change the index table\n",
    "from pandas.io.html import read_html\n",
    "page = 'https://en.wikipedia.org/wiki/List_of_countries_by_population_(United_Nations)'\n",
    "\n",
    "wikitables = read_html(page, index_col=1, attrs={\"class\":\"wikitable\"})\n",
    "\n",
    "print (\"Extracted {num} wikitables\".format(num=len(wikitables)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Country or area</th>\n",
       "      <td>Rank</td>\n",
       "      <td>UN continentalregion[2]</td>\n",
       "      <td>UN statisticalregion[2]</td>\n",
       "      <td>Population(1 July 2016)[3]</td>\n",
       "      <td>Population(1 July 2017)[3]</td>\n",
       "      <td>Change</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>World</th>\n",
       "      <td>—</td>\n",
       "      <td>—</td>\n",
       "      <td>—</td>\n",
       "      <td>7466964280</td>\n",
       "      <td>7550262101</td>\n",
       "      <td>+1.1%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>China[a]</th>\n",
       "      <td>1</td>\n",
       "      <td>Asia</td>\n",
       "      <td>Eastern Asia</td>\n",
       "      <td>1403500365</td>\n",
       "      <td>1409517397</td>\n",
       "      <td>+0.4%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>India</th>\n",
       "      <td>2</td>\n",
       "      <td>Asia</td>\n",
       "      <td>Southern Asia</td>\n",
       "      <td>1324171354</td>\n",
       "      <td>1339180127</td>\n",
       "      <td>+1.1%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>United States</th>\n",
       "      <td>3</td>\n",
       "      <td>Americas</td>\n",
       "      <td>Northern America</td>\n",
       "      <td>322179605</td>\n",
       "      <td>324459463</td>\n",
       "      <td>+0.7%</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    0                        2                        3  \\\n",
       "1                                                                         \n",
       "Country or area  Rank  UN continentalregion[2]  UN statisticalregion[2]   \n",
       "World               —                        —                        —   \n",
       "China[a]            1                     Asia             Eastern Asia   \n",
       "India               2                     Asia            Southern Asia   \n",
       "United States       3                 Americas         Northern America   \n",
       "\n",
       "                                          4                           5  \\\n",
       "1                                                                         \n",
       "Country or area  Population(1 July 2016)[3]  Population(1 July 2017)[3]   \n",
       "World                            7466964280                  7550262101   \n",
       "China[a]                         1403500365                  1409517397   \n",
       "India                            1324171354                  1339180127   \n",
       "United States                     322179605                   324459463   \n",
       "\n",
       "                      6  \n",
       "1                        \n",
       "Country or area  Change  \n",
       "World             +1.1%  \n",
       "China[a]          +0.4%  \n",
       "India             +1.1%  \n",
       "United States     +0.7%  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wikitables[0].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracted 1 wikitables\n"
     ]
    }
   ],
   "source": [
    "# works with different languages ( option encoding is available if needed)\n",
    "from pandas.io.html import read_html\n",
    "page = 'https://zh.wikipedia.org/wiki/%E4%B8%96%E7%95%8C%E5%9B%BD%E5%AE%B6%E5%92%8C%E5%9C%B0%E5%8C%BA%E4%BA%BA%E5%8F%A3%E6%8E%92%E5%90%8D%E5%88%97%E8%A1%A8'\n",
    "\n",
    "wikitables = read_html(page, index_col=0, attrs={\"class\":\"wikitable\"})\n",
    "\n",
    "print (\"Extracted {num} wikitables\".format(num=len(wikitables)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>排名</th>\n",
       "      <td>国家或者地区</td>\n",
       "      <td>大洲[2]</td>\n",
       "      <td>統計地區[2]</td>\n",
       "      <td>人口(2016年7月1日)[3]</td>\n",
       "      <td>人口(2017年7月1日)[3]</td>\n",
       "      <td>变化率</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>—</th>\n",
       "      <td>世界</td>\n",
       "      <td>—</td>\n",
       "      <td>—</td>\n",
       "      <td>7466964280</td>\n",
       "      <td>7550262101</td>\n",
       "      <td>+1.1%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>中华人民共和国[a]</td>\n",
       "      <td>亚洲</td>\n",
       "      <td>东亚</td>\n",
       "      <td>1403500365</td>\n",
       "      <td>1409517397</td>\n",
       "      <td>+0.4%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>印度</td>\n",
       "      <td>亚洲</td>\n",
       "      <td>南亚</td>\n",
       "      <td>1324171354</td>\n",
       "      <td>1339180127</td>\n",
       "      <td>+1.1%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>美國</td>\n",
       "      <td>美洲</td>\n",
       "      <td>北美</td>\n",
       "      <td>322179605</td>\n",
       "      <td>324459463</td>\n",
       "      <td>+0.7%</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             1      2        3                 4                 5      6\n",
       "0                                                                        \n",
       "排名      国家或者地区  大洲[2]  統計地區[2]  人口(2016年7月1日)[3]  人口(2017年7月1日)[3]    变化率\n",
       "—           世界      —        —        7466964280        7550262101  +1.1%\n",
       "1   中华人民共和国[a]     亚洲       东亚        1403500365        1409517397  +0.4%\n",
       "2           印度     亚洲       南亚        1324171354        1339180127  +1.1%\n",
       "3           美國     美洲       北美         322179605         324459463  +0.7%"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wikitables[0].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read wiki Infoboxes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracted 1 infoboxes\n",
      "Extracted 4 wikitables\n"
     ]
    }
   ],
   "source": [
    "from pandas.io.html import read_html\n",
    "page = 'https://en.wikipedia.org/wiki/University_of_California,_Berkeley'\n",
    "infoboxes = read_html(page, index_col=0, attrs={\"class\":\"infobox\"})\n",
    "wikitables = read_html(page, index_col=0, attrs={\"class\":\"wikitable\"})\n",
    "\n",
    "print (\"Extracted {num} infoboxes\".format(num=len(infoboxes)))\n",
    "print (\"Extracted {num} wikitables\".format(num=len(wikitables)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>University rankings</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>National</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ARWU[106]</th>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Forbes[107]</th>\n",
       "      <td>14.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>U.S. News &amp; World Report[108]</th>\n",
       "      <td>22.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Washington Monthly[109]</th>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Global</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ARWU[110]</th>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>QS[111]</th>\n",
       "      <td>27.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Times[112]</th>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>U.S. News &amp; World Report[113]</th>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  1\n",
       "0                                  \n",
       "University rankings             NaN\n",
       "National                        NaN\n",
       "ARWU[106]                       4.0\n",
       "Forbes[107]                    14.0\n",
       "U.S. News & World Report[108]  22.0\n",
       "Washington Monthly[109]         7.0\n",
       "Global                          NaN\n",
       "ARWU[110]                       5.0\n",
       "QS[111]                        27.0\n",
       "Times[112]                     15.0\n",
       "U.S. News & World Report[113]   4.0"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "infoboxes[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracted 1 infoboxes\n",
      "Extracted 1 wikitables\n"
     ]
    }
   ],
   "source": [
    "from pandas.io.html import read_html\n",
    "page = 'https://en.wikipedia.org/wiki/Lisbon'\n",
    "infoboxes = read_html(page, index_col=0, attrs={\"class\":\"infobox geography vcard\"})\n",
    "wikitables = read_html(page, index_col=0, attrs={\"class\":\"wikitable\"})\n",
    "\n",
    "print (\"Extracted {num} infoboxes\".format(num=len(infoboxes)))\n",
    "print (\"Extracted {num} wikitables\".format(num=len(wikitables)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Country</th>\n",
       "      <td>Portugal</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NUTS II Region</th>\n",
       "      <td>Lisbon metropolitan area</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NUTS III Subregion</th>\n",
       "      <td>Lisbon metropolitan area</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>District</th>\n",
       "      <td>Lisbon</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Municipality</th>\n",
       "      <td>Lisbon</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Settlement</th>\n",
       "      <td>Prior to Roman rule</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>City</th>\n",
       "      <td>c. 1256</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Civil parishes</th>\n",
       "      <td>(see text)</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Government</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>• Type</th>\n",
       "      <td>LAU</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           1    2\n",
       "0                                                \n",
       "Country                             Portugal  NaN\n",
       "NUTS II Region      Lisbon metropolitan area  NaN\n",
       "NUTS III Subregion  Lisbon metropolitan area  NaN\n",
       "District                              Lisbon  NaN\n",
       "Municipality                          Lisbon  NaN\n",
       "Settlement               Prior to Roman rule  NaN\n",
       "City                                 c. 1256  NaN\n",
       "Civil parishes                    (see text)  NaN\n",
       "Government                               NaN  NaN\n",
       "• Type                                   LAU  NaN"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "infoboxes[0][10:20]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scrape non wiki tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>NaN</th>\n",
       "      <td>Player ID</td>\n",
       "      <td>Player Name</td>\n",
       "      <td>Total (Overall)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Highest Paying Game</td>\n",
       "      <td>Total (Game)</td>\n",
       "      <td>% of Total</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.0</th>\n",
       "      <td>KuroKy</td>\n",
       "      <td>Kuro Takhasomi</td>\n",
       "      <td>$4,136,926.95</td>\n",
       "      <td>|</td>\n",
       "      <td>Dota 2</td>\n",
       "      <td>$4,135,203.61</td>\n",
       "      <td>99.96%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2.0</th>\n",
       "      <td>N0tail</td>\n",
       "      <td>Johan Sundstein</td>\n",
       "      <td>$3,742,055.59</td>\n",
       "      <td>|</td>\n",
       "      <td>Dota 2</td>\n",
       "      <td>$3,733,903.98</td>\n",
       "      <td>99.78%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3.0</th>\n",
       "      <td>Miracle-</td>\n",
       "      <td>Amer Al-Barkawi</td>\n",
       "      <td>$3,701,337.28</td>\n",
       "      <td>|</td>\n",
       "      <td>Dota 2</td>\n",
       "      <td>$3,701,337.28</td>\n",
       "      <td>100.00%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4.0</th>\n",
       "      <td>MinD_ContRoL</td>\n",
       "      <td>Ivan Ivanov</td>\n",
       "      <td>$3,492,411.76</td>\n",
       "      <td>|</td>\n",
       "      <td>Dota 2</td>\n",
       "      <td>$3,492,411.76</td>\n",
       "      <td>100.00%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5.0</th>\n",
       "      <td>Matumbaman</td>\n",
       "      <td>Lasse Urpalainen</td>\n",
       "      <td>$3,476,116.04</td>\n",
       "      <td>|</td>\n",
       "      <td>Dota 2</td>\n",
       "      <td>$3,476,116.04</td>\n",
       "      <td>100.00%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6.0</th>\n",
       "      <td>JerAx</td>\n",
       "      <td>Jesse Vainikka</td>\n",
       "      <td>$3,313,463.82</td>\n",
       "      <td>|</td>\n",
       "      <td>Dota 2</td>\n",
       "      <td>$3,313,463.82</td>\n",
       "      <td>100.00%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7.0</th>\n",
       "      <td>SumaiL</td>\n",
       "      <td>Sumail Hassan</td>\n",
       "      <td>$3,305,914.94</td>\n",
       "      <td>|</td>\n",
       "      <td>Dota 2</td>\n",
       "      <td>$3,305,914.94</td>\n",
       "      <td>100.00%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8.0</th>\n",
       "      <td>GH</td>\n",
       "      <td>Maroun Merhej</td>\n",
       "      <td>$3,095,344.84</td>\n",
       "      <td>|</td>\n",
       "      <td>Dota 2</td>\n",
       "      <td>$3,095,344.84</td>\n",
       "      <td>100.00%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9.0</th>\n",
       "      <td>UNiVeRsE</td>\n",
       "      <td>Saahil Arora</td>\n",
       "      <td>$3,035,737.67</td>\n",
       "      <td>|</td>\n",
       "      <td>Dota 2</td>\n",
       "      <td>$3,035,737.67</td>\n",
       "      <td>100.00%</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 1                 2                3    4  \\\n",
       "0                                                            \n",
       "NaN      Player ID       Player Name  Total (Overall)  NaN   \n",
       " 1.0        KuroKy    Kuro Takhasomi    $4,136,926.95    |   \n",
       " 2.0        N0tail   Johan Sundstein    $3,742,055.59    |   \n",
       " 3.0      Miracle-   Amer Al-Barkawi    $3,701,337.28    |   \n",
       " 4.0  MinD_ContRoL       Ivan Ivanov    $3,492,411.76    |   \n",
       " 5.0    Matumbaman  Lasse Urpalainen    $3,476,116.04    |   \n",
       " 6.0         JerAx    Jesse Vainikka    $3,313,463.82    |   \n",
       " 7.0        SumaiL     Sumail Hassan    $3,305,914.94    |   \n",
       " 8.0            GH     Maroun Merhej    $3,095,344.84    |   \n",
       " 9.0      UNiVeRsE      Saahil Arora    $3,035,737.67    |   \n",
       "\n",
       "                        5              6           7  \n",
       "0                                                     \n",
       "NaN   Highest Paying Game   Total (Game)  % of Total  \n",
       " 1.0               Dota 2  $4,135,203.61      99.96%  \n",
       " 2.0               Dota 2  $3,733,903.98      99.78%  \n",
       " 3.0               Dota 2  $3,701,337.28     100.00%  \n",
       " 4.0               Dota 2  $3,492,411.76     100.00%  \n",
       " 5.0               Dota 2  $3,476,116.04     100.00%  \n",
       " 6.0               Dota 2  $3,313,463.82     100.00%  \n",
       " 7.0               Dota 2  $3,305,914.94     100.00%  \n",
       " 8.0               Dota 2  $3,095,344.84     100.00%  \n",
       " 9.0               Dota 2  $3,035,737.67     100.00%  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pandas.io.html import read_html\n",
    "page = 'https://www.esportsearnings.com/players'\n",
    "infoboxes = read_html(page, index_col=0, attrs={\"class\":\"detail_list_table\"})\n",
    "\n",
    "infoboxes[0].head(10)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convert html tables to csv/excel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pandas.io.html import read_html\n",
    "page = 'https://www.esportsearnings.com/players'\n",
    "infoboxes = read_html(page, index_col=0, attrs={\"class\":\"detail_list_table\"})\n",
    "\n",
    "file_name = './my_file.csv'\n",
    "infoboxes[0].to_csv(file_name, sep='\\t')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./my_file.csv\r\n",
      "./csv/movie_metadata.csv\r\n"
     ]
    }
   ],
   "source": [
    "!find . -type f -name \"*.csv\" "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Web Scraping Wikipedia Tables using BeautifulSoup and Python\n",
    "\n",
    "source: https://github.com/stewync/Web-Scraping-Wiki-tables-using-BeautifulSoup-and-Python/blob/master/Scraping%2BWiki%2Btable%2Busing%2BPython%2Band%2BBeautifulSoup.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "website_url = requests.get('https://en.wikipedia.org/wiki/List_of_UFC_events').text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "soup = BeautifulSoup(website_url,'lxml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "My_table = soup.find('table',{'class':'wikitable'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "links = My_table.findAll('a')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "events = []\n",
    "for link in links:\n",
    "    events.append(link.get('title'))    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>events</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>UFC on ESPN 4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TBA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>UFC on ESPN+ 11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TBA</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            events\n",
       "0    UFC on ESPN 4\n",
       "1              TBA\n",
       "2             None\n",
       "3  UFC on ESPN+ 11\n",
       "4              TBA"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "df = pd.DataFrame()\n",
    "df['events'] = events\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Other"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### wiki-table-scrape\n",
    "https://github.com/rocheio/wiki-table-scrape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scraping Wikipedia Tables with Python\n",
    "https://roche.io/2016/05/scrape-wikipedia-with-python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}