web-scraping-tutorial.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import Python's requests library\n",
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare the request\n",
    "url = 'https://jupyter.org'\n",
    "req = requests.Request('GET', url)\n",
    "req.headers['User-Agent'] = 'Mozilla/5.0'\n",
    "req = req.prepare()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Print the docstring for the prepared request string\n",
    "req?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run the request and store the response in a variable\n",
    "# We use the with statement so that the scope of the session is limited to the indented block\n",
    "# This way, the session is closed automatically and we don't need to close it explicitly\n",
    "with requests.Session() as sess:\n",
    "    page = sess.send(req)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Response [200]>\n"
     ]
    }
   ],
   "source": [
    "# Investigate the response - check to confirm a 200 status by printing the string representation\n",
    "print(page)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<!DOCTYPE html>\\n<html>\\n\\n  <head>\\n\\n    <meta charset=\"utf-8\">\\n    <meta http-equiv=\"X-UA-Compatible\" content=\"IE=edge\">\\n    <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\\n    <meta name=\"description\" content=\"\">\\n    <meta name=\"author\" content=\"\">\\n\\n    <title>Project Jupyter | Home</title>\\n    <meta property=\"og:title\" content=\"Project Jupyter\" />\\n    <meta property=\"og:description\" content=\"The Jupyter Notebook is a web-based interactive computing platform. The notebook combines live code, equations, narrative text, visualizations, interactive dashboards and other media.\\n\">\\n    <meta property=\"og:url\" content=\"https://www.jupyter.org\" />\\n    <meta property=\"og:image\" content=\"https://jupyter.org/assets/homepage.png\" />\\n    <!-- Bootstrap Core CSS -->\\n    <link rel=\"stylesheet\" href=\"/css/bootstrap.min.css\">\\n    <link rel=\"stylesheet\" href=\"/css/logo-nav.css?1594832988204333248\">\\n    <link rel=\"stylesheet\" href=\"/css/cardlist.css\">\\n    <link rel=\"stylesheet\" href=\"'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Print the first part of the html\n",
    "page_html = page.text\n",
    "page_html[:1000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<!DOCTYPE html>\n",
      "<html>\n",
      " <head>\n",
      "  <meta charset=\"utf-8\"/>\n",
      "  <meta content=\"IE=edge\" http-equiv=\"X-UA-Compatible\"/>\n",
      "  <meta content=\"width=device-width, initial-scale=1\" name=\"viewport\"/>\n",
      "  <meta content=\"\" name=\"description\"/>\n",
      "  <meta content=\"\" name=\"author\"/>\n",
      "  <title>\n",
      "   Project Jupyter | Home\n",
      "  </title>\n",
      "  <meta content=\"Project Jupyter\" property=\"og:title\">\n",
      "   <meta content=\"The Jupyter Notebook is a web-based interactive computing platform. The notebook combines live code, equations, narrative text, visualizations, interactive dashboards and other media.\n",
      "\" property=\"og:description\"/>\n",
      "   <meta content=\"https://www.jupyter.org\" property=\"og:url\">\n",
      "    <meta content=\"https://jupyter.org/assets/homepage.png\" property=\"og:image\">\n",
      "     <!-- Bootstrap Core CSS -->\n",
      "     <link href=\"/css/bootstrap.min.css\" rel=\"stylesheet\"/>\n",
      "     <link href=\"/css/logo-nav.css?1594832988204333248\" rel=\"stylesheet\"/>\n",
      "     <link href=\"/css/cardlist.css\" rel=\"stylesheet\"/>\n",
      "     <link href=\"/css/github-buttons.cs\n"
     ]
    }
   ],
   "source": [
    "# Pretty print the output using the BeautifulSoup library\n",
    "from bs4 import BeautifulSoup\n",
    "print(BeautifulSoup(page_html, 'html.parser').prettify()[:1000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<!DOCTYPE html>\n",
       "<html>\n",
       "\n",
       "  <head>\n",
       "\n",
       "    <meta charset=\"utf-8\">\n",
       "    <meta http-equiv=\"X-UA-Compatible\" content=\"IE=edge\">\n",
       "    <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n",
       "    <meta name=\"description\" content=\"\">\n",
       "    <meta name=\"author\" content=\"\">\n",
       "\n",
       "    <title>Project Jupyter | Home</title>\n",
       "    <meta property=\"og:title\" content=\"Project Jupyter\" />\n",
       "    <meta property=\"og:description\" content=\"The Jupyter Notebook is a web-based interactive computing platform. The notebook combines live code, equations, narrative text, visualizations, interactive dashboards and other media.\n",
       "\">\n",
       "    <meta property=\"og:url\" content=\"https://www.jupyter.org\" />\n",
       "    <meta property=\"og:image\" content=\"https://jupyter.org/assets/homepage.png\" />\n",
       "    <!-- Bootstrap Core CSS -->\n",
       "    <link rel=\"stylesheet\" href=\"/css/bootstrap.min.css\">\n",
       "    <link rel=\"stylesheet\" href=\"/css/logo-nav.css?1594832988204333248\">\n",
       "    <link rel=\"stylesheet\" href=\"/css/cardlist.css\">\n",
       "    <link rel=\"stylesheet\" href=\"/css/github-buttons.css\">\n",
       "    <link rel=\"stylesheet\" href=\"/css/markdown_page.css\">\n",
       "    <link rel=\"stylesheet\" href=\"/css/about.css\">\n",
       "    <link rel=\"icon\" type=\"image/png\" href=\"favicon.ico\" />\n",
       "    <link href=\"/assets/apple-touch-icon.png\" rel=\"apple-touch-icon\" />\n",
       "    <link href=\"/assets/apple-touch-icon-76x76.png\" rel=\"apple-touch-icon\" sizes=\"76x76\" />\n",
       "    <link href=\"/assets/apple-touch-icon-120x120.png\" rel=\"apple-touch-icon\" sizes=\"120x120\" />\n",
       "    <link href=\"/assets/apple-touch-icon-152x152.png\" rel=\"apple-touch-icon\" sizes=\"152x152\" />\n",
       "    <link href=\"/assets/apple-touch-icon-180x180.png\" rel=\"apple-touch-icon\" sizes=\"180x180\" />\n",
       "    <link href=\"/assets/icon-hires.png\" rel=\"icon\" sizes=\"192x192\" />\n",
       "    <link href=\"/assets/icon-normal.png\" rel=\"icon\" sizes=\"128x128\" />\n",
       "    <link rel=\"canonical\" href=\"https://jupyter.org/\">\n",
       "\n",
       "    <!-- HTML5 Shim and Respond.js IE8 support of HTML5 elements and media queries -->\n",
       "    <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->\n",
       "    <!--[if lt IE 9]>\n",
       "        <script src=\"https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js\"></script>\n",
       "        <script src=\"https://oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js\"></script>\n",
       "    <![endif]-->\n",
       "    <script src=\"//use.typekit.net/kpd5yrh.js\"></script>\n",
       "    <script>try{Typekit.load();}catch(e){}</script>\n",
       "    <script>\n",
       "      (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){\n",
       "      (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),\n",
       "      m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)\n",
       "      })(window,document,'script','//www.google-analytics.com/analytics.js','ga');\n",
       "\n",
       "      ga('create', 'UA-52617120-1', 'auto', {'storage': 'none'});\n",
       "      ga('set', 'anonymizeIp', true);\n",
       "      ga('send', 'pageview');\n",
       "\n",
       "    </script>\n",
       "    <script>console.log('Welcome to Project Jupyter! Explore the various tools available and their corresponding documentation. If you are interested in contributing to the platform, please visit the community resources section at https://jupyter.org/community.html.')</script>\n",
       "</head>\n",
       "\n",
       "\n",
       "  <body>\n",
       "\n",
       "    <nav class=\"navbar navbar-fixed-top\" role=\"navigation\" id=\"navbar\" >\n",
       "        <div class=\"navbar-inner\">\n",
       "        <div class=\"container\">\n",
       "            <!-- Brand and toggle get grouped for better mobile display -->\n",
       "            <div class=\"navbar-header\">\n",
       "                <button type=\"button\" class=\"navbar-toggle\" data-toggle=\"collapse\" data-target=\"#bs-example-navbar-collapse-1\">\n",
       "                    <span class=\"sr-only\">Toggle navigation</span>\n",
       "                    \n",
       "                      <span class=\"icon-bar white-icon-bar\"></span>\n",
       "                      <span class=\"icon-bar white-icon-bar\"></span>\n",
       "                      <span class=\"icon-bar white-icon-bar\"></span>\n",
       "                    \n",
       "                </button>\n",
       "                <a class=\"navbar-brand\" href=\"/index.html\">\n",
       "                    <img id=\"jupyter-nav-logo\" class=\"navbar-logo\" src=\"/assets/nav_logo.svg\" alt=\"Jupyter logo\">\n",
       "                </a>\n",
       "            </div>\n",
       "            <div class=\"blur\"></div>\n",
       "            <!-- Collect the nav links, forms, and other content for toggling -->\n",
       "            <div class=\"collapse navbar-collapse\" id=\"bs-example-navbar-collapse-1\">\n",
       "                <ul class=\"nav navbar-nav navbar-right\">\n",
       "                               <li>\n",
       "                                 <a href=\"/install\" class=\"tab\">Install</a>\n",
       "                               </li>\n",
       "                               <li>\n",
       "                                   <a href=\"/about\" class=\"tab\">About Us</a>\n",
       "                               </li>\n",
       "                               <li>\n",
       "                                 <a href=\"/community\" class=\"tab\">Community</a>\n",
       "                               </li>\n",
       "                               <li>\n",
       "                                   <a href=\"/documentation\" class=\"tab\">Documentation</a>\n",
       "                               </li>\n",
       "                               <li>\n",
       "                                 <a href=\"https://nbviewer.jupyter.org\" class=\"tab\" target=\"_blank\">NBViewer</a>\n",
       "                               </li>\n",
       "                               <li>\n",
       "                                   <a href=\"/hub\" class=\"tab\">JupyterHub</a>\n",
       "                               </li>\n",
       "                               <li>\n",
       "                                   <a href=\"/widgets\" class=\"tab\">Widgets</a>\n",
       "                               </li>\n",
       "                               <li>\n",
       "                                 <a href=\"https://blog.jupyter.org\" class=\"tab\" target=\"_blank\">Blog</a>\n",
       "                               </li>\n",
       "                </ul>\n",
       "            </div>\n",
       "            <!-- /.navbar-collapse -->\n",
       "        </div>\n",
       "        <!-- /.container -->\n",
       "    </div>\n",
       "</nav>\n",
       "\n",
       "    <section>\n",
       "    <section>\n",
       "<div class=\"jumbotron con-jumbotron\">\t\n",
       "        <div class=\"row\">\t\n",
       "            <div class=\"col-md-12\">\t\n",
       "                <div class=\"con-container img-responsive\">\t\n",
       "                    <div class=\"col-lg-8 con-content md-offset-2\">\t\n",
       "                        <div class=\"con-date\">October 5-17th</div>\t\n",
       "                        <img class=\"con-title img-responsive\" srcset=\"assets/jupytercon-logo.png, assets/jupytercon-logo-white.png\" />\t\n",
       "                        <h2 class=\"con-subtitle\">JupyterCon 2020 Call For Proposals</h2>\n",
       "                        <div class=\"con-copy\">\t\n",
       "                            Due July 20th.\n",
       "                        </div>\t\n",
       "                        <a class=\"orange-button con-button\" href=\"https://jupytercon.com/talk-poster-cfp/\" target=\"_blank\">Learn more</a>\t\n",
       "                    </div>\t\n",
       "                </div>\t\n",
       "            </div>\t\n",
       "        </div>\t\n",
       "    </div>\n",
       "    <div class=\"jumbotron\">\n",
       "        <div class=\"row\">\n",
       "            <div class=\"col-md-12\">\n",
       "                <div class=\"img-container\">\n",
       "                    <span><img class=\"img-responsive fade-in one\" src=\"assets/circle1.svg\" alt=\"circle of programming language icons\"></span>\n",
       "                </div>\n",
       "                <div class=\"img-container\">\n",
       "                    <span><img class=\"img-responsive fade-in two\" src=\"assets/circle2.svg\" alt=\"circle of programming language icons\"></span>\n",
       "                </div>\n",
       "                <div class=\"img-container\">\n",
       "                    <span><img class=\"img-responsive fade-in three\" src=\"assets/circle3.svg\" alt=\"circle of programming language icons\"></span>\n",
       "                </div>\n",
       "                <div class=\"img-container\">\n",
       "                    <img class=\"img-responsive\" id=\"main-logo\" src=\"assets/main-logo.svg\" alt=\"jupyter logo\">\n",
       "                </div>\n",
       "                <img class=\"img-responsive\" src=\"assets/white-background.svg\" alt=\"white background\">\n",
       "            </div>\n",
       "        </div>\n",
       "    </div>\n",
       "    <div class=\"container\">\n",
       "        <div class=\"row\">\n",
       "            <div class=\"col-md-12\">\n",
       "                <div class=\"col-md-8 col-md-offset-2\">\n",
       "                <h4 class=\"jumbotron-text\">Project Jupyter exists to develop open-source software, open-standards,\n",
       "                    and services for interactive computing across dozens of programming languages.</h4>\n",
       "                </div>\n",
       "             </div>\n",
       "        </div>\n",
       "    </div>\n",
       "</section>\n",
       "<section id=\"jupyterlab\">\n",
       "    <div class=\"section-grey\">\n",
       "        <div class=\"lab-preview\">\n",
       "            <div class=\"container\">\n",
       "                <div class=\"row\">\n",
       "                    <div class=\"col-md-6\">\n",
       "                        <img class=\"img-responsive\" src=\"assets/labpreview.png\" id=\"portal\" alt=\"examples of jupyterlab workspaces in single document and multiple document workspaces\"/>\n",
       "                    </div>\n",
       "                    <div class=\"col-md-6 nb-highlight-text\">\n",
       "                        <h3>JupyterLab 1.0: Jupyter’s Next-Generation Notebook Interface</h3>\n",
       "                        <h4 class=\"nb-desc\">JupyterLab is a web-based interactive development environment for Jupyter notebooks, code, and data. JupyterLab is flexible: configure and arrange the user interface to support a wide range of workflows in data science, scientific computing, and machine learning. JupyterLab is extensible and modular: write plugins that add new components and integrate with existing ones.</h4>\n",
       "                        <div class=\"button-container\">\n",
       "                            <a class=\"orange-button\" href=\"try\">Try it in your browser</a>\n",
       "                            <a class=\"orange-button install-button\" href=\"install.html\">Install JupyterLab</a>\n",
       "                        </div>\n",
       "                    </div>\n",
       "                </div>\n",
       "            </div>\n",
       "        </div>\n",
       "    </div>\n",
       "</section>\n",
       "<section id=\"about-notebook\">\n",
       "    <div class=\"section-white\">\n",
       "        <div class=\"notebook-preview\">\n",
       "            <div class=\"container\">\n",
       "                <div class=\"row\">\n",
       "                    <div class=\"col-md-6\">\n",
       "                        <img class=\"img-responsive\" src=\"assets/jupyterpreview.png\" id=\"portal\" alt=\"example notebook of Lorenz differential equations\"/>\n",
       "                    </div>\n",
       "                    <div class=\"col-md-6 nb-highlight-text\">\n",
       "                        <h3>The Jupyter Notebook</h3>\n",
       "                        <h4 class=\"nb-desc\">The Jupyter Notebook is an open-source web application that allows you to create and share\n",
       "                            documents that contain live code, equations, visualizations and narrative text. Uses include: data cleaning\n",
       "                            and transformation, numerical simulation, statistical modeling, data visualization, machine learning, and much more.</h4>\n",
       "                        <div class=\"button-container\">\n",
       "                            <a class=\"orange-button\" href=\"try\">Try it in your browser</a>\n",
       "                            <a class=\"orange-button install-button\" href=\"install.html\">Install the Notebook</a>\n",
       "                        </div>\n",
       "                    </div>\n",
       "                </div>\n",
       "            </div>\n",
       "        </div>\n",
       "        <div class=\"notebook-features\">\n",
       "            <div class=\"container\">\n",
       "                <div class=\"row\">\n",
       "                    <div class=\"col-md-3 notebook-feature\">\n",
       "                        <img class=\"img-responsive language\" src=\"assets/language.svg\" id=\"feature\" alt=\"icon to represent a language\">\n",
       "                        <h4>Language of choice</h4>\n",
       "                        <p>Jupyter supports over 40 programming languages, including Python, R, Julia, and Scala.</p>\n",
       "                    </div>\n",
       "                    <div class=\"col-md-3 notebook-feature\">\n",
       "                        <img class=\"img-responsive share\" src=\"assets/share.svg\" alt=\"icon to represent sharing a notebook\">\n",
       "                        <h4>Share notebooks</h4>\n",
       "                        <p> Notebooks can be shared with others using email, Dropbox, GitHub and the <a href=\"https://nbviewer.jupyter.org/\">Jupyter Notebook Viewer</a>.</p>\n",
       "                    </div>\n",
       "                    <div class=\"col-md-3 notebook-feature\">\n",
       "                        <img class=\"img-responsive widget\" src=\"assets/widget.svg\" alt=\"icon to represent an interactive widget\">\n",
       "                        <h4>Interactive output</h4>\n",
       "                        <p>Your code can produce rich, interactive output: HTML, images, videos, LaTeX, and custom MIME types.</p>\n",
       "                    </div>\n",
       "                    <div class=\"col-md-3 notebook-feature\">\n",
       "                        <img class=\"img-responsive data\" src=\"assets/spark.svg\" alt=\"spark logo to represent big data\">\n",
       "                        <h4> Big data integration</h4>\n",
       "                        <p>Leverage big data tools, such as Apache Spark, from Python, R and Scala. Explore that same data with pandas, scikit-learn, ggplot2, TensorFlow.</p>\n",
       "                    </div>\n",
       "                </div>\n",
       "            </div>\n",
       "        </div>\n",
       "    </div>\n",
       "</section>\n",
       "<section id=\"jupyterhub\">\n",
       "    <div class=\"section-grey\">\n",
       "        <div class=\"container\">\n",
       "            <div class=\"row\">\n",
       "            <div>\n",
       "                <div class=\"hublogo col-md-offset-4 col-md-4 col-md-offset-4\">\n",
       "                    <img src=\"assets/hublogo.svg\" class=\"img-responsive\" style=\"margin:0 auto;\" id=\"hublogo\" alt=\"jupyterhub logo\">\n",
       "                </div>\n",
       "                <div class=\"col-md-offset-3 col-md-6 col-md-offset-3\" style=\"margin-bottom:-36px\">\n",
       "                    <p style=\"padding-top:24px; text-align: center\">A multi-user version of the notebook designed for companies, classrooms and research labs</p>\n",
       "                </div>\n",
       "            </div>\n",
       "            <div class=\"hubfeatures col-md-12\">\n",
       "                <div class=\"feature1 hubfeature col-md-3 authentication\">\n",
       "                    <img src=\"assets/authentication.svg\" class=\"img-responsive\" alt=\"authentication icon\">\n",
       "                    <h4>Pluggable authentication</h4>\n",
       "                    <p>Manage users and authentication with PAM, OAuth or integrate with your own directory service system.</p>\n",
       "                </div>\n",
       "                <div class=\"feature-middle hubfeature col-md-3\">\n",
       "                    <img src=\"assets/central.svg\" class=\"img-responsive\" alt=\"icon to represent centralized deployment\">\n",
       "                    <h4>Centralized deployment</h4>\n",
       "                    <p>Deploy the Jupyter Notebook to thousands of users in your organization on centralized infrastructure on- or off-site.</p>\n",
       "                </div>\n",
       "                <div class=\"feature-middle hubfeature col-md-3\">\n",
       "                    <img src=\"assets/container.svg\" class=\"img-responsive\" alt=\"container icon\">\n",
       "                    <h4>Container friendly</h4>\n",
       "                    <p>Use Docker and Kubernetes to scale your deployment, isolate user processes, and simplify software installation.</p>\n",
       "                </div>\n",
       "                <div class=\"feature4 hubfeature col-md-3\">\n",
       "                    <img src=\"assets/data.svg\" class=\"img-responsive\" id=\"data\" alt=\"icon to represent data\">\n",
       "                    <h4>Code meets data</h4>\n",
       "                    <p>Deploy the Notebook next to your data to provide unified software management and data access within your organization.</p>\n",
       "                </div>\n",
       "            </div>\n",
       "            <div class=\"col-md-12\" style=\"text-align: center; margin-top: 24px;\">\n",
       "                <a class=\"orange-button\" href=\"https://github.com/jupyter/jupyterhub\">Learn more about JupyterHub</a>\n",
       "            </div>\n",
       "            </div>\n",
       "        </div>\n",
       "    </div>\n",
       "</section>\n",
       "<section id=\"voila\">\n",
       "    <div class=\"section-white\">\n",
       "        <div class=\"voila-preview\">\n",
       "            <div class=\"container\">\n",
       "                <div class=\"row\">\n",
       "                    <div class=\"col-md-6\">\n",
       "                        <img class=\"img-responsive\" src=\"assets/voilapreview.png\" id=\"portal\" alt=\"examples of Voilà dashboards\"/>\n",
       "                    </div>\n",
       "                    <div class=\"col-md-6 nb-highlight-text\">\n",
       "                        <h3>Communicate your results with Voilà.</h3>\n",
       "                        <h4 class=\"nb-desc\">\n",
       "                            Voilà helps you communicate insights, by transforming a Jupyter Notebook into a stand-alone web application you can share. It gives you control over what your readers experience in a secure and customizable interactive dashboard.\n",
       "                        </h4>\n",
       "                        <div class=\"button-container\">\n",
       "                            <a class=\"orange-button install-button\" href=\"install.html\">Install Voilà</a>\n",
       "                        </div>\n",
       "                    </div>\n",
       "                </div>\n",
       "            </div>\n",
       "        </div>\n",
       "    </div>\n",
       "</section>\n",
       "<section id=\"current-use\">\n",
       "    <div class=\"section-grey\">\n",
       "        <div class=\"container\">\n",
       "            <div class=\"row\">\n",
       "                <h3 class=\"section-header\">Currently in use at</h3>\n",
       "                <ul class=\"companies\">\n",
       "                    <li class=\"col\"><a href=\"https://www.google.com\"><img src=\"assets/google.svg\" class=\"greydout\" alt=\"Google\"></a></li>\n",
       "                    <li class=\"col\"><a href=\"https://www.microsoft.com\"><img src=\"assets/Microsoft.svg\" class=\"greydout\" alt=\"Microsoft\"></a></li>\n",
       "                    <li class=\"col\"><a href=\"https://www.ibm.com\"><img src=\"assets/IBM.svg\" class=\"greydout\" alt=\"IBM\"></a></li>\n",
       "                    <li class=\"col\"><a href=\"https://www.bloomberg.com\"><img src=\"assets/bloomberg.svg\" class=\"greydout\" alt=\"Bloomberg\"></a></li>\n",
       "                    <li class=\"col\"><a href=\"https://www.oreilly.com\"><img src=\"assets/oreilly.svg\" class=\"greydout\" alt=\"OReilly\"></a></li>\n",
       "                    <li class=\"col\"><a href=\"https://www.anaconda.com/\"><img src=\"assets/anaconda.svg\" class=\"greydout\" alt=\"Anaconda\"></a></li>\n",
       "                    <li class=\"col\"><a href=\"https://developer.rackspace.com\"><img src=\"assets/rackspace.svg\" class=\"greydout\" alt=\"Rackspace\"></a></li>\n",
       "                    <li class=\"col\"><a href=\"https://www.soundcloud.com\"><img src=\"assets/soundcloud.svg\" class=\"greydout\" alt=\"SoundCloud\"></a></li>\n",
       "                    <li class=\"col\"><a href=\"https://www.quantopian.com/\"><img src=\"assets/quantopian.svg\" class=\"greydout\" alt=\"Quantopian\"></a></li>\n",
       "                    <li class=\"col\"><a href=\"https://www.netapp.com/us/\"><img src=\"assets/netapp.svg\" class=\"greydout\" alt=\"NetApp\"></a></li>\n",
       "                    <li class=\"col\"><a href=\"https://software-carpentry.org\"><img src=\"assets/carpentry.svg\" class=\"greydout\" alt=\"Carpentry\"></a></li>\n",
       "                    <li class=\"col\"><a href=\"https://www.janelia.org/\"><img src=\"assets/janelia.svg\" class=\"greydout\" alt=\"Janelia\"></a></li>\n",
       "                    <li class=\"col\"><a href=\"https://codeneuro.org/\"><img src=\"assets/codeneuro.svg\" class=\"greydout\" alt=\"CodeNeuro\"></a></li>\n",
       "                    <li class=\"col\"><a href=\"\"><img src=\"assets/nsite.svg\" class=\"greydout\" alt=\"NSite\"></a></li>\n",
       "                    <li class=\"col\"><a href=\"https://cocalc.com/\"><img src=\"assets/cocalc.svg\" class=\"greydout\" alt=\"CoCalc\"></a></li>\n",
       "                    <li class=\"col\"><a href=\"https://www.brynmawr.edu\"><img src=\"assets/brynmawr.svg\" class=\"greydout\" alt=\"BrynMawr\"></a></li>\n",
       "                    <li class=\"col\"><a href=\"https://www.calpoly.edu\"><img src=\"assets/calpoly.svg\" class=\"greydout\" alt=\"CalPoly\"></a></li>\n",
       "                    <li class=\"col\"><a href=\"https://www.berkeley.edu\"><img src=\"assets/Berkeley.svg\" class=\"greydout\" alt=\"Berkeley\"></a></li>\n",
       "                    <li class=\"col\"><a href=\"https://www.sheffield.ac.uk/\"><img src=\"assets/sheffield.svg\" class=\"greydout\" alt=\"Sheffield\"></a></li>\n",
       "                    <li class=\"col\"><a href=\"https://www.gwu.edu/\"><img src=\"assets/washington.svg\" class=\"greydout\" alt=\"Washington\"></a></li>\n",
       "                    <li class=\"col\"><a href=\"https://www.clemson.edu/\"><img src=\"assets/clemson.svg\" class=\"greydout\" alt=\"Clemson\"></a></li>\n",
       "                    <li class=\"col\"><a href=\"https://msu.edu/\"><img src=\"assets/michigan-state.svg\" class=\"greydout\" alt=\"Michigan State University\"></a></li>\n",
       "                    <li class=\"col\"><a href=\"https://www.northwestern.edu/\"><img src=\"assets/northwestern.svg\" class=\"greydout\" alt=\"Northwestern University\"></a></li>\n",
       "                    <li class=\"col\"><a href=\"https://www.nyu.edu\"><img src=\"assets/NYU.svg\" class=\"greydout\" alt=\"NYU\"></a></li>\n",
       "                    <li class=\"col\"><a href=\"https://www.nasa.gov\"><img src=\"assets/NASA.svg\" class=\"greydout\" alt=\"NASA\"></a></li>\n",
       "                    <li class=\"col\"><a href=\"https://ayasdi.com\"><img src=\"assets/ayasdi.svg\" class=\"greydout\" alt=\"Ayasdi\"></a></li>\n",
       "                    <li class=\"col\"><a href=\"https://www.thedataincubator.com\"><img src=\"assets/tdi.svg\" class=\"greydout\" alt=\"The Data Incubator\"></a></li>\n",
       "                    <li class=\"col\"><a href=\"https://quantstack.net\"><img src=\"assets/quantstack.svg\" class=\"greydout\" alt=\"QuantStack\"></a></li>\n",
       "                </ul>\n",
       "            </div>\n",
       "        </div>\n",
       "    </div>\n",
       "</section>\n",
       "<section id=\"architecture\">\n",
       "    <div class=\"section-white\">\n",
       "        <div class=\"container\">\n",
       "            <div class=\"architecturedescription\">\n",
       "                <img src=\"assets/architectureicon2.svg\" class=\"section-icon\" alt=\"icon to represent jupyter architecture\">\n",
       "                <h3 class=\"section-header\">Open Standards for Interactive Computing</h3>\n",
       "                <h4 class=\"support-paragraph\">The Jupyter Notebook is based on a set of open standards for interactive computing. Think HTML and CSS for interactive computing on the web. These open standards can be leveraged by third party developers to build customized applications with embedded interactive computing.</h4>\n",
       "            </div>\n",
       "            <div class=\"architecturecontent row\">\n",
       "                <div class=\"col-sm-6 col-md-4\">\n",
       "                    <div class=\"card\">\n",
       "                        <div class=\"arcfeature\" id=\"arcfeature1\">\n",
       "                            <div class=\"front front1\">\n",
       "                                <img src=\"assets/notebook-format.svg\" alt=\"icon to represent notebook format\">\n",
       "                                <a class=\"arc-button bt\">Learn more</a>\n",
       "                            </div>\n",
       "                            <div class=\"front front1 flipped content-back\">\n",
       "                                <h3>The Notebook Document Format</h3>\n",
       "                                <p id=\"document\">Jupyter Notebooks are an open document format based on JSON. They contain a complete record of the user's sessions and include code, narrative text, equations and rich output. </p>\n",
       "                                <a class=\"arc-button bt\">Go back</a>\n",
       "                            </div>\n",
       "                        </div>\n",
       "                    </div>\n",
       "                </div>\n",
       "                <div class=\"col-sm-6 col-md-4\">\n",
       "                    <div class=\"card\">\n",
       "                        <div class=\"arcfeature\" id=\"protocol\">\n",
       "                            <div class=\"front front2\">\n",
       "                                <img src=\"assets/protocol.svg\" alt=\"icon to represent a protocol\">\n",
       "                                <a class=\"arc-button bt1\">Learn more</a>\n",
       "                            </div>\n",
       "                            <div class=\"front front2 flipped content-back\">\n",
       "                                <h3>Interactive Computing Protocol</h3>\n",
       "                                <p>The Notebook communicates with computational Kernels using the Interactive Computing Protocol, an open network protocol based on JSON data over ZMQ and WebSockets.</p>\n",
       "                                <a class=\"arc-button bt1\">Go back</a>\n",
       "                            </div>\n",
       "                        </div>\n",
       "                    </div>\n",
       "                </div>\n",
       "                <div class=\"col-sm-12 col-md-4\">\n",
       "                    <div class=\"card\">\n",
       "                        <div class=\"arcfeature\" id=\"kernel\">\n",
       "                            <div class=\"front front3\">\n",
       "                                <img src=\"assets/kernel.svg\" alt=\"icon that looks like a terminal represents kernel\">\n",
       "                                <a class=\"arc-button bt2\">Learn more</a>\n",
       "                            </div>\n",
       "                            <div class=\"front front3 flipped content-back\">\n",
       "                                <h3>Kernels</h3>\n",
       "                                <p id=\"block3\">Kernels are processes that run interactive code in a particular programming language and return output to the user. Kernels also respond to tab completion and introspection requests.</p>\n",
       "                                <a class=\"arc-button bt2\">Go back</a>\n",
       "                            </div>\n",
       "                        </div>\n",
       "                    </div>\n",
       "                </div>\n",
       "            </div>\n",
       "        </div>\n",
       "    </div>\n",
       "</section>\n",
       "\n",
       "    </section>\n",
       "\n",
       "    <footer>\n",
       "    <div class=\"footer\" role=\"navigation\">\n",
       "        <div class=\"container\">\n",
       "            <div class=\"navbar-text\">\n",
       "                <ul class=\"footer-text\">\n",
       "                    <li><a href=\"/index.html\">Home</a></li>\n",
       "                               <li><a href=\"/install\" target=\"_blank\">Install</a></li>\n",
       "                               <li><a href=\"/about\">About Us</a></li>\n",
       "                               <li><a href=\"/community\" target=\"_blank\">Community</a></li>\n",
       "                               <li><a href=\"/documentation\">Documentation</a></li>\n",
       "                               <li><a href=\"https://nbviewer.jupyter.org\" target=\"_blank\">NBViewer</a></li>\n",
       "                               <li><a href=\"/hub\">JupyterHub</a></li>\n",
       "                               <li><a href=\"/widgets\">Widgets</a></li>\n",
       "                               <li><a href=\"https://blog.jupyter.org\" target=\"_blank\">Blog</a></li>\n",
       "                    <!--2020-07-15 17:09:48 +0000-->\n",
       "                </ul>\n",
       "                <p>Copyright © 2020  Project Jupyter –\n",
       "                Last updated Wed, Jul 15, 2020</p>\n",
       "                <p>The Jupyter Trademark is registered with the U.S. Patent &amp; Trademark Office.</p>\n",
       "            </div>\n",
       "            <div class=\"follow\">\n",
       "                <a href=\"https://github.com/jupyter\" class=\"button github-button\" aria-label=\"Follow @jupyter on GitHub\"><svg version=\"1.1\" width=\"14\" height=\"14\" viewBox=\"0 0 16 16\" class=\"octicon octicon-mark-github\" aria-hidden=\"true\"><path fill-rule=\"evenodd\" d=\"M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0 0 16 8c0-4.42-3.58-8-8-8z\"></path></svg> <span>Follow @jupyter</span></a>\n",
       "                <br><a href=\"https://twitter.com/ProjectJupyter\" class=\"twitter-follow-button\" data-show-count=\"false\">Follow @ProjectJupyter</a>\n",
       "            </div>\n",
       "        </div>\n",
       "    </div>\n",
       "</footer>\n",
       "\n",
       "    <!-- jQuery -->\n",
       "    <script src=\"/js/jquery.js\"></script>\n",
       "\n",
       "    <!-- Bootstrap Core JavaScript -->\n",
       "    <script src=\"/js/bootstrap.min.js\"></script>\n",
       "    <script src=\"/js/architecture.js\"></script>\n",
       "    <script src=\"/js/simon.js\"></script>\n",
       "\n",
       "    <script>!function(d,s,id){var js,fjs=d.getElementsByTagName(s)[0],p=/^https:/.test(d.location)?'https':'http';if(!d.getElementById(id)){js=d.createElement(s);js.id=id;js.src=p+'://platform.twitter.com/widgets.js';fjs.parentNode.insertBefore(js,fjs);}}(document, 'script', 'twitter-wjs');</script>\n",
       "\n",
       "\n",
       "  </body>\n",
       "\n",
       "</html>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Use IPython to display the rendered HTML - note that no JS is executed and files with relatives paths aren't loaded\n",
    "from IPython.display import HTML\n",
    "HTML(page_html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Compare this output to the live website, which we can open using an iFrame\n",
    "from IPython.display import IFrame\n",
    "IFrame(src=url, height=800, width=800)\n",
    "\n",
    "# NOTE: After doing this, it's a good idea to close the iFrame so that it doesn't eat memory.\n",
    "# We do this by clicking the Current Outputs -> Clear option from the Cell menu."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [200]>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Shorthand method for making requests - this doesn't allow header customisation but involves less code\n",
    "url = 'http://www.python.org/'\n",
    "page = requests.get(url)\n",
    "page"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://www.python.org/'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Show the URL attribute of the response - note that it has been redirected to HTTPS\n",
    "page.url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<Response [301]>]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Show the redirect history\n",
    "page.history"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parsing HTML with Python\n",
    "We will scrape the contents of the central bank interest rates Wikipedia page. Essentially, this will involve the following steps:\n",
    "- First, view the page in a browser. Use Chrome Developer Tools to identify the elements of interest that we want to scrape for data.\n",
    "- Get the page content using the requests library.\n",
    "- Parse the content using BeautifulSoup.\n",
    "- Display the attributes and methods available for the BeautifulSoup object and the tags we want to parse. In addition to the tables containing data, we might also want to get lists of image URLs, headings, etc.\n",
    "\n",
    "The URL of the page we will scrape is https://en.wikipedia.org/wiki/List_of_countries_by_central_bank_interest_rates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the page HTML\n",
    "url = 'https://en.wikipedia.org/wiki/List_of_countries_by_central_bank_interest_rates'\n",
    "page = requests.get(url)\n",
    "soup = BeautifulSoup(page.content, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\u001b[0;33mproperty:\u001b[0m\n",
       "    \u001b[0;36mASCII_SPACES\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36mDEFAULT_BUILDER_FEATURES\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36mNO_PARSER_SPECIFIED_WARNING\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36mROOT_TAG_NAME\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36m_most_recent_element\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36m_namespaces\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36mattrs\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36mbuilder\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36mcan_be_empty_element\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36mcdata_list_attributes\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36mcontains_replacement_characters\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36mcontents\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36mcurrentTag\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36mcurrent_data\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36mdeclared_html_encoding\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36melement_classes\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36mhidden\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36mis_xml\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36mknown_xml\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36mmarkup\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36mname\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36mnamespace\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36mnext_element\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36mnext_sibling\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36moriginal_encoding\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36mparent\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36mparse_only\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36mprefix\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36mpreserve_whitespace_tag_stack\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36mpreserve_whitespace_tags\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36mprevious_element\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36mprevious_sibling\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36mstring_container_stack\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36mtagStack\u001b[0m\n",
       "\u001b[0;33mspecial attribute:\u001b[0m\n",
       "    \u001b[0;36m__class__\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36m__dict__\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36m__doc__\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36m__module__\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36m__weakref__\u001b[0m\n",
       "\u001b[0;33mabstract class:\u001b[0m\n",
       "    \u001b[0;36m__subclasshook__\u001b[0m\n",
       "\u001b[0;33mobject customization:\u001b[0m\n",
       "    \u001b[0;36m__bool__\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36m__format__\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36m__hash__\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36m__init__\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36m__new__\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36m__repr__\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36m__sizeof__\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36m__str__\u001b[0m\n",
       "\u001b[0;33mrich comparison:\u001b[0m\n",
       "    \u001b[0;36m__eq__\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36m__ge__\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36m__gt__\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36m__le__\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36m__lt__\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36m__ne__\u001b[0m\n",
       "\u001b[0;33mattribute access:\u001b[0m\n",
       "    \u001b[0;36m__delattr__\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36m__dir__\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36m__getattr__\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36m__getattribute__\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36m__setattr__\u001b[0m\n",
       "\u001b[0;33mclass customization:\u001b[0m\n",
       "    \u001b[0;36m__init_subclass__\u001b[0m\n",
       "\u001b[0;33mcontainer:\u001b[0m\n",
       "    \u001b[0;36m__contains__\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36m__delitem__\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36m__getitem__\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36m__iter__\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36m__len__\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36m__setitem__\u001b[0m\n",
       "\u001b[0;33mcopy:\u001b[0m\n",
       "    \u001b[0;36m__copy__\u001b[0m\n",
       "\u001b[0;33mpickle:\u001b[0m\n",
       "    \u001b[0;36m__getstate__\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36m__reduce__\u001b[0m\u001b[1;30m, \u001b[0m\u001b[0;36m__reduce_ex__\u001b[0m\n",
       "\u001b[0;33mdescriptor:\u001b[0m\n",
       "    \u001b[0;36m_check_markup_is_url\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mclass classmethod with getter, classmethod(function) -> method\u001b[0m\n",
       "    \u001b[0;36m_decode_markup\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mclass classmethod with getter, classmethod(function) -> method\u001b[0m\n",
       "    \u001b[0;36m_is_xml\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30m@property with getter, Is this element part of an XML tree or an HTML tree?\u001b[0m\n",
       "    \u001b[0;36mchildren\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30m@property with getter, Iterate over all direct children of this PageElement.\u001b[0m\n",
       "    \u001b[0;36mdecomposed\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30m@property with getter, Check whether a PageElement has been decomposed.\u001b[0m\n",
       "    \u001b[0;36mdescendants\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30m@property with getter, Iterate over all children of this PageElement in a\u001b[0m\n",
       "    \u001b[0;36misSelfClosing\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30m@property with getter, Is this tag an empty-element tag? (aka a self-closing tag)\u001b[0m\n",
       "    \u001b[0;36mis_empty_element\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30m@property with getter, Is this tag an empty-element tag? (aka a self-closing tag)\u001b[0m\n",
       "    \u001b[0;36mnext\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30m@property with getter, The PageElement, if any, that was parsed just after this one.\u001b[0m\n",
       "    \u001b[0;36mnextSibling\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30m@property with getter, setter\u001b[0m\n",
       "    \u001b[0;36mnext_elements\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30m@property with getter, All PageElements that were parsed after this one.\u001b[0m\n",
       "    \u001b[0;36mnext_siblings\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30m@property with getter, All PageElements that are siblings of this one but were parsed\u001b[0m\n",
       "    \u001b[0;36mparents\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30m@property with getter, All PageElements that are parents of this PageElement.\u001b[0m\n",
       "    \u001b[0;36mparserClass\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30m@property with getter, setter\u001b[0m\n",
       "    \u001b[0;36mprevious\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30m@property with getter, The PageElement, if any, that was parsed just before this one.\u001b[0m\n",
       "    \u001b[0;36mpreviousSibling\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30m@property with getter, setter\u001b[0m\n",
       "    \u001b[0;36mprevious_elements\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30m@property with getter, All PageElements that were parsed before this one.\u001b[0m\n",
       "    \u001b[0;36mprevious_siblings\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30m@property with getter, All PageElements that are siblings of this one but were parsed\u001b[0m\n",
       "    \u001b[0;36mstring\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30m@property with getter, setter, Convenience property to get the single string within this\u001b[0m\n",
       "    \u001b[0;36mstrings\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30m@property with getter, Yield all strings of certain classes, possibly stripping them.\u001b[0m\n",
       "    \u001b[0;36mstripped_strings\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30m@property with getter, Yield all strings in the document, stripping them first.\u001b[0m\n",
       "    \u001b[0;36mtext\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30m@property with getter, Get all child strings, concatenated using the given separator.\u001b[0m\n",
       "\u001b[0;33mclass:\u001b[0m\n",
       "    \u001b[0;36mparser_class\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mA data structure representing a parsed HTML or XML document.\u001b[0m\n",
       "\u001b[0;33mfunction:\u001b[0m\n",
       "    \u001b[0;36m__unicode__\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mRenders this PageElement as a Unicode string.\u001b[0m\n",
       "    \u001b[0;36m_all_strings\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mYield all strings of certain classes, possibly stripping them.\u001b[0m\n",
       "    \u001b[0;36m_feed\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mInternal method that parses previously set markup, creating a large\u001b[0m\n",
       "    \u001b[0;36m_find_all\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mIterates over a generator looking for things that match.\u001b[0m\n",
       "    \u001b[0;36m_find_one\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30m\u001b[0m\n",
       "    \u001b[0;36m_lastRecursiveChild\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mFinds the last element beneath this object to be parsed.\u001b[0m\n",
       "    \u001b[0;36m_last_descendant\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mFinds the last element beneath this object to be parsed.\u001b[0m\n",
       "    \u001b[0;36m_linkage_fixer\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mMake sure linkage of this fragment is sound.\u001b[0m\n",
       "    \u001b[0;36m_popToTag\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mPops the tag stack up to and including the most recent\u001b[0m\n",
       "    \u001b[0;36m_should_pretty_print\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mShould this tag be pretty-printed?\u001b[0m\n",
       "    \u001b[0;36mappend\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mAppends the given PageElement to the contents of this one.\u001b[0m\n",
       "    \u001b[0;36mchildGenerator\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mDeprecated generator.\u001b[0m\n",
       "    \u001b[0;36mclear\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mWipe out all children of this PageElement by calling extract()\u001b[0m\n",
       "    \u001b[0;36mdecode\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mReturns a string or Unicode representation of the parse tree\u001b[0m\n",
       "    \u001b[0;36mdecode_contents\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mRenders the contents of this tag as a Unicode string.\u001b[0m\n",
       "    \u001b[0;36mdecompose\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mRecursively destroys this PageElement and its children.\u001b[0m\n",
       "    \u001b[0;36mencode\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mRender a bytestring representation of this PageElement and its\u001b[0m\n",
       "    \u001b[0;36mencode_contents\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mRenders the contents of this PageElement as a bytestring.\u001b[0m\n",
       "    \u001b[0;36mendData\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mMethod called by the TreeBuilder when the end of a data segment\u001b[0m\n",
       "    \u001b[0;36mextend\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mAppends the given PageElements to this one's contents.\u001b[0m\n",
       "    \u001b[0;36mextract\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mDestructively rips this element out of the tree.\u001b[0m\n",
       "    \u001b[0;36mfetchNextSiblings\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mFind all siblings of this PageElement that match the given criteria\u001b[0m\n",
       "    \u001b[0;36mfetchParents\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mFind all parents of this PageElement that match the given criteria.\u001b[0m\n",
       "    \u001b[0;36mfetchPrevious\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mLook backwards in the document from this PageElement and find all\u001b[0m\n",
       "    \u001b[0;36mfetchPreviousSiblings\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mReturns all siblings to this PageElement that match the\u001b[0m\n",
       "    \u001b[0;36mfind\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mLook in the children of this PageElement and find the first\u001b[0m\n",
       "    \u001b[0;36mfindAll\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mLook in the children of this PageElement and find all\u001b[0m\n",
       "    \u001b[0;36mfindAllNext\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mFind all PageElements that match the given criteria and appear\u001b[0m\n",
       "    \u001b[0;36mfindAllPrevious\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mLook backwards in the document from this PageElement and find all\u001b[0m\n",
       "    \u001b[0;36mfindChild\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mLook in the children of this PageElement and find the first\u001b[0m\n",
       "    \u001b[0;36mfindChildren\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mLook in the children of this PageElement and find all\u001b[0m\n",
       "    \u001b[0;36mfindNext\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mFind the first PageElement that matches the given criteria and\u001b[0m\n",
       "    \u001b[0;36mfindNextSibling\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mFind the closest sibling to this PageElement that matches the\u001b[0m\n",
       "    \u001b[0;36mfindNextSiblings\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mFind all siblings of this PageElement that match the given criteria\u001b[0m\n",
       "    \u001b[0;36mfindParent\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mFind the closest parent of this PageElement that matches the given\u001b[0m\n",
       "    \u001b[0;36mfindParents\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mFind all parents of this PageElement that match the given criteria.\u001b[0m\n",
       "    \u001b[0;36mfindPrevious\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mLook backwards in the document from this PageElement and find the\u001b[0m\n",
       "    \u001b[0;36mfindPreviousSibling\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mReturns the closest sibling to this PageElement that matches the\u001b[0m\n",
       "    \u001b[0;36mfindPreviousSiblings\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mReturns all siblings to this PageElement that match the\u001b[0m\n",
       "    \u001b[0;36mfind_all\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mLook in the children of this PageElement and find all\u001b[0m\n",
       "    \u001b[0;36mfind_all_next\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mFind all PageElements that match the given criteria and appear\u001b[0m\n",
       "    \u001b[0;36mfind_all_previous\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mLook backwards in the document from this PageElement and find all\u001b[0m\n",
       "    \u001b[0;36mfind_next\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mFind the first PageElement that matches the given criteria and\u001b[0m\n",
       "    \u001b[0;36mfind_next_sibling\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mFind the closest sibling to this PageElement that matches the\u001b[0m\n",
       "    \u001b[0;36mfind_next_siblings\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mFind all siblings of this PageElement that match the given criteria\u001b[0m\n",
       "    \u001b[0;36mfind_parent\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mFind the closest parent of this PageElement that matches the given\u001b[0m\n",
       "    \u001b[0;36mfind_parents\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mFind all parents of this PageElement that match the given criteria.\u001b[0m\n",
       "    \u001b[0;36mfind_previous\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mLook backwards in the document from this PageElement and find the\u001b[0m\n",
       "    \u001b[0;36mfind_previous_sibling\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mReturns the closest sibling to this PageElement that matches the\u001b[0m\n",
       "    \u001b[0;36mfind_previous_siblings\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mReturns all siblings to this PageElement that match the\u001b[0m\n",
       "    \u001b[0;36mformat_string\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mFormat the given string using the given formatter.\u001b[0m\n",
       "    \u001b[0;36mformatter_for_name\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mLook up or create a Formatter for the given identifier,\u001b[0m\n",
       "    \u001b[0;36mget\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mReturns the value of the 'key' attribute for the tag, or\u001b[0m\n",
       "    \u001b[0;36mgetText\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mGet all child strings, concatenated using the given separator.\u001b[0m\n",
       "    \u001b[0;36mget_attribute_list\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mThe same as get(), but always returns a list.\u001b[0m\n",
       "    \u001b[0;36mget_text\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mGet all child strings, concatenated using the given separator.\u001b[0m\n",
       "    \u001b[0;36mhandle_data\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mCalled by the tree builder when a chunk of textual data is encountered.\u001b[0m\n",
       "    \u001b[0;36mhandle_endtag\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mCalled by the tree builder when an ending tag is encountered.\u001b[0m\n",
       "    \u001b[0;36mhandle_starttag\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mCalled by the tree builder when a new tag is encountered.\u001b[0m\n",
       "    \u001b[0;36mhas_attr\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mDoes this PageElement have an attribute with the given name?\u001b[0m\n",
       "    \u001b[0;36mhas_key\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mDeprecated method. This was kind of misleading because has_key()\u001b[0m\n",
       "    \u001b[0;36mindex\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mFind the index of a child by identity, not value.\u001b[0m\n",
       "    \u001b[0;36minsert\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mInsert a new PageElement in the list of this PageElement's children.\u001b[0m\n",
       "    \u001b[0;36minsert_after\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mThis method is part of the PageElement API, but `BeautifulSoup` doesn't implement\u001b[0m\n",
       "    \u001b[0;36minsert_before\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mThis method is part of the PageElement API, but `BeautifulSoup` doesn't implement\u001b[0m\n",
       "    \u001b[0;36mnew_string\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mCreate a new NavigableString associated with this BeautifulSoup\u001b[0m\n",
       "    \u001b[0;36mnew_tag\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mCreate a new Tag associated with this BeautifulSoup object.\u001b[0m\n",
       "    \u001b[0;36mnextGenerator\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30m\u001b[0m\n",
       "    \u001b[0;36mnextSiblingGenerator\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30m\u001b[0m\n",
       "    \u001b[0;36mobject_was_parsed\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mMethod called by the TreeBuilder to integrate an object into the parse tree.\u001b[0m\n",
       "    \u001b[0;36mparentGenerator\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30m\u001b[0m\n",
       "    \u001b[0;36mpopTag\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mInternal method called by _popToTag when a tag is closed.\u001b[0m\n",
       "    \u001b[0;36mprettify\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mPretty-print this PageElement as a string.\u001b[0m\n",
       "    \u001b[0;36mpreviousGenerator\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30m\u001b[0m\n",
       "    \u001b[0;36mpreviousSiblingGenerator\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30m\u001b[0m\n",
       "    \u001b[0;36mpushTag\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mInternal method called by handle_starttag when a tag is opened.\u001b[0m\n",
       "    \u001b[0;36mrecursiveChildGenerator\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mDeprecated generator.\u001b[0m\n",
       "    \u001b[0;36mrenderContents\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mDeprecated method for BS3 compatibility.\u001b[0m\n",
       "    \u001b[0;36mreplaceWith\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mReplace this PageElement with another one, keeping the rest of the\u001b[0m\n",
       "    \u001b[0;36mreplaceWithChildren\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mReplace this PageElement with its contents.\u001b[0m\n",
       "    \u001b[0;36mreplace_with\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mReplace this PageElement with another one, keeping the rest of the\u001b[0m\n",
       "    \u001b[0;36mreplace_with_children\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mReplace this PageElement with its contents.\u001b[0m\n",
       "    \u001b[0;36mreset\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mReset this object to a state as though it had never parsed any\u001b[0m\n",
       "    \u001b[0;36mselect\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mPerform a CSS selection operation on the current element.\u001b[0m\n",
       "    \u001b[0;36mselect_one\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mPerform a CSS selection operation on the current element.\u001b[0m\n",
       "    \u001b[0;36msetup\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mSets up the initial relations between this element and\u001b[0m\n",
       "    \u001b[0;36msmooth\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mSmooth out this element's children by consolidating consecutive\u001b[0m\n",
       "    \u001b[0;36mstring_container\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30m\u001b[0m\n",
       "    \u001b[0;36munwrap\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mReplace this PageElement with its contents.\u001b[0m\n",
       "    \u001b[0;36mwrap\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mWrap this PageElement inside another one.\u001b[0m\n",
       "\u001b[0;33mmagic:\u001b[0m\n",
       "    \u001b[0;36m__call__\u001b[0m\u001b[0;36m: \u001b[0m\u001b[1;30mCalling a Tag like a function is the same as calling its\u001b[0m"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Display the attributes and methods for the BS4 object\n",
    "try:\n",
    "    import pdir\n",
    "    dir = pdir\n",
    "except:\n",
    "    print('You need to install pdir with:\\npip install pdir2')\n",
    "dir(soup)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<h1 class=\"firstHeading\" id=\"firstHeading\" lang=\"en\">List of countries by central bank interest rates</h1>]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get the h1 heading for the page\n",
    "h1 = soup.find_all('h1')\n",
    "h1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 'firstHeading', 'class': ['firstHeading'], 'lang': 'en'}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Print the HTML element attributes using the .attrs property\n",
    "h1 = h1[0]\n",
    "h1.attrs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "List of countries by central bank interest rates\n"
     ]
    }
   ],
   "source": [
    "# Get the visible text by printing h1.text\n",
    "print(h1.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "93"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get all the images on the page\n",
    "imgs = soup.find_all('img')\n",
    "len(imgs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['//upload.wikimedia.org/wikipedia/commons/thumb/3/36/Flag_of_Albania.svg/21px-Flag_of_Albania.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/9/9d/Flag_of_Angola.svg/23px-Flag_of_Angola.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/1/1a/Flag_of_Argentina.svg/23px-Flag_of_Argentina.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/2/2f/Flag_of_Armenia.svg/23px-Flag_of_Armenia.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/8/88/Flag_of_Australia_%28converted%29.svg/23px-Flag_of_Australia_%28converted%29.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Flag_of_Azerbaijan.svg/23px-Flag_of_Azerbaijan.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/9/93/Flag_of_the_Bahamas.svg/23px-Flag_of_the_Bahamas.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/2/2c/Flag_of_Bahrain.svg/23px-Flag_of_Bahrain.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/f/f9/Flag_of_Bangladesh.svg/23px-Flag_of_Bangladesh.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/e/ef/Flag_of_Barbados.svg/23px-Flag_of_Barbados.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/8/85/Flag_of_Belarus.svg/23px-Flag_of_Belarus.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/f/fa/Flag_of_Botswana.svg/23px-Flag_of_Botswana.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/en/thumb/0/05/Flag_of_Brazil.svg/22px-Flag_of_Brazil.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/9/9a/Flag_of_Bulgaria.svg/23px-Flag_of_Bulgaria.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/d/d9/Flag_of_Canada_%28Pantone%29.svg/23px-Flag_of_Canada_%28Pantone%29.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/3/38/Flag_of_Cape_Verde.svg/23px-Flag_of_Cape_Verde.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/7/78/Flag_of_Chile.svg/23px-Flag_of_Chile.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/f/fa/Flag_of_the_People%27s_Republic_of_China.svg/23px-Flag_of_the_People%27s_Republic_of_China.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/c/cb/Flag_of_the_Czech_Republic.svg/23px-Flag_of_the_Czech_Republic.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/2/21/Flag_of_Colombia.svg/23px-Flag_of_Colombia.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/6/6f/Flag_of_the_Democratic_Republic_of_the_Congo.svg/20px-Flag_of_the_Democratic_Republic_of_the_Congo.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/1/1b/Flag_of_Croatia.svg/23px-Flag_of_Croatia.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/9/9c/Flag_of_Denmark.svg/20px-Flag_of_Denmark.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/9/9f/Flag_of_the_Dominican_Republic.svg/23px-Flag_of_the_Dominican_Republic.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/f/fe/Flag_of_Egypt.svg/23px-Flag_of_Egypt.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/b/b7/Flag_of_Europe.svg/23px-Flag_of_Europe.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/b/ba/Flag_of_Fiji.svg/23px-Flag_of_Fiji.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/7/77/Flag_of_The_Gambia.svg/23px-Flag_of_The_Gambia.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/0/0f/Flag_of_Georgia.svg/23px-Flag_of_Georgia.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/1/19/Flag_of_Ghana.svg/23px-Flag_of_Ghana.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/5/5b/Flag_of_Hong_Kong.svg/23px-Flag_of_Hong_Kong.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/c/c1/Flag_of_Hungary.svg/23px-Flag_of_Hungary.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/c/ce/Flag_of_Iceland.svg/21px-Flag_of_Iceland.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/en/thumb/4/41/Flag_of_India.svg/23px-Flag_of_India.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/9/9f/Flag_of_Indonesia.svg/23px-Flag_of_Indonesia.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/c/ca/Flag_of_Iran.svg/23px-Flag_of_Iran.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/d/d4/Flag_of_Israel.svg/21px-Flag_of_Israel.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/0/0a/Flag_of_Jamaica.svg/23px-Flag_of_Jamaica.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/en/thumb/9/9e/Flag_of_Japan.svg/23px-Flag_of_Japan.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/c/c0/Flag_of_Jordan.svg/23px-Flag_of_Jordan.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/d/d3/Flag_of_Kazakhstan.svg/23px-Flag_of_Kazakhstan.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/4/49/Flag_of_Kenya.svg/23px-Flag_of_Kenya.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/a/aa/Flag_of_Kuwait.svg/23px-Flag_of_Kuwait.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/c/c7/Flag_of_Kyrgyzstan.svg/23px-Flag_of_Kyrgyzstan.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/7/79/Flag_of_North_Macedonia.svg/23px-Flag_of_North_Macedonia.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/d/d1/Flag_of_Malawi.svg/23px-Flag_of_Malawi.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/6/66/Flag_of_Malaysia.svg/23px-Flag_of_Malaysia.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/7/77/Flag_of_Mauritius.svg/23px-Flag_of_Mauritius.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/f/fc/Flag_of_Mexico.svg/23px-Flag_of_Mexico.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/2/27/Flag_of_Moldova.svg/23px-Flag_of_Moldova.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/4/4c/Flag_of_Mongolia.svg/23px-Flag_of_Mongolia.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/2/2c/Flag_of_Morocco.svg/23px-Flag_of_Morocco.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/d/d0/Flag_of_Mozambique.svg/23px-Flag_of_Mozambique.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/0/00/Flag_of_Namibia.svg/23px-Flag_of_Namibia.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/3/3e/Flag_of_New_Zealand.svg/23px-Flag_of_New_Zealand.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/7/79/Flag_of_Nigeria.svg/23px-Flag_of_Nigeria.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/d/d9/Flag_of_Norway.svg/21px-Flag_of_Norway.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/3/32/Flag_of_Pakistan.svg/23px-Flag_of_Pakistan.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/2/27/Flag_of_Paraguay.svg/23px-Flag_of_Paraguay.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/c/cf/Flag_of_Peru.svg/23px-Flag_of_Peru.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/9/99/Flag_of_the_Philippines.svg/23px-Flag_of_the_Philippines.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/en/thumb/1/12/Flag_of_Poland.svg/23px-Flag_of_Poland.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/6/65/Flag_of_Qatar.svg/23px-Flag_of_Qatar.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/7/73/Flag_of_Romania.svg/23px-Flag_of_Romania.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/en/thumb/f/f3/Flag_of_Russia.svg/23px-Flag_of_Russia.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/1/17/Flag_of_Rwanda.svg/23px-Flag_of_Rwanda.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/3/31/Flag_of_Samoa.svg/23px-Flag_of_Samoa.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/0/0d/Flag_of_Saudi_Arabia.svg/23px-Flag_of_Saudi_Arabia.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/f/ff/Flag_of_Serbia.svg/23px-Flag_of_Serbia.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/1/17/Flag_of_Sierra_Leone.svg/23px-Flag_of_Sierra_Leone.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/a/af/Flag_of_South_Africa.svg/23px-Flag_of_South_Africa.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/0/09/Flag_of_South_Korea.svg/23px-Flag_of_South_Korea.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/1/11/Flag_of_Sri_Lanka.svg/23px-Flag_of_Sri_Lanka.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/en/thumb/4/4c/Flag_of_Sweden.svg/23px-Flag_of_Sweden.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/f/f3/Flag_of_Switzerland.svg/16px-Flag_of_Switzerland.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/7/72/Flag_of_the_Republic_of_China.svg/23px-Flag_of_the_Republic_of_China.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/d/d0/Flag_of_Tajikistan.svg/23px-Flag_of_Tajikistan.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/a/a9/Flag_of_Thailand.svg/23px-Flag_of_Thailand.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/6/64/Flag_of_Trinidad_and_Tobago.svg/23px-Flag_of_Trinidad_and_Tobago.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/c/ce/Flag_of_Tunisia.svg/23px-Flag_of_Tunisia.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/b/b4/Flag_of_Turkey.svg/23px-Flag_of_Turkey.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/4/4e/Flag_of_Uganda.svg/23px-Flag_of_Uganda.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/4/49/Flag_of_Ukraine.svg/23px-Flag_of_Ukraine.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/c/cb/Flag_of_the_United_Arab_Emirates.svg/23px-Flag_of_the_United_Arab_Emirates.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/en/thumb/a/ae/Flag_of_the_United_Kingdom.svg/23px-Flag_of_the_United_Kingdom.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/en/thumb/a/a4/Flag_of_the_United_States.svg/23px-Flag_of_the_United_States.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/f/fe/Flag_of_Uruguay.svg/23px-Flag_of_Uruguay.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/8/84/Flag_of_Uzbekistan.svg/23px-Flag_of_Uzbekistan.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/2/21/Flag_of_Vietnam.svg/23px-Flag_of_Vietnam.svg.png',\n",
       " '//upload.wikimedia.org/wikipedia/commons/thumb/0/06/Flag_of_Zambia.svg/23px-Flag_of_Zambia.svg.png',\n",
       " '//en.wikipedia.org/wiki/Special:CentralAutoLogin/start?type=1x1',\n",
       " '/static/images/wikimedia-button.png',\n",
       " '/static/images/poweredby_mediawiki_88x31.png']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Print the source of each image\n",
    "# We use a list comprehension here to iterate through each element, find the src atttribute and select it if it exists\n",
    "[element.attrs['src'] for element in imgs if 'src' in element.attrs.keys()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# From Chrome dev tools, we know that the div containing the table has id=\"bodyContent\". Select it.\n",
    "body_content = soup.find('div', {'id': 'bodyContent'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<th>Country or<br/>currency union</th>,\n",
       " <th>Central bank <br/> interest rate (%)</th>,\n",
       " <th>Date of last <br/> change\n",
       " </th>,\n",
       " <th>Average inflation rate 2013-2017 (%)\n",
       " <p>by <a href=\"/wiki/World_Bank\" title=\"World Bank\">WB</a> and <a href=\"/wiki/International_Monetary_Fund\" title=\"International Monetary Fund\">IMF</a><sup class=\"reference\" id=\"cite_ref-1\"><a href=\"#cite_note-1\">[1]</a></sup><sup class=\"reference\" id=\"cite_ref-2\"><a href=\"#cite_note-2\">[2]</a></sup> as in the <a href=\"/wiki/List_of_countries_by_inflation_rate\" title=\"List of countries by inflation rate\">List</a>\n",
       " </p>\n",
       " </th>,\n",
       " <th>Central bank interest rate <br/> minus <br/> average inflation rate (2013-2017)\n",
       " </th>,\n",
       " <th>Central bank interest rate <br/> divided by <br/> average inflation rate (2013-2017)\n",
       " </th>]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get the table headers\n",
    "table_headers = body_content.find_all('th')[:6]\n",
    "table_headers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Country orcurrency union',\n",
       " 'Central bank  interest rate (%)',\n",
       " 'Date of last  change ',\n",
       " 'Average inflation rate 2013-2017 (%) by WB and IMF[1][2] as in the List  ',\n",
       " 'Central bank interest rate  minus  average inflation rate (2013-2017) ',\n",
       " 'Central bank interest rate  divided by  average inflation rate (2013-2017) ']"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get the text itself without the tags and <br/>s\n",
    "table_headers = [element.get_text().replace('\\n', ' ') for element in table_headers]\n",
    "table_headers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the data for each cell in the second row - note we have six columns to parse\n",
    "row_number = 2\n",
    "d1, d2, d3, d4, d5, d6 = body_content.find_all('tr')[row_number].find_all('td')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<td align=\"left\"><span class=\"flagicon\"><img alt=\"\" class=\"thumbborder\" data-file-height=\"600\" data-file-width=\"900\" decoding=\"async\" height=\"15\" src=\"//upload.wikimedia.org/wikipedia/commons/thumb/9/9d/Flag_of_Angola.svg/23px-Flag_of_Angola.svg.png\" srcset=\"//upload.wikimedia.org/wikipedia/commons/thumb/9/9d/Flag_of_Angola.svg/35px-Flag_of_Angola.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/9/9d/Flag_of_Angola.svg/45px-Flag_of_Angola.svg.png 2x\" width=\"23\"/> </span><a href=\"/wiki/Angola\" title=\"Angola\">Angola</a></td>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Print d1 and its text attribute\n",
    "d1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\xa0Angola'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d1.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Angola'"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# We want to remove the undesirable characters from the front - so we just want the text from the anchor tags\n",
    "d1.find('a').text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<td><span data-sort-value=\"000000002019-05-24-0000\" style=\"white-space:nowrap\">24 May 2019</span><sup class=\"reference\" id=\"cite_ref-CentralBankNews_3-1\"><a href=\"#cite_note-CentralBankNews-3\">[3]</a></sup>\n",
       "</td>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Show d3 and its text\n",
    "d3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'24 May 2019[3]\\n'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d3.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'24 May 2019'"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Parse this text properly\n",
    "d3.find_all('span')[0].text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ignoring row 0 because it does not have six fields\n",
      "Ignoring row 26 because exception raised\n",
      "Ignoring row 36 because exception raised\n",
      "Ignoring row 89 because exception raised\n",
      "Ignoring row 94 because it does not have six fields\n",
      "Ignoring row 95 because it does not have six fields\n",
      "Ignoring row 96 because it does not have six fields\n",
      "Ignoring row 97 because it does not have six fields\n",
      "Ignoring row 98 because it does not have six fields\n",
      "Ignoring row 99 because it does not have six fields\n",
      "Ignoring row 100 because it does not have six fields\n",
      "Ignoring row 101 because it does not have six fields\n",
      "Ignoring row 102 because it does not have six fields\n",
      "Ignoring row 103 because it does not have six fields\n",
      "Ignoring row 104 because it does not have six fields\n",
      "Ignoring row 105 because it does not have six fields\n",
      "Ignoring row 106 because it does not have six fields\n",
      "Ignoring row 107 because it does not have six fields\n",
      "Ignoring row 108 because it does not have six fields\n",
      "Ignoring row 109 because it does not have six fields\n"
     ]
    }
   ],
   "source": [
    "# Perform the full scrape by iterating over the elements\n",
    "data = []\n",
    "for i, row in enumerate(body_content.find_all('tr')):\n",
    "    row_data = row.find_all('td')\n",
    "    if len(row_data) != 6:\n",
    "        print ('Ignoring row {} because it does not have six fields'.format(i))\n",
    "        continue\n",
    "    d1, d2, d3, d4, d5, d6 = row_data\n",
    "    try:\n",
    "        d1 = d1.find('a').text\n",
    "        d2 = float(d2.text)\n",
    "        d3 = d3.find_all('span')[0].text\n",
    "    except:\n",
    "        print('Ignoring row {} because exception raised'.format(i))\n",
    "        continue\n",
    "    data.append([d1, d2, d3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['Albania', 1.0, '6 June 2016'], ['Angola', 15.5, '24 May 2019'], ['Argentina', 38.0, '5 March 2020'], ['Armenia', 5.75, '29 January 2019'], ['Australia', 0.25, '19 March 2020'], ['Azerbaijan', 7.25, '1 May 2020'], ['Bahamas', 4.0, '22 December 2016'], ['Bahrain', 2.5, '31 July 2019'], ['Bangladesh', 6.0, '6 April 2018'], ['Barbados', 7.0, '1 June 2009']]\n"
     ]
    }
   ],
   "source": [
    "# Print the head of the scraped data\n",
    "print(data[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the data to a CSV file for later use\n",
    "f_path = './country_interest_rates.csv'\n",
    "with open(f_path, 'w') as f:\n",
    "    f.write('{},{},{}\\n'.format(*table_headers))\n",
    "    for d in data:\n",
    "        f.write('{},{},{}\\n'.format(*d))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Collect some more data - this time population data. We will use this and the previous data for creating visualisations\n",
    "url = 'http://www.worldometers.info/world-population/population-by-country/'\n",
    "page = requests.get(url)\n",
    "soup = BeautifulSoup(page.content, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "body_content = soup.find('table', {'id': 'example2'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<th>#</th>,\n",
       " <th>Country (or dependency)</th>,\n",
       " <th>Population<br/> (2020)</th>,\n",
       " <th>Yearly<br/> Change</th>,\n",
       " <th>Net<br/> Change</th>,\n",
       " <th>Density<br/> (P/Km²)</th>,\n",
       " <th>Land Area<br/> (Km²)</th>,\n",
       " <th>Migrants<br/> (net)</th>,\n",
       " <th>Fert.<br/> Rate</th>,\n",
       " <th>Med.<br/> Age</th>,\n",
       " <th>Urban<br/> Pop %</th>,\n",
       " <th>World<br/> Share</th>]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table_headers = body_content.find_all('th')\n",
    "table_headers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We are only interested in scraping the first three columns after the index number column at the start\n",
    "table_headers = table_headers[1:4]\n",
    "table_headers = [t.text.replace('\\n', '') for t in table_headers]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Country (or dependency)', 'Population (2020)', 'Yearly Change']"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table_headers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get data for a sample row\n",
    "row_number = 2\n",
    "row_data = soup.find_all('tr')[row_number].find_all('td')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12\n"
     ]
    }
   ],
   "source": [
    "print(len(row_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<td>2</td>, <td style=\"font-weight: bold; font-size:15px; text-align:left\"><a href=\"/world-population/india-population/\">India</a></td>, <td style=\"font-weight: bold;\">1,380,004,385</td>, <td>0.99 %</td>]\n"
     ]
    }
   ],
   "source": [
    "# Print the first elements\n",
    "print(row_data[:4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We want indicdes 1, 2 and 3\n",
    "d1, d2, d3 = row_data[1:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "India\n",
      "1,380,004,385\n",
      "0.99 %\n"
     ]
    }
   ],
   "source": [
    "print(d1.find('a').text)\n",
    "print(d2.text)\n",
    "print(d3.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error parsing row 0\n"
     ]
    }
   ],
   "source": [
    "# Since this works, scrape the data\n",
    "data = []\n",
    "for i, row in enumerate(soup.find_all('tr')):\n",
    "    try:\n",
    "        d1, d2, d3 = row.find_all('td')[1:4]\n",
    "        d1 = d1.find('a').text\n",
    "        d2 = d2.text\n",
    "        d3 = d3.text\n",
    "        data.append([d1, d2, d3])\n",
    "    except:\n",
    "        print('Error parsing row {}'.format(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['China', '1,439,323,776', '0.39 %'], ['India', '1,380,004,385', '0.99 %'], ['United States', '331,002,651', '0.59 %'], ['Indonesia', '273,523,615', '1.07 %'], ['Pakistan', '220,892,340', '2.00 %'], ['Brazil', '212,559,417', '0.72 %'], ['Nigeria', '206,139,589', '2.58 %'], ['Bangladesh', '164,689,383', '1.01 %'], ['Russia', '145,934,462', '0.04 %'], ['Mexico', '128,932,753', '1.06 %']]\n"
     ]
    }
   ],
   "source": [
    "# Print the head of the scraped data\n",
    "print(data[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save this data to another CSV for later use\n",
    "f_path = './country_populations.csv'\n",
    "with open(f_path, 'w') as f:\n",
    "    f.write('{},{},{}\\n'.format(*table_headers))\n",
    "    for d in data:\n",
    "        f.write('{},{},{}\\n'.format(*d))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}