20220218/1_3_Course_Webscraping.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Course on webscraping\n",
    "\n",
    "*By Olav ten Bosch, Darius Keijdener, Dick Windmeijer*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports:\n",
    "import requests                     # for issueing HTTP requests\n",
    "from bs4 import BeautifulSoup       # for parsing and navigating HTML results\n",
    "import time                         # for sleeping between multiple requests\n",
    "\n",
    "# Not related to webscraping\n",
    "from IPython.display import Image   # for viewing images in a python notebook"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Documentation:\n",
    "- [Requests.py](http://docs.python-requests.org)\n",
    "- [Beautifulsoup.py](https://www.crummy.com/software/BeautifulSoup/bs4/doc/)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Request, headers, user-agent, parameters, sleeping:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Retrieving home page of Statistics Netherlands:\n",
    "r1 = requests.get('https://www.cbs.nl/en-gb')\n",
    "\n",
    "#r1.headers['content-type']\n",
    "print(r1.status_code, r1.headers['content-type'], r1.encoding)\n",
    "#print(r1.headers)\n",
    "#print(r1.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Retrieving home page of Statistics Netherlands with user-agent string:\n",
    "headers = {'user-agent': 'scrapingCourseBot'}\n",
    "r2 = requests.get('https://www.cbs.nl/en-gb', headers=headers)\n",
    "\n",
    "# Headers of the request:\n",
    "print(r2.request.headers)\n",
    "\n",
    "# Headers of the response:\n",
    "#print(r2.headers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Issue a request with parameters:\n",
    "pars = {'products': 2, 'years': 2}\n",
    "r3 = requests.get('http://testing-ground.webscraping.pro/table-middle.html?', params=pars, headers=headers)        \n",
    "print(r3.url)\n",
    "#print(r3.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# In a loop, always add some idle time (time.sleep) to not overload server:\n",
    "for products in range(1, 4):\n",
    "    for years in range(1, 3):\n",
    "        pars = {'products': products, 'years': years}\n",
    "        r4 = requests.get('http://testing-ground.webscraping.pro/table-middle.html?', params=pars, headers=headers)\n",
    "        print(r4.url, r4.status_code)\n",
    "        time.sleep(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We can also querry other types of files\n",
    "r5 = requests.get(\"http://testing-ground.webscraping.pro/img/logo.png\", headers=headers)\n",
    "r5.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This is not just text.\n",
    "r5.content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The .content attribute returns a binary string, which can be interpreted by other functions.\n",
    "Image(r5.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r6 = requests.get(\"https://www.cbs.nl/-/media/_excel/2021/34/tozo_reg_2021_juni.xls\", headers=headers)\n",
    "r6.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Note that we write the binary. If you are merely writinng text, save respons.text and omit the \"b\" option.\n",
    "# Always be mindfull when you download and save files, especially in binary!\n",
    "if r6.status_code == 200:\n",
    "    with open(\"bestand_tozo.xls\", \"wb\") as file:\n",
    "        file.write(r6.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This works only if you are running this locally, pandas is not installed in the binder machines\n",
    "import pandas as pd\n",
    "pd.read_excel(\"bestand_tozo.xls\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Beautifulsoup with native syntax (find, find_all):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get page with requests:\n",
    "r4 = requests.get('https://www.cbs.nl/en-gb', headers=headers)\n",
    "print(r4.url, r4.status_code)\n",
    "#print(r4.request.headers)\n",
    "#print(r4.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Using soup to parse the html page:\n",
    "soup = BeautifulSoup(r4.text, 'lxml')              # use lxml, is faster and more relaxed in parsing \n",
    "# find returns the first element:\n",
    "print(soup.find(\"h2\"))\n",
    "#print(soup.find(\"h2\").text)\n",
    "#print(soup.find(\"h3\").text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# find the first element which belongs to a class:\n",
    "part = soup.find(\"a\", class_=\"thumbnail\")\n",
    "print(part)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# find the first element with an id:\n",
    "aside = soup.find(\"section\", id=\"aside-main\")\n",
    "print(aside)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# You can use find on a find result:\n",
    "print(part.find(\"h3\").text)\n",
    "print(soup.find(\"a\", class_=\"thumbnail\").find(\"h3\").text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# how to get a URL from an a tag:\n",
    "print(part['href'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# find_all returns a list:\n",
    "print(soup.find_all(\"h2\"))\n",
    "print(\"\")\n",
    "print(soup.find_all(\"h3\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# You can iterate through a list output of find_all:\n",
    "# Get the URLS to all news articles of CBS using find_all and find:\n",
    "articles = soup.find_all(\"a\", class_='thumbnail')\n",
    "for article in articles:\n",
    "    link = article['href']   # we retrieve the attribute href of the a tag\n",
    "    print(link)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Follow the links and get all texts of the news articles:\n",
    "articles = soup.find_all(\"a\", class_='thumbnail')\n",
    "links3 = []\n",
    "for article in articles:\n",
    "    links3.append(article['href'])\n",
    "\n",
    "for link in links3:\n",
    "    r = requests.get(link, headers=headers)\n",
    "    #print(r.url)\n",
    "    soup2 = BeautifulSoup(r.text, 'lxml')\n",
    "    leadtext = soup2.find('section', class_='leadtext')\n",
    "    if leadtext is None: continue\n",
    "    print(leadtext.text)\n",
    "    time.sleep(1) # in robots.txt CBS advises a delay of 1 second"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}