20230911/3_2_Headless_Selenium.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Headless scraping using Selenium\n",
    "\n",
    "This notebook does not work in the binder, needs local driver install, see below"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Documentation:\n",
    "- [Requests.py](http://docs.python-requests.org)\n",
    "- [Beautifulsoup.py](https://www.crummy.com/software/BeautifulSoup/bs4/doc/)\n",
    "- [Selenium with python](https://selenium-python.readthedocs.io/)\n",
    "- [Selenium](https://www.selenium.dev/documentation/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Install Selenium: This needs to be executed only once!!!!!\n",
    "!pip install selenium\n",
    "\n",
    "# Download the selenium driver for your browser and machine from\n",
    "#         https://selenium-python.readthedocs.io/installation.html#drivers\n",
    "# and add it to this local folder:\n",
    "import os\n",
    "import sys\n",
    "os.path.dirname(sys.executable)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports:\n",
    "import time\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Static Price list page the old way (requests, beautifulsoup):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "headers = {'user-agent': 'scrapingCourseBot'}\n",
    "\n",
    "# Pricelist first product name via beautifulsoup:\n",
    "r = requests.get(\"https://snstatcomp.github.io/webscrapingtests/course/prices1.html\", headers=headers)\n",
    "soup = BeautifulSoup(r.text, 'lxml')\n",
    "print(soup.select('div.name')[0].text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Static Price list page headless via Chrome:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "options = webdriver.ChromeOptions()\n",
    "options.add_argument(\"--headless=new\")\n",
    "options.add_argument('user-agent=scrapingCourseBot')\n",
    "\n",
    "browser = webdriver.Chrome(options=options)\n",
    "browser.get(\"https://snstatcomp.github.io/webscrapingtests/course/prices1.html\")\n",
    "\n",
    "# Retrieve first item name:\n",
    "elem = browser.find_element(By.CSS_SELECTOR, 'div.name')\n",
    "print(elem.text)\n",
    "\n",
    "browser.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Headless request to price list (Firefox):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium.webdriver.firefox.options import Options\n",
    "options = Options()\n",
    "options.add_argument(\"--headless=new\")\n",
    "options.add_argument('user-agent=scrapingCourseBot')\n",
    "\n",
    "browser = webdriver.Firefox(options=options)\n",
    "browser.get('https://snstatcomp.github.io/webscrapingtests/course/prices1.html')\n",
    "\n",
    "# Retrieve first item name:\n",
    "elem = browser.find_element(By.CSS_SELECTOR, 'div.name')\n",
    "print(elem.text)\n",
    "\n",
    "browser.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Headless getting Olympic winter games (Chrome): "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "options = webdriver.ChromeOptions()\n",
    "#options.add_argument(\"--headless=new\")\n",
    "#options.add_argument('user-agent=scrapingCourseBot')\n",
    "\n",
    "browser = webdriver.Chrome(options=options)\n",
    "browser.get('https://olympics.com/en/sports/winter-olympics')\n",
    "\n",
    "time.sleep(5)\n",
    "\n",
    "# Retrieve first item name:\n",
    "# elements = browser.find_elements_by_css_selector('div.-sportlist h1.article--title') # this worked in Sep 2021\n",
    "elements = browser.find_elements(By.CSS_SELECTOR, 'li.b2p-sports-wrapper__item') # this works in Sep 2022\n",
    "for e in elements:\n",
    "    print(e.text)\n",
    "\n",
    "#browser.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}