Web Scraping_Darty-Final.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#Darty---Web-Scraping-Washing-Machines\" data-toc-modified-id=\"Darty---Web-Scraping-Washing-Machines-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>Darty - Web Scraping Washing Machines</a></span><ul class=\"toc-item\"><li><span><a href=\"#Automate-the-process-to-web-scrape-every-page,-clean-and-export-to-MySQL\" data-toc-modified-id=\"Automate-the-process-to-web-scrape-every-page,-clean-and-export-to-MySQL-1.1\"><span class=\"toc-item-num\">1.1&nbsp;&nbsp;</span>Automate the process to web scrape every page, clean and export to MySQL</a></span></li></ul></li></ul></div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#                         Darty - Web Scraping Washing Machines"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"https://www.darty.com/static/7uNa/catalog/version_desktop/common/images/darty_sprite/sprite_darty_logo.png\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Automate the process to web scrape every page, clean and export to MySQL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing all the libraries\n",
    "\n",
    "from datetime import datetime\n",
    "import requests as r\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "from time import sleep\n",
    "from random import randint\n",
    "from requests.exceptions import SSLError, Timeout, TooManyRedirects, RequestException\n",
    "from contextlib import closing\n",
    "from IPython.display import Markdown, display\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "timeout_glob=2\n",
    "verify_glob=True\n",
    "allow_glob=False\n",
    "\n",
    "# Defining the headers to access the webpage\n",
    "\n",
    "headers = \"\"\"accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9\n",
    "accept-encoding: gzip, deflate, br\n",
    "accept-language: pt-PT,pt;q=0.9,en-US;q=0.8,en;q=0.7,fr;q=0.6,cs;q=0.5,es;q=0.4\n",
    "cache-control: no-cache\n",
    "cookie: rxVisitor=1590242224286J69VFHRIQQ1S8B9U5GF1UJ80DRQ0G9V6; tCdebugLib=1; s_ecid=MCMID%7C12464273977194400053212204472571811876; segm=30; kameleoonVisitorCode=_js_2pjr0b13qqm0pmtx; cikneeto_uuid=id:4af89f12-80b5-46aa-8b30-ee6b90a27b33; OptanonAlertBoxClosed=2020-07-11T08:53:59.109Z; _gcl_au=1.1.336720708.1594457640; sto__vuid=c342c0ee9c87f7a5df2a12b21358c3da; _ga=GA1.2.791524087.1594457815; eb-profile-cluster=3; basket=%7B%22channel%22%3A%22DOT_COM%22%2C%22lines%22%3A%5B%5D%2C%22coursierSlotCheck%22%3Afalse%7D; PCKORDER=0_0_0.00; USERZIPCODE=78100; USERINSEECODE=78551; PCKCITY=ST GERMAIN EN LAYE; _hjid=42759858-8ae4-4966-a4aa-bf4b73f155ef; DARTY_COM_COMPARATOR=4213297:4045360:4084535:4010051; _pin_unauth=dWlkPU1XWTBOMk0yTXpJdE5HVmhNeTAwT0Rrd0xXRXpPVGt0TkdOa056bGhNR1EzT1RFdyZycD1kSEoxWlE; eb-profile-clusters={%22587747fbec8040bea82640dc%22:{%221599007678673%22:15%2C%221598921331813%22:3}}; session_user=w4halw2Ewr9LuyndtVgK; eb-lastactivity-hash=-833916260; dartLev63=SEO_www.google.com_classique_www.google.com%7C1599134590358; _derived_epik=dj0yJnU9VzJlLW9ieUpKTmNQbFZlU01PZXFINU9HX1pyTjlOdlEmbj1iVE1LU040ZzhLZVZMUjBoQWpKY3hBJm09MSZ0PUFBQUFBRjlTQWhZJnJtPTEmcnQ9QUFBQUFGOVNBaFk; uuid230=85A56553-6EA6-4D7F-A931-9534618E5BE6; tc_payment=1; PCKEMAIL=-x0j0iajxD4Ick-03_kshJVayi6_fI0keLYacxFxThM; PCKFIRSTNAME=TIAGO; PCKECSESSION=eEJaBmjDQ8eryHbdLu7Q3fqoxM-luydblsDcsJHZipA; sid_dv=5f5629f6a6d345f5629f6a6d78; s_dartCanStc=SEO%3EReferents%3ESEO%3EReferents; s_dartCode=SEO_www.google.com_classique_www.google.com%7C1594457602058%3Eref_www.mistergooddeal.com_www.mistergooddeal.com_www.mistergooddeal.com%7C1596798480688%3ESEO_www.google.com_classique_www.google.com%7C1599128658447%3Eref_abonnes.fnac.com_abonnes.fnac.com_abonnes.fnac.com%7C1599485436030%3Eref_localhost%3A8888_localhost%3A8888_localhost%3A8888%7C1599581524687; dartLev=ref_localhost%3A8888_localhost%3A8888_localhost%3A8888%7C1599581532657; etuix=4b7jvovUimcWB45TLDSKbaShByfuKJJeM9VeT3qiR6WP93h9p7Z_qA--; ak_bmsc=B76FA556EB7627B6430CC8B6DC4E2E5558DD91969B570000CD125A5F5BC5FD5A~plGYWc/H0gfvSlPU51rDLlJ4KbnWje+4YBzY3PjSHkyHpn7s40zSl45iVabK0yyHZFmDkohYDlt5zqoYj4QBmQKRwguqJnq4NUNcLuPDOt5onqd+0lH2C55QLUzzBqcp2Ik5abARxxZ0HLpBQwdTNJkn9kTaf6akqz2j5IUj1rUdf0LAw+aclA6QB8sDRTtsJeVtj0fPgxWHJUMHlX2RN1QA9srEuw9bnkC5Lkoj67ztA=; JSESSIONID=0000mLrl_uOjy4t6iAcff7baPe3:19858inu2; session=bgw1bo2e4ionn24md4a34p; tc_cj_v2=m_iZZZ%22**%22%27%20ZZZKOSSKLRPORLJOZZZ%5D777%5Ecl_%5Dny%5B%5D%5D_mmZZZZZZKOSSKLRPPSSJMZZZ%5D777m_iZZZ%22**%22%27%20ZZZKOSSKLSQJKSMPZZZ%5D777%5Ecl_%5Dny%5B%5D%5D_mmZZZZZZKOSSKMLORPRJQZZZ%5D777m_iZZZ%22**%22%27%20ZZZKOSSKMNOSJLNQZZZ%5D777%5Ecl_%5Dny%5B%5D%5D_mmZZZZZZKOSSNROMMPRKPZZZ%5D777_rn_lh%5BfyfcheZZZ%7B%7C*%29%29%20.H%21%29%7B%7DH%7D*%28ZZZKOSSNRONMSSRQZZZ%5D777%5Ecl_%5Dny%5B%5D%5D_mmZZZZZZKOSSOQRLLJKJNZZZ%5D777_rn_lh%5BfyfcheZZZ%27*%7D%7B%27%23*./TRRRRZZZKOSSORKOMJSOOZZZ%5D777%5Ecl_%5Dny%5B%5D%5D_mmZZZZZZKOSSQMROQNKPSZZZ%5D; sto__session=1599738576144; _hjIncludedInSessionSample=1; _hjTLDTest=1; _hjAbsoluteSessionInProgress=0; AMCVS_0C4B401053DABFF10A490D4C%40AdobeOrg=1; AMCV_0C4B401053DABFF10A490D4C%40AdobeOrg=870038026%7CMCIDTS%7C18515%7CMCMID%7C12464273977194400053212204472571811876%7CMCAAMLH-1600343376%7C6%7CMCAAMB-1600343376%7CRKhpRz8krg2tLO6pguXWp5olkAcUniQYPHaMWWgdJ3xzPWQmdj0y%7CMCOPTOUT-1599745776s%7CNONE%7CMCAID%7CNONE%7CvVersion%7C5.0.0; s_cc=true; eb-profile=89b357b3-626d-47bf-9507-3a58d97443a9:149609516:1599738576446; dtLatC=50; OptanonConsent=isIABGlobal=false&datestamp=Thu+Sep+10+2020+13%3A49%3A40+GMT%2B0200+(Hora+de+ver%C3%A3o+da+Europa+Central)&version=6.1.0&consentId=285cc4d2-c883-4da4-864a-843719390514&interactionCount=1&landingPath=NotLandingPage&groups=C0001%3A1%2CC0002%3A1%2CC0003%3A1%2CC0004%3A1&hosts=&legInt=&geolocation=%3B&AwaitingReconsent=false; sto__count=1; dartProfPa=2; gpv_p3=www%2Fcatalogue%2Fgros_electromenager%2Flave_linge; dartDateLastCall=1599738581472; bm_sv=B05361CCCB8562BDB702EFE2422BE6DC~eOK/bmsMER3KZM3QNJa5LtzIH9T0OJM8YCmmSa7iRFzXoKJbe/ACML41Sq9zHOzA5Weqc9PFLnVwQZz5QcM3KEAsAG9AdvxYyOleClyhlQZ9BUFUkY69O87cKS2topMNW4Wgn8/h9COaihqUjmP3m6DhycL3X5LKg3bW2IHDWB4=; dtPC=2$138579168_518h-vPPRPUPKKBFAOMTKTHMBHMERQTNHLBJPP-0e3; akavpau_VP_WaitingRoom=1599738883~id=163eb1d47c1531bc8fccd8ee06125efd; datadome=30TYCErqR80OhnQqLltvFtELNKHDUxIMIZL-O9fYiQCx051VsCTNL9dfzcmOvz-MTZlRqLlc.g~2ovVfa8jlSYMqGS0XqlXdNYuLNGyIwH; rxvt=1599740392859|1599738573344; s_sq=darty-prod%3D%2526c.%2526a.%2526activitymap.%2526page%253Dwww%25252Fcatalogue%25252Fgros_electromenager%25252Flave_linge%2526link%253D2%2526region%253Dmain_pagination_top%2526pageIDType%253D1%2526.activitymap%2526.a%2526.c%2526pid%253Dwww%25252Fcatalogue%25252Fgros_electromenager%25252Flave_linge%2526pidt%253D1%2526oid%253Dhttps%25253A%25252F%25252Fwww.darty.com%25252Fnav%25252Fachat%25252Fgros_electromenager%25252Flave-linge%25252Fpage2.html%2526ot%253DA; cikneeto=date:1599738595976; dtSa=true%7CC%7C-1%7C2%7C-%7C1599738595961%7C138579168_518%7Chttps%3A%2F%2Fwww.darty.com%2Fnav%2Fachat%2Fgros_5Felectromenager%2Flave-linge%2Findex.html%7CLave-linge%20-%20Livr%C3%A9%5Ec%20install%C3%A9%20gratuitement%7C1599738589229%7Cdartyclic%3DX_gros-elec_lave-ling%7C; dtCookie=v_4_srv_2_sn_UC49D3F8UIIA1FTO2K7JNDQGNFK5JSNB_app-3Ae8e01c74db6645d0_0_ol_0_perc_100000_mul_1\n",
    "pragma: no-cache\n",
    "referer: https://www.darty.com/nav/achat/gros_electromenager/lave-linge/index.html\n",
    "sec-fetch-dest: document\n",
    "sec-fetch-mode: navigate\n",
    "sec-fetch-site: same-origin\n",
    "sec-fetch-user: ?1\n",
    "upgrade-insecure-requests: 1\n",
    "user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36\"\"\"\n",
    "headers=dict([i.strip().split(': ') for i in headers.split('\\n')])\n",
    "\n",
    "# Defining the error handling functions\n",
    "\n",
    "def printmd(string):\n",
    "    display(Markdown('**'+string+'**'))\n",
    "def is_good_response(x):\n",
    "    \"\"\"\n",
    "    Returns True if the response seems to be an HTML, False otherwise\n",
    "    x is response\n",
    "    \"\"\"\n",
    "    content_type=x.headers['Content-Type'].lower()\n",
    "    return ((x.status_code==200 and content_type is not None and content_type.find('html')>1), x.status_code)\n",
    "def log_error(e):\n",
    "    print(e)\n",
    "def status_handler(status):\n",
    "    if status<300:\n",
    "        print('Alles gut!')\n",
    "    elif status>=400 and status<500:\n",
    "        print('request has failed due to you mistake bro. Check the link, headers, server and whether you have an access to this page.')\n",
    "    else:\n",
    "        print('Yoooo, I have no clue what has happened. But you are stuck, since you didnt receive any output from server')\n",
    "    return None\n",
    "def adaptive_scraping(url, timeout=timeout_glob, verify=verify_glob):\n",
    "    global timeout_glob\n",
    "    global verify_glob\n",
    "    global allow_glob\n",
    "    import requests as r\n",
    "    try:\n",
    "        with closing(r.get(url, stream=True, timeout=timeout_glob, verify=verify_glob, allow_redirects=allow_glob, headers=headers)) as resp:\n",
    "            test1=is_good_response(resp)\n",
    "            print(resp)\n",
    "            if test1[0]:\n",
    "                return resp.content\n",
    "            else:\n",
    "                return status_handler(test1[1])\n",
    "    except Timeout as e:\n",
    "        log_error(f'Error during request to {url}: {str(e)}')\n",
    "        printmd('We can increase response waiting time. Do you want to continue?')\n",
    "        ans=input('\\n[Yes|No]\\t')\n",
    "        if ans.lower()=='yes':\n",
    "            timeout_glob*=2\n",
    "            return adaptive_scraping(url)\n",
    "        else:\n",
    "            return None\n",
    "    except SSLError as e:\n",
    "        log_error(f'Error during request to {url}: {str(e)}')\n",
    "        printmd('We can skip verification test. Do you want to continue?')\n",
    "        ans=input('\\n[Yes|No]\\t')\n",
    "        if ans.lower()=='yes':\n",
    "            verify_glob=False\n",
    "            return adaptive_scraping(url)\n",
    "        else:\n",
    "            return None\n",
    "    except TooManyRedirects as e:\n",
    "        log_error(f'Error during request to {url}: {str(e)}')\n",
    "        printmd('We can increase the number of allowed redirects. Do you want to continue?')\n",
    "        ans=input('\\n[Yes|No]\\t')\n",
    "        if ans.lower()=='yes':\n",
    "            allow_glob=True\n",
    "            return adaptive_scraping(url)\n",
    "        else:\n",
    "            return None\n",
    "    except RequestException as e:\n",
    "        log_error(f'Error during request to {url}: {str(e)}')\n",
    "        printmd('Unfortunately, we have no clue what to do.  Please try again later. ')\n",
    "        return None\n",
    "    \n",
    "# Defining the function to extract the information from the webpage\n",
    "\n",
    "def extract():\n",
    "\n",
    "    from datetime import datetime\n",
    "    pages = [*range(1,1000)]\n",
    "    ratings = []\n",
    "    prices = []\n",
    "    info = []\n",
    "    name = []\n",
    "    category = []\n",
    "    discount = []\n",
    "    stock = []\n",
    "\n",
    "    # Creating the loop to web scrape each page\n",
    "\n",
    "    for page in pages:\n",
    "        url = 'https://www.darty.com/nav/achat/gros_electromenager/lave-linge/page'+str(page)+'.html'\n",
    "        adaptive_scraping(url)\n",
    "        response = r.get(url, headers=headers)\n",
    "        soup = BeautifulSoup(response.content)\n",
    "        container = soup.select('div.product_main_container')\n",
    "        \n",
    "        if container ==[]:\n",
    "            break\n",
    "        else:\n",
    "\n",
    "            print('Retrieving data from page '+str(page))\n",
    "            sleep(randint(2,6))\n",
    "\n",
    "            # Append the information related to Ratings in the Ratings List\n",
    "\n",
    "            for i in container:\n",
    "                l=[i.text.strip() for i in i.select('span.rating_avis')]\n",
    "                if len(l)==0:\n",
    "                    ratings.append('0')\n",
    "                else:\n",
    "                    ratings.append(''.join(l))\n",
    "\n",
    "             # Append the information related to Prices in the Prices List\n",
    "\n",
    "            for i in container:\n",
    "                l=[i.text.strip() for i in i.select('span.darty_prix')]\n",
    "                if len(l)==0:\n",
    "                    prices.append('0')\n",
    "                else:\n",
    "                    prices.append(''.join(l))\n",
    "\n",
    "             # Append the information related to the features of the washer in the Info List\n",
    "\n",
    "            for i in container:\n",
    "                l=[i.text.strip() for i in i.select('ul.infos_strenghts')]\n",
    "                if len(l)==0:\n",
    "                    info.append('0')\n",
    "                else:\n",
    "                    info.append(''.join(l))\n",
    "\n",
    "             # Append the information related to the category, brand and model in the Name List\n",
    "\n",
    "            for i in container:\n",
    "                l=[i.text.strip() for i in i.select('div.prd-family')]\n",
    "                if len(l)==0:\n",
    "                    name.append('0')\n",
    "                else:\n",
    "                    name.append(''.join(l))\n",
    "\n",
    "             # Append the information related to Discounts in the Discounts List\n",
    "\n",
    "            for i in container:\n",
    "                l=[i.text.strip().replace('- ','') for i in i.select('span.striped_price')]\n",
    "                if len(l)==0:\n",
    "                    discount.append('0')\n",
    "                else:\n",
    "                    discount.append(''.join(l))\n",
    "\n",
    "             # Append the information related to stock in the stock List\n",
    "\n",
    "            for i in container:\n",
    "                l=[i.text.strip() for i in i.select('div.status_delivery p.delivery_date')]\n",
    "                if len(l)==0:\n",
    "                    stock.append('0')\n",
    "                else:\n",
    "                    stock.append(''.join(l))\n",
    "                \n",
    "    # Merge every list into one dataframe\n",
    "\n",
    "    df = pd.DataFrame(\n",
    "        {'Name': name,\n",
    "         'Info': info,\n",
    "         'Price': prices,\n",
    "         'Discount': discount,\n",
    "         'Stock': stock,\n",
    "         'Ratings': ratings\n",
    "        })\n",
    "    \n",
    "    return df\n",
    "\n",
    "# FUNCTION TO CLEAN THE DATA\n",
    "\n",
    "def clean(df):\n",
    "\n",
    "    categories = []\n",
    "    \n",
    "    # Splitting the Ratings column in order to have the number of Reviews separate from the Ratings\n",
    "    \n",
    "    split_ratings = df.Ratings.str.split('(',1, expand = True)\n",
    "    df['Ratings']= split_ratings[0]\n",
    "    df['Total_Reviews'] = split_ratings[1]\n",
    "    df.Total_Reviews = df.Total_Reviews.str.replace(' avis\\)','')\n",
    "    df['Total_Reviews'] = pd.to_numeric(df.Total_Reviews)\n",
    "    df.Ratings = df.Ratings.str.replace('/5','')\n",
    "    df.Ratings = df.Ratings.str.replace(',','.')\n",
    "    df = df.astype({'Ratings': 'float'})\n",
    "    \n",
    "    # Creating the categories column based on the category present in the Name column\n",
    "\n",
    "    for i in df.Name:\n",
    "        if i.find('hublot') != -1:\n",
    "            categories.append('Lave linge hublot')\n",
    "        elif i.find('top') != -1:\n",
    "            categories.append('Lave-linge top')\n",
    "        elif i.find('séchant') != -1:\n",
    "            categories.append('Lave linge séchant')\n",
    "        elif i.find('mini') != -1:\n",
    "            categories.append('Mini lave linge')\n",
    "        else:\n",
    "            categories.append('Other')\n",
    "    df.insert(0, 'Category', categories)\n",
    "    \n",
    "    # Removing the Categories from the Name column\n",
    "    \n",
    "    df.Name = df.Name.str.replace('Lave linge hublot','')\n",
    "    df.Name = df.Name.str.replace('Lave-linge top','')\n",
    "    df.Name = df.Name.str.replace('Lave linge séchant','')\n",
    "    df.Name = df.Name.str.replace('Mini lave linge','')\n",
    "    \n",
    "    \n",
    "\n",
    "    df.Price = df.Price.str.replace('*','')\n",
    "    df['Class'] = df['Info'].str.extract(r'([A-B][+]*)')\n",
    "    df['Washing_Capacity'] = df['Info'].str.extract(r'(\\d*.\\d* kg)')\n",
    "    df['Washing_Capacity'] = df['Washing_Capacity'].str.replace(' kg','')\n",
    "    df['Washing_Capacity'] = df['Washing_Capacity'].str.replace(',','.')\n",
    "    df = df.astype({'Washing_Capacity': 'float'})\n",
    "    \n",
    "    # Splitting the Info into 3 Feature columns so each feature is separate\n",
    "    \n",
    "    df['feature1'] = [i[1] if 1 < len(i) else None for i in df['Info'].str.split('\\n')]\n",
    "    df['feature2'] = [i[2] if 2 < len(i) else None for i in df['Info'].str.split('\\n')]\n",
    "    df['feature3'] = [i[3] if 3 < len(i) else None for i in df['Info'].str.split('\\n')]\n",
    "    \n",
    "    # Splitting the Model from the Brand\n",
    "    \n",
    "    df['Model'] = [i[1] for i in df['Name'].str.split(' ',1)]\n",
    "    df['Brand'] = [i[0] for i in df['Name'].str.split(' ',1)]\n",
    "    df = df.drop(columns = ['Info','Name'])\n",
    "    \n",
    "    # Adding the Import Date column with current datetime\n",
    "    \n",
    "    df['Import_Date'] = datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
    "    df['Import_Date']= pd.to_datetime(df['Import_Date'])\n",
    "    \n",
    "    # Reordering the columns\n",
    "    \n",
    "    columns = ['Category','Brand','Model','Class','Washing_Capacity','feature1','feature2','feature3','Price','Discount','Stock','Ratings','Total_Reviews','Import_Date']\n",
    "    df = df[columns]\n",
    "    \n",
    "    return df\n",
    "\n",
    "# FUNCTION TO EXPORT TO MYSQL\n",
    "\n",
    "def export(df):\n",
    "    import pymysql\n",
    "    from sqlalchemy import create_engine\n",
    "    from getpass import getpass\n",
    "    from sqlalchemy import inspect\n",
    "    username='root'\n",
    "    print('Please enter the password the access the MySQL database')\n",
    "    password=getpass()\n",
    "    engine=create_engine(f'mysql+pymysql://{username}:{password}@localhost/web_scraping')\n",
    "    df.to_sql('darty_washers', engine, if_exists='append', index=False)\n",
    "    print('Export successful!')\n",
    "    \n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Response [200]>\n",
      "Retrieving data from page 1\n",
      "<Response [200]>\n",
      "Retrieving data from page 2\n",
      "<Response [200]>\n",
      "Retrieving data from page 3\n",
      "<Response [200]>\n",
      "Retrieving data from page 4\n",
      "<Response [200]>\n",
      "Retrieving data from page 5\n",
      "<Response [200]>\n",
      "Retrieving data from page 6\n",
      "<Response [200]>\n",
      "Retrieving data from page 7\n",
      "<Response [200]>\n",
      "Retrieving data from page 8\n",
      "<Response [200]>\n",
      "Retrieving data from page 9\n",
      "<Response [200]>\n",
      "Retrieving data from page 10\n",
      "<Response [200]>\n",
      "Retrieving data from page 11\n",
      "<Response [200]>\n",
      "Retrieving data from page 12\n",
      "<Response [200]>\n",
      "Retrieving data from page 13\n",
      "<Response [200]>\n",
      "Retrieving data from page 14\n",
      "<Response [200]>\n",
      "Retrieving data from page 15\n",
      "<Response [200]>\n",
      "Retrieving data from page 16\n",
      "<Response [200]>\n",
      "Retrieving data from page 17\n",
      "<Response [200]>\n",
      "Retrieving data from page 18\n",
      "<Response [200]>\n",
      "Retrieving data from page 19\n",
      "<Response [200]>\n",
      "Retrieving data from page 20\n",
      "<Response [200]>\n",
      "Please enter the password the access the MySQL database\n",
      "········\n",
      "Export successful!\n"
     ]
    }
   ],
   "source": [
    "extract().pipe(clean).pipe(export)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}