Zoopla_data_gathering_script.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "115ef7e4",
   "metadata": {},
   "source": [
    "<h1 style='background-color: BLACK; padding: 10px; color: white; text-align: center;'>ZOOPLA </h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8e175e1e",
   "metadata": {},
   "source": [
    "<h1 style='background-color: BLACK; padding: 10px; color: white'> Sales </h1>\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7b65328d",
   "metadata": {},
   "source": [
    "First things first, we will import the required libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e4062122",
   "metadata": {},
   "outputs": [],
   "source": [
    "import undetected_chromedriver as webdriver\n",
    "from selenium.webdriver.chrome.service import Service\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "import time\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2dc57114",
   "metadata": {},
   "source": [
    "Next, we will load in the dataset containing the list of postcode for this project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1195321d",
   "metadata": {},
   "outputs": [],
   "source": [
    "codes = pd.read_csv('London postcode districts.xlsx - PC DIST.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e98a82ab",
   "metadata": {},
   "source": [
    "Now, we will create a function called 'zoopla_sales' to scrap properties that are for sale on the Zoopla website"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "080b8952",
   "metadata": {},
   "outputs": [],
   "source": [
    "def zoopla_sales(postcodes, Trans_type, website, df):\n",
    "    \"\"\"\n",
    "    This function scrapes property data from Rightmove for a given list of postcodes and transaction type,\n",
    "    and returns the data as a pandas DataFrame.\n",
    "\n",
    "    Arguments:\n",
    "    postcodes -- a list of postcodes for which to scrape property data\n",
    "    Trans_type -- the transaction type of the properties to be scraped ('sales' or 'rent')\n",
    "    website -- the name of the website being scraped (in this case, 'Rightmove')\n",
    "    df -- an empty pandas DataFrame to store the scraped data\n",
    "\n",
    "    Returns:\n",
    "    df1 -- a pandas DataFrame containing the scraped property data\n",
    "    \n",
    "    \"\"\"    \n",
    "    # Set the path to the ChromeDriver executable\n",
    "    s = Service(\"C:\\\\Users\\\\user\\\\Downloads\\\\Set ups\\\\chromedriver_win32\\\\chromedriver.exe\")\n",
    "    \n",
    "    # Launch the ChromeDriver with the specified service\n",
    "    driver = webdriver.Chrome(service= s)\n",
    "    \n",
    "    # Navigate to the specified URL \n",
    "    driver.get('https://www.zoopla.co.uk/') \n",
    "\n",
    "    # Maximize the window\n",
    "    time.sleep(2)\n",
    "    driver.maximize_window()\n",
    "\n",
    "    # Find the search bar  \n",
    "    time.sleep(2.2)\n",
    "    #search = driver.find_element(By.XPATH, '/html/body/div[3]/div/div[1]/main/div[1]/div/div/div[1]/div[2]/div/div/div[2]/div/form/div/div[1]/div/div/div/div/div/div/input')\n",
    "    try:\n",
    "        search = driver.find_element(By.XPATH, \"//input[contains(@id,'downshift')]\")\n",
    "        search.send_keys(postcode)\n",
    "        time.sleep(1.3)\n",
    "    except:\n",
    "        search = driver.find_element(By.XPATH, \"//input[contains(@class,'_1qzmny55 _1ftx2fq8')]\")\n",
    "        search.send_keys(postcode)\n",
    "        time.sleep(1.3)\n",
    "    \n",
    "    \n",
    "    # Click the search button\n",
    "    time.sleep(1.5)   \n",
    "    driver.find_element(By.XPATH, \"//button[@class='x8jo560 x8jo562 x8jo56a _1ftx2fq8'][1]\").click()\n",
    "    \n",
    "    \n",
    "    \n",
    "    # Initialize empty lists for storing scraped data\n",
    "    Trans_type = []\n",
    "    address = []\n",
    "    types = []\n",
    "    bedrooms = []\n",
    "    bathrooms = []\n",
    "    prices = []\n",
    "    desc = []\n",
    "    date_added = []\n",
    "    agent_list = []\n",
    "    property_url = []\n",
    "    website = []\n",
    "    \n",
    "    # Create an empty dataframe to store the scraped data\n",
    "    df = pd.DataFrame()\n",
    "    \n",
    "     # Setting the page number to be 1 \n",
    "    i = 1\n",
    "    while True: \n",
    "        # Scrapping data for the required features in the first page\n",
    "        time.sleep(1) \n",
    "        print(\"{} {} {} {}\".format('scraping page', i,'from', postcode ))\n",
    "        \n",
    "        time.sleep(2.1)\n",
    "        try: \n",
    "            driver.find_element(By.XPATH, \"//*[contains(@class,'u94mg1 u94mg4 u94mg9')]\").click()\n",
    "            time.sleep(1.5)\n",
    "            \n",
    "        except:\n",
    "            time.sleep(1.0)\n",
    "            \n",
    "        time.sleep(1.4)    \n",
    "        address_list =driver.find_elements(By.XPATH, \"//h2[contains(@class,'_1ankud51 _1ftx2fq8')]\")\n",
    "        type_list =driver.find_elements(By.XPATH, \"//h3[contains(@class,'_1ankud52 _1ftx2fq9')]\")\n",
    "        bedroom_list =driver.find_elements(By.XPATH, \"//div[contains(@id,'listing')]//li[1]//span[2]\")\n",
    "        bathroom_list =driver.find_elements(By.XPATH, \"//div[contains(@id,'listing')]//li[3]//span[2]\")\n",
    "        price_list =driver.find_elements(By.XPATH, \"//p[contains(@class,'_170k6632 _1ftx2fq6')]\")\n",
    "        desc_list =driver.find_elements(By.XPATH, \"//p[contains(@class,'_1ankud53 _1ftx2fq9')]\")\n",
    "        date_added_list =driver.find_elements(By.XPATH, \"//li[contains(@class,'_18cib8e1')]\")\n",
    "        agent_list_list =driver.find_elements(By.XPATH, \"//img[contains(@class,'_12bxhf70')]\")\n",
    "        property_url_list = driver.current_url\n",
    "        Trans_type_list = Trans_type\n",
    "        website_list = website\n",
    "        for address_item, type_item, bedroom_item, bathroom_item, price_item, desc_item, date_added_item, agent_list_item in zip(address_list, type_list, bedroom_list, bathroom_list, price_list, desc_list, date_added_list, agent_list_list):\n",
    "            address.append(address_item.text)\n",
    "            types.append(type_item.text)\n",
    "            bedrooms.append(bedroom_item.text)\n",
    "            bathrooms.append(bathroom_item.text)\n",
    "            prices.append(price_item.text)\n",
    "            desc.append(desc_item.text)\n",
    "            date_added.append(date_added_item.text)\n",
    "            agent_list.append(agent_list_item.get_attribute('alt'))\n",
    "            property_url.append(property_url_list)\n",
    "            Trans_type.append(Trans_type_list)\n",
    "            website.append(website_list)\n",
    "\n",
    "\n",
    "        time.sleep(1.3)\n",
    "        # get the height of the page\n",
    "        page_height = driver.execute_script(\"return document.body.scrollHeight;\")\n",
    "\n",
    "        \n",
    "        # Cancel the pop-up window\n",
    "        j= i + 1\n",
    "        url = f'https://www.zoopla.co.uk/for-sale/property/{postcode}/?q={postcode}&search_source=home&pn={j}'\n",
    "        if len(address_list) or len(type_list) != 0:\n",
    "            driver.get(url)\n",
    "        else:\n",
    "            print('------------------------------- SCRAPING COMPLETED FOR ' + postcode)\n",
    "            break\n",
    "        \n",
    "            \n",
    "        #Increment the page number\n",
    "        i += 1\n",
    "        time.sleep(1)\n",
    "                \n",
    "        \n",
    "    # Create a dataframe to store data scrapped for each postcode    \n",
    "    df1 = pd.DataFrame({'Location': postcode, 'Tansaction_Type': 'Sales','Property_Type':types, 'Address' :address, 'Bedrooms': bedrooms, 'Bathrooms':bathrooms, 'Price':prices, 'Description': desc, 'Listing_Date':date_added, 'Agent':agent_list, 'Listing_Source': 'Zoopla', 'listing_URL':property_url})\n",
    "    df1.insert(0, 'Unique_Id', [f'{postcode}S{i+1:05d}ZP' for i in range(len(df1))])\n",
    "    print('Total numbers of properties available in ' + postcode + ' is ' + str(df1.shape[0]))\n",
    "\n",
    "     # Concat the dataframe obtain for all postcodes\n",
    "    df = pd.concat([df, df1], ignore_index=True)\n",
    "    \n",
    "     # Return a dataframe\n",
    "    return df1\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "467a5750",
   "metadata": {},
   "source": [
    "Now, the data for properties available for sale at Zoopla is scrapped"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "cd140770",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scraping page 1 from BR2\n",
      "scraping page 2 from BR2\n",
      "scraping page 3 from BR2\n",
      "scraping page 4 from BR2\n",
      "scraping page 5 from BR2\n",
      "scraping page 6 from BR2\n",
      "scraping page 7 from BR2\n",
      "scraping page 8 from BR2\n",
      "scraping page 9 from BR2\n",
      "scraping page 10 from BR2\n",
      "scraping page 11 from BR2\n",
      "------------------------------- SCRAPING COMPLETED FOR BR2\n",
      "Total numbers of properties available in BR2 is 202\n",
      "scraping page 1 from BR3\n",
      "scraping page 2 from BR3\n",
      "scraping page 3 from BR3\n",
      "scraping page 4 from BR3\n",
      "scraping page 5 from BR3\n",
      "scraping page 6 from BR3\n",
      "scraping page 7 from BR3\n",
      "scraping page 8 from BR3\n",
      "scraping page 9 from BR3\n",
      "scraping page 10 from BR3\n",
      "------------------------------- SCRAPING COMPLETED FOR BR3\n",
      "Total numbers of properties available in BR3 is 158\n",
      "scraping page 1 from BR4\n",
      "scraping page 2 from BR4\n",
      "scraping page 3 from BR4\n",
      "scraping page 4 from BR4\n",
      "scraping page 5 from BR4\n",
      "------------------------------- SCRAPING COMPLETED FOR BR4\n",
      "Total numbers of properties available in BR4 is 78\n",
      "scraping page 1 from BR5\n",
      "scraping page 2 from BR5\n",
      "scraping page 3 from BR5\n",
      "scraping page 4 from BR5\n",
      "scraping page 5 from BR5\n",
      "scraping page 6 from BR5\n",
      "scraping page 7 from BR5\n",
      "scraping page 8 from BR5\n",
      "scraping page 9 from BR5\n",
      "scraping page 10 from BR5\n",
      "------------------------------- SCRAPING COMPLETED FOR BR5\n",
      "Total numbers of properties available in BR5 is 192\n",
      "scraping page 1 from BR6\n",
      "scraping page 2 from BR6\n",
      "scraping page 3 from BR6\n",
      "scraping page 4 from BR6\n",
      "scraping page 5 from BR6\n",
      "scraping page 6 from BR6\n",
      "scraping page 7 from BR6\n",
      "scraping page 8 from BR6\n",
      "scraping page 9 from BR6\n",
      "scraping page 10 from BR6\n",
      "scraping page 11 from BR6\n",
      "scraping page 12 from BR6\n",
      "scraping page 13 from BR6\n",
      "scraping page 14 from BR6\n",
      "------------------------------- SCRAPING COMPLETED FOR BR6\n",
      "Total numbers of properties available in BR6 is 275\n",
      "scraping page 1 from BR7\n",
      "scraping page 2 from BR7\n",
      "scraping page 3 from BR7\n",
      "scraping page 4 from BR7\n",
      "scraping page 5 from BR7\n",
      "scraping page 6 from BR7\n",
      "------------------------------- SCRAPING COMPLETED FOR BR7\n",
      "Total numbers of properties available in BR7 is 106\n",
      "scraping page 1 from BR8\n",
      "scraping page 2 from BR8\n",
      "scraping page 3 from BR8\n",
      "scraping page 4 from BR8\n",
      "scraping page 5 from BR8\n",
      "scraping page 6 from BR8\n",
      "------------------------------- SCRAPING COMPLETED FOR BR8\n",
      "Total numbers of properties available in BR8 is 118\n",
      "scraping page 1 from CR0\n",
      "scraping page 2 from CR0\n",
      "scraping page 3 from CR0\n",
      "scraping page 4 from CR0\n",
      "scraping page 5 from CR0\n",
      "scraping page 6 from CR0\n",
      "scraping page 7 from CR0\n",
      "scraping page 8 from CR0\n",
      "scraping page 9 from CR0\n",
      "scraping page 10 from CR0\n",
      "scraping page 11 from CR0\n",
      "scraping page 12 from CR0\n",
      "scraping page 13 from CR0\n",
      "scraping page 14 from CR0\n",
      "scraping page 15 from CR0\n",
      "scraping page 16 from CR0\n",
      "scraping page 17 from CR0\n",
      "scraping page 18 from CR0\n",
      "scraping page 19 from CR0\n",
      "scraping page 20 from CR0\n",
      "scraping page 21 from CR0\n",
      "scraping page 22 from CR0\n",
      "scraping page 23 from CR0\n",
      "scraping page 24 from CR0\n",
      "scraping page 25 from CR0\n",
      "scraping page 26 from CR0\n",
      "scraping page 27 from CR0\n",
      "scraping page 28 from CR0\n",
      "scraping page 29 from CR0\n",
      "scraping page 30 from CR0\n",
      "scraping page 31 from CR0\n",
      "scraping page 32 from CR0\n",
      "scraping page 33 from CR0\n",
      "scraping page 34 from CR0\n",
      "scraping page 35 from CR0\n",
      "scraping page 36 from CR0\n",
      "scraping page 37 from CR0\n",
      "scraping page 38 from CR0\n",
      "scraping page 39 from CR0\n",
      "scraping page 40 from CR0\n",
      "scraping page 41 from CR0\n",
      "scraping page 42 from CR0\n",
      "------------------------------- SCRAPING COMPLETED FOR CR0\n",
      "Total numbers of properties available in CR0 is 905\n",
      "scraping page 1 from CR2\n",
      "scraping page 2 from CR2\n",
      "scraping page 3 from CR2\n",
      "scraping page 4 from CR2\n",
      "scraping page 5 from CR2\n",
      "scraping page 6 from CR2\n",
      "scraping page 7 from CR2\n",
      "scraping page 8 from CR2\n",
      "scraping page 9 from CR2\n",
      "scraping page 10 from CR2\n",
      "scraping page 11 from CR2\n",
      "scraping page 12 from CR2\n",
      "scraping page 13 from CR2\n",
      "scraping page 14 from CR2\n",
      "------------------------------- SCRAPING COMPLETED FOR CR2\n",
      "Total numbers of properties available in CR2 is 270\n",
      "scraping page 1 from CR3\n",
      "scraping page 2 from CR3\n",
      "scraping page 3 from CR3\n",
      "scraping page 4 from CR3\n",
      "scraping page 5 from CR3\n",
      "scraping page 6 from CR3\n",
      "scraping page 7 from CR3\n",
      "scraping page 8 from CR3\n",
      "scraping page 9 from CR3\n",
      "scraping page 10 from CR3\n",
      "scraping page 11 from CR3\n",
      "scraping page 12 from CR3\n",
      "scraping page 13 from CR3\n",
      "scraping page 14 from CR3\n",
      "------------------------------- SCRAPING COMPLETED FOR CR3\n",
      "Total numbers of properties available in CR3 is 291\n",
      "scraping page 1 from CR4\n",
      "scraping page 2 from CR4\n",
      "scraping page 3 from CR4\n",
      "scraping page 4 from CR4\n",
      "scraping page 5 from CR4\n",
      "scraping page 6 from CR4\n",
      "scraping page 7 from CR4\n",
      "scraping page 8 from CR4\n",
      "scraping page 9 from CR4\n",
      "scraping page 10 from CR4\n",
      "scraping page 11 from CR4\n",
      "scraping page 12 from CR4\n",
      "------------------------------- SCRAPING COMPLETED FOR CR4\n",
      "Total numbers of properties available in CR4 is 219\n",
      "scraping page 1 from CR5\n",
      "scraping page 2 from CR5\n",
      "scraping page 3 from CR5\n",
      "scraping page 4 from CR5\n",
      "scraping page 5 from CR5\n",
      "scraping page 6 from CR5\n",
      "scraping page 7 from CR5\n",
      "------------------------------- SCRAPING COMPLETED FOR CR5\n",
      "Total numbers of properties available in CR5 is 126\n",
      "scraping page 1 from CR6\n",
      "scraping page 2 from CR6\n",
      "scraping page 3 from CR6\n",
      "scraping page 4 from CR6\n",
      "------------------------------- SCRAPING COMPLETED FOR CR6\n",
      "Total numbers of properties available in CR6 is 61\n",
      "scraping page 1 from CR7\n",
      "scraping page 2 from CR7\n",
      "scraping page 3 from CR7\n",
      "scraping page 4 from CR7\n",
      "scraping page 5 from CR7\n",
      "scraping page 6 from CR7\n",
      "scraping page 7 from CR7\n",
      "scraping page 8 from CR7\n",
      "scraping page 9 from CR7\n",
      "scraping page 10 from CR7\n",
      "------------------------------- SCRAPING COMPLETED FOR CR7\n",
      "Total numbers of properties available in CR7 is 194\n",
      "scraping page 1 from CR8\n",
      "scraping page 2 from CR8\n",
      "scraping page 3 from CR8\n",
      "scraping page 4 from CR8\n",
      "scraping page 5 from CR8\n",
      "scraping page 6 from CR8\n",
      "scraping page 7 from CR8\n",
      "scraping page 8 from CR8\n",
      "scraping page 9 from CR8\n",
      "scraping page 10 from CR8\n",
      "scraping page 11 from CR8\n",
      "scraping page 12 from CR8\n",
      "scraping page 13 from CR8\n",
      "scraping page 14 from CR8\n",
      "scraping page 15 from CR8\n",
      "------------------------------- SCRAPING COMPLETED FOR CR8\n",
      "Total numbers of properties available in CR8 is 291\n",
      "scraping page 1 from CR9\n",
      "scraping page 2 from CR9\n",
      "------------------------------- SCRAPING COMPLETED FOR CR9\n",
      "Total numbers of properties available in CR9 is 10\n",
      "scraping page 1 from DA1\n",
      "scraping page 2 from DA1\n",
      "scraping page 3 from DA1\n",
      "scraping page 4 from DA1\n",
      "scraping page 5 from DA1\n",
      "scraping page 6 from DA1\n",
      "scraping page 7 from DA1\n",
      "scraping page 8 from DA1\n",
      "scraping page 9 from DA1\n",
      "scraping page 10 from DA1\n",
      "scraping page 11 from DA1\n",
      "scraping page 12 from DA1\n",
      "------------------------------- SCRAPING COMPLETED FOR DA1\n",
      "Total numbers of properties available in DA1 is 228\n",
      "scraping page 1 from DA14\n",
      "scraping page 2 from DA14\n",
      "scraping page 3 from DA14\n",
      "scraping page 4 from DA14\n",
      "scraping page 5 from DA14\n",
      "scraping page 6 from DA14\n",
      "scraping page 7 from DA14\n",
      "------------------------------- SCRAPING COMPLETED FOR DA14\n",
      "Total numbers of properties available in DA14 is 122\n",
      "scraping page 1 from DA15\n",
      "scraping page 2 from DA15\n",
      "scraping page 3 from DA15\n",
      "scraping page 4 from DA15\n",
      "scraping page 5 from DA15\n",
      "scraping page 6 from DA15\n",
      "------------------------------- SCRAPING COMPLETED FOR DA15\n",
      "Total numbers of properties available in DA15 is 120\n",
      "scraping page 1 from DA16\n",
      "scraping page 2 from DA16\n",
      "------------------------------- SCRAPING COMPLETED FOR DA16\n",
      "Total numbers of properties available in DA16 is 23\n",
      "scraping page 1 from DA17\n",
      "scraping page 2 from DA17\n",
      "scraping page 3 from DA17\n",
      "scraping page 4 from DA17\n",
      "------------------------------- SCRAPING COMPLETED FOR DA17\n",
      "Total numbers of properties available in DA17 is 73\n",
      "scraping page 1 from DA18\n",
      "scraping page 2 from DA18\n",
      "------------------------------- SCRAPING COMPLETED FOR DA18\n",
      "Total numbers of properties available in DA18 is 16\n",
      "scraping page 1 from DA5\n",
      "scraping page 2 from DA5\n",
      "scraping page 3 from DA5\n",
      "scraping page 4 from DA5\n",
      "scraping page 5 from DA5\n",
      "scraping page 6 from DA5\n",
      "------------------------------- SCRAPING COMPLETED FOR DA5\n",
      "Total numbers of properties available in DA5 is 109\n",
      "scraping page 1 from DA6\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scraping page 2 from DA6\n",
      "scraping page 3 from DA6\n",
      "scraping page 4 from DA6\n",
      "scraping page 5 from DA6\n",
      "------------------------------- SCRAPING COMPLETED FOR DA6\n",
      "Total numbers of properties available in DA6 is 60\n",
      "scraping page 1 from DA7\n",
      "scraping page 2 from DA7\n",
      "scraping page 3 from DA7\n",
      "scraping page 4 from DA7\n",
      "scraping page 5 from DA7\n",
      "scraping page 6 from DA7\n",
      "scraping page 7 from DA7\n",
      "------------------------------- SCRAPING COMPLETED FOR DA7\n",
      "Total numbers of properties available in DA7 is 134\n",
      "scraping page 1 from DA8\n",
      "scraping page 2 from DA8\n",
      "scraping page 3 from DA8\n",
      "scraping page 4 from DA8\n",
      "scraping page 5 from DA8\n",
      "scraping page 6 from DA8\n",
      "------------------------------- SCRAPING COMPLETED FOR DA8\n",
      "Total numbers of properties available in DA8 is 112\n",
      "scraping page 1 from E1\n",
      "scraping page 2 from E1\n",
      "scraping page 3 from E1\n",
      "scraping page 4 from E1\n",
      "scraping page 5 from E1\n",
      "scraping page 6 from E1\n",
      "scraping page 7 from E1\n",
      "scraping page 8 from E1\n",
      "scraping page 9 from E1\n",
      "scraping page 10 from E1\n",
      "scraping page 11 from E1\n",
      "scraping page 12 from E1\n",
      "scraping page 13 from E1\n",
      "scraping page 14 from E1\n",
      "scraping page 15 from E1\n",
      "scraping page 16 from E1\n",
      "scraping page 17 from E1\n",
      "scraping page 18 from E1\n",
      "scraping page 19 from E1\n",
      "scraping page 20 from E1\n",
      "scraping page 21 from E1\n",
      "scraping page 22 from E1\n",
      "scraping page 23 from E1\n",
      "scraping page 24 from E1\n",
      "scraping page 25 from E1\n",
      "scraping page 26 from E1\n",
      "scraping page 27 from E1\n",
      "scraping page 28 from E1\n",
      "scraping page 29 from E1\n",
      "scraping page 30 from E1\n",
      "scraping page 31 from E1\n",
      "scraping page 32 from E1\n",
      "scraping page 33 from E1\n",
      "scraping page 34 from E1\n",
      "------------------------------- SCRAPING COMPLETED FOR E1\n",
      "Total numbers of properties available in E1 is 695\n",
      "scraping page 1 from E10\n",
      "scraping page 2 from E10\n",
      "scraping page 3 from E10\n",
      "scraping page 4 from E10\n",
      "scraping page 5 from E10\n",
      "scraping page 6 from E10\n",
      "scraping page 7 from E10\n",
      "scraping page 8 from E10\n",
      "scraping page 9 from E10\n",
      "scraping page 10 from E10\n",
      "------------------------------- SCRAPING COMPLETED FOR E10\n",
      "Total numbers of properties available in E10 is 194\n",
      "scraping page 1 from E11\n",
      "------------------------------- SCRAPING COMPLETED FOR E11\n",
      "Total numbers of properties available in E11 is 0\n",
      "scraping page 1 from E12\n"
     ]
    },
    {
     "ename": "MaxRetryError",
     "evalue": "HTTPConnectionPool(host='localhost', port=53740): Max retries exceeded with url: /session/e9b1deeba0f182775fc7c6105969ab2f/elements (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x000001A7610A0610>: Failed to establish a new connection: [WinError 10061] No connection could be made because the target machine actively refused it'))",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mConnectionRefusedError\u001b[0m                    Traceback (most recent call last)",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\urllib3\\connection.py:174\u001b[0m, in \u001b[0;36mHTTPConnection._new_conn\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    173\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m--> 174\u001b[0m     conn \u001b[38;5;241m=\u001b[39m \u001b[43mconnection\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate_connection\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    175\u001b[0m \u001b[43m        \u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_dns_host\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mport\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mextra_kw\u001b[49m\n\u001b[0;32m    176\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    178\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m SocketTimeout:\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\urllib3\\util\\connection.py:95\u001b[0m, in \u001b[0;36mcreate_connection\u001b[1;34m(address, timeout, source_address, socket_options)\u001b[0m\n\u001b[0;32m     94\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m err \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m---> 95\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m err\n\u001b[0;32m     97\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m socket\u001b[38;5;241m.\u001b[39merror(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgetaddrinfo returns an empty list\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\urllib3\\util\\connection.py:85\u001b[0m, in \u001b[0;36mcreate_connection\u001b[1;34m(address, timeout, source_address, socket_options)\u001b[0m\n\u001b[0;32m     84\u001b[0m     sock\u001b[38;5;241m.\u001b[39mbind(source_address)\n\u001b[1;32m---> 85\u001b[0m sock\u001b[38;5;241m.\u001b[39mconnect(sa)\n\u001b[0;32m     86\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m sock\n",
      "\u001b[1;31mConnectionRefusedError\u001b[0m: [WinError 10061] No connection could be made because the target machine actively refused it",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mNewConnectionError\u001b[0m                        Traceback (most recent call last)",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\urllib3\\connectionpool.py:703\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[1;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[0;32m    702\u001b[0m \u001b[38;5;66;03m# Make the request on the httplib connection object.\u001b[39;00m\n\u001b[1;32m--> 703\u001b[0m httplib_response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    704\u001b[0m \u001b[43m    \u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    705\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    706\u001b[0m \u001b[43m    \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    707\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    708\u001b[0m \u001b[43m    \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    709\u001b[0m \u001b[43m    \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    710\u001b[0m \u001b[43m    \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    711\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    713\u001b[0m \u001b[38;5;66;03m# If we're going to release the connection in ``finally:``, then\u001b[39;00m\n\u001b[0;32m    714\u001b[0m \u001b[38;5;66;03m# the response doesn't need to know about the connection. Otherwise\u001b[39;00m\n\u001b[0;32m    715\u001b[0m \u001b[38;5;66;03m# it will also try to release it and we'll have a double-release\u001b[39;00m\n\u001b[0;32m    716\u001b[0m \u001b[38;5;66;03m# mess.\u001b[39;00m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\urllib3\\connectionpool.py:398\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[1;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[0;32m    397\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m--> 398\u001b[0m         \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mhttplib_request_kw\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    400\u001b[0m \u001b[38;5;66;03m# We are swallowing BrokenPipeError (errno.EPIPE) since the server is\u001b[39;00m\n\u001b[0;32m    401\u001b[0m \u001b[38;5;66;03m# legitimately able to close the connection after sending a valid response.\u001b[39;00m\n\u001b[0;32m    402\u001b[0m \u001b[38;5;66;03m# With this behaviour, the received response is still readable.\u001b[39;00m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\urllib3\\connection.py:244\u001b[0m, in \u001b[0;36mHTTPConnection.request\u001b[1;34m(self, method, url, body, headers)\u001b[0m\n\u001b[0;32m    243\u001b[0m     headers[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUser-Agent\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m _get_default_user_agent()\n\u001b[1;32m--> 244\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mHTTPConnection\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\http\\client.py:1283\u001b[0m, in \u001b[0;36mHTTPConnection.request\u001b[1;34m(self, method, url, body, headers, encode_chunked)\u001b[0m\n\u001b[0;32m   1282\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Send a complete request to the server.\"\"\"\u001b[39;00m\n\u001b[1;32m-> 1283\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_send_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\http\\client.py:1329\u001b[0m, in \u001b[0;36mHTTPConnection._send_request\u001b[1;34m(self, method, url, body, headers, encode_chunked)\u001b[0m\n\u001b[0;32m   1328\u001b[0m     body \u001b[38;5;241m=\u001b[39m _encode(body, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbody\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m-> 1329\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mendheaders\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencode_chunked\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\http\\client.py:1278\u001b[0m, in \u001b[0;36mHTTPConnection.endheaders\u001b[1;34m(self, message_body, encode_chunked)\u001b[0m\n\u001b[0;32m   1277\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m CannotSendHeader()\n\u001b[1;32m-> 1278\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_send_output\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmessage_body\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencode_chunked\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\http\\client.py:1038\u001b[0m, in \u001b[0;36mHTTPConnection._send_output\u001b[1;34m(self, message_body, encode_chunked)\u001b[0m\n\u001b[0;32m   1037\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_buffer[:]\n\u001b[1;32m-> 1038\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmsg\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1040\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m message_body \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m   1041\u001b[0m \n\u001b[0;32m   1042\u001b[0m     \u001b[38;5;66;03m# create a consistent interface to message_body\u001b[39;00m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\http\\client.py:976\u001b[0m, in \u001b[0;36mHTTPConnection.send\u001b[1;34m(self, data)\u001b[0m\n\u001b[0;32m    975\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mauto_open:\n\u001b[1;32m--> 976\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    977\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\urllib3\\connection.py:205\u001b[0m, in \u001b[0;36mHTTPConnection.connect\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    204\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mconnect\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m--> 205\u001b[0m     conn \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_new_conn\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    206\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_prepare_conn(conn)\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\urllib3\\connection.py:186\u001b[0m, in \u001b[0;36mHTTPConnection._new_conn\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    185\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m SocketError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m--> 186\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m NewConnectionError(\n\u001b[0;32m    187\u001b[0m         \u001b[38;5;28mself\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFailed to establish a new connection: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m e\n\u001b[0;32m    188\u001b[0m     )\n\u001b[0;32m    190\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m conn\n",
      "\u001b[1;31mNewConnectionError\u001b[0m: <urllib3.connection.HTTPConnection object at 0x000001A7610A0610>: Failed to establish a new connection: [WinError 10061] No connection could be made because the target machine actively refused it",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mMaxRetryError\u001b[0m                             Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[6], line 7\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[38;5;66;03m# loop through postcodes\u001b[39;00m\n\u001b[0;32m      5\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m postcode \u001b[38;5;129;01min\u001b[39;00m codes\u001b[38;5;241m.\u001b[39mloc[\u001b[38;5;241m1\u001b[39m:][\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mPostcode district\u001b[39m\u001b[38;5;124m'\u001b[39m]:\n\u001b[0;32m      6\u001b[0m     \u001b[38;5;66;03m# call the function and pass the empty DataFrame as an argument\u001b[39;00m\n\u001b[1;32m----> 7\u001b[0m     df1 \u001b[38;5;241m=\u001b[39m \u001b[43mzoopla_sales\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpostcode\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mSales\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mZoopla\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdf\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m      8\u001b[0m     \u001b[38;5;66;03m# append the df1 DataFrame to the empty DataFrame\u001b[39;00m\n\u001b[0;32m      9\u001b[0m     df \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mconcat([df, df1], ignore_index\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
      "Cell \u001b[1;32mIn[3], line 80\u001b[0m, in \u001b[0;36mzoopla_sales\u001b[1;34m(postcodes, Trans_type, website, df)\u001b[0m\n\u001b[0;32m     77\u001b[0m     time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m1.0\u001b[39m)\n\u001b[0;32m     79\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m1.4\u001b[39m)    \n\u001b[1;32m---> 80\u001b[0m address_list \u001b[38;5;241m=\u001b[39m\u001b[43mdriver\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfind_elements\u001b[49m\u001b[43m(\u001b[49m\u001b[43mBy\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mXPATH\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m//h2[contains(@class,\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m_1ankud51 _1ftx2fq8\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m)]\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[0;32m     81\u001b[0m type_list \u001b[38;5;241m=\u001b[39mdriver\u001b[38;5;241m.\u001b[39mfind_elements(By\u001b[38;5;241m.\u001b[39mXPATH, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m//h3[contains(@class,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m_1ankud52 _1ftx2fq9\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)]\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m     82\u001b[0m bedroom_list \u001b[38;5;241m=\u001b[39mdriver\u001b[38;5;241m.\u001b[39mfind_elements(By\u001b[38;5;241m.\u001b[39mXPATH, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m//div[contains(@id,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlisting\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)]//li[1]//span[2]\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py:861\u001b[0m, in \u001b[0;36mWebDriver.find_elements\u001b[1;34m(self, by, value)\u001b[0m\n\u001b[0;32m    857\u001b[0m     value \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m[name=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mvalue\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m]\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[0;32m    859\u001b[0m \u001b[38;5;66;03m# Return empty list if driver returns null\u001b[39;00m\n\u001b[0;32m    860\u001b[0m \u001b[38;5;66;03m# See https://github.com/SeleniumHQ/selenium/issues/4555\u001b[39;00m\n\u001b[1;32m--> 861\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[43mCommand\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mFIND_ELEMENTS\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43musing\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mby\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mvalue\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m}\u001b[49m\u001b[43m)\u001b[49m[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mvalue\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;129;01mor\u001b[39;00m []\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py:438\u001b[0m, in \u001b[0;36mWebDriver.execute\u001b[1;34m(self, driver_command, params)\u001b[0m\n\u001b[0;32m    435\u001b[0m     \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msessionId\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m params:\n\u001b[0;32m    436\u001b[0m         params[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msessionId\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msession_id\n\u001b[1;32m--> 438\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcommand_executor\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdriver_command\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    439\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m response:\n\u001b[0;32m    440\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39merror_handler\u001b[38;5;241m.\u001b[39mcheck_response(response)\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\selenium\\webdriver\\remote\\remote_connection.py:290\u001b[0m, in \u001b[0;36mRemoteConnection.execute\u001b[1;34m(self, command, params)\u001b[0m\n\u001b[0;32m    288\u001b[0m data \u001b[38;5;241m=\u001b[39m utils\u001b[38;5;241m.\u001b[39mdump_json(params)\n\u001b[0;32m    289\u001b[0m url \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_url\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00mpath\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m--> 290\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcommand_info\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\selenium\\webdriver\\remote\\remote_connection.py:311\u001b[0m, in \u001b[0;36mRemoteConnection._request\u001b[1;34m(self, method, url, body)\u001b[0m\n\u001b[0;32m    308\u001b[0m     body \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m    310\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mkeep_alive:\n\u001b[1;32m--> 311\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_conn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    312\u001b[0m     statuscode \u001b[38;5;241m=\u001b[39m response\u001b[38;5;241m.\u001b[39mstatus\n\u001b[0;32m    313\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\urllib3\\request.py:78\u001b[0m, in \u001b[0;36mRequestMethods.request\u001b[1;34m(self, method, url, fields, headers, **urlopen_kw)\u001b[0m\n\u001b[0;32m     74\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrequest_encode_url(\n\u001b[0;32m     75\u001b[0m         method, url, fields\u001b[38;5;241m=\u001b[39mfields, headers\u001b[38;5;241m=\u001b[39mheaders, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39murlopen_kw\n\u001b[0;32m     76\u001b[0m     )\n\u001b[0;32m     77\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m---> 78\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest_encode_body\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m     79\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfields\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfields\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43murlopen_kw\u001b[49m\n\u001b[0;32m     80\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\urllib3\\request.py:170\u001b[0m, in \u001b[0;36mRequestMethods.request_encode_body\u001b[1;34m(self, method, url, fields, headers, encode_multipart, multipart_boundary, **urlopen_kw)\u001b[0m\n\u001b[0;32m    167\u001b[0m extra_kw[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mheaders\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mupdate(headers)\n\u001b[0;32m    168\u001b[0m extra_kw\u001b[38;5;241m.\u001b[39mupdate(urlopen_kw)\n\u001b[1;32m--> 170\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mextra_kw\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\urllib3\\poolmanager.py:376\u001b[0m, in \u001b[0;36mPoolManager.urlopen\u001b[1;34m(self, method, url, redirect, **kw)\u001b[0m\n\u001b[0;32m    374\u001b[0m     response \u001b[38;5;241m=\u001b[39m conn\u001b[38;5;241m.\u001b[39murlopen(method, url, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkw)\n\u001b[0;32m    375\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m--> 376\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mu\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest_uri\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkw\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    378\u001b[0m redirect_location \u001b[38;5;241m=\u001b[39m redirect \u001b[38;5;129;01mand\u001b[39;00m response\u001b[38;5;241m.\u001b[39mget_redirect_location()\n\u001b[0;32m    379\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m redirect_location:\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\urllib3\\connectionpool.py:815\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[1;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[0;32m    810\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m conn:\n\u001b[0;32m    811\u001b[0m     \u001b[38;5;66;03m# Try again\u001b[39;00m\n\u001b[0;32m    812\u001b[0m     log\u001b[38;5;241m.\u001b[39mwarning(\n\u001b[0;32m    813\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRetrying (\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m) after connection broken by \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m, retries, err, url\n\u001b[0;32m    814\u001b[0m     )\n\u001b[1;32m--> 815\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    816\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    817\u001b[0m \u001b[43m        \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    818\u001b[0m \u001b[43m        \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    819\u001b[0m \u001b[43m        \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    820\u001b[0m \u001b[43m        \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    821\u001b[0m \u001b[43m        \u001b[49m\u001b[43mredirect\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    822\u001b[0m \u001b[43m        \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    823\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    824\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpool_timeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpool_timeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    825\u001b[0m \u001b[43m        \u001b[49m\u001b[43mrelease_conn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrelease_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    826\u001b[0m \u001b[43m        \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    827\u001b[0m \u001b[43m        \u001b[49m\u001b[43mbody_pos\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody_pos\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    828\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mresponse_kw\u001b[49m\n\u001b[0;32m    829\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    831\u001b[0m \u001b[38;5;66;03m# Handle redirect?\u001b[39;00m\n\u001b[0;32m    832\u001b[0m redirect_location \u001b[38;5;241m=\u001b[39m redirect \u001b[38;5;129;01mand\u001b[39;00m response\u001b[38;5;241m.\u001b[39mget_redirect_location()\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\urllib3\\connectionpool.py:815\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[1;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[0;32m    810\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m conn:\n\u001b[0;32m    811\u001b[0m     \u001b[38;5;66;03m# Try again\u001b[39;00m\n\u001b[0;32m    812\u001b[0m     log\u001b[38;5;241m.\u001b[39mwarning(\n\u001b[0;32m    813\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRetrying (\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m) after connection broken by \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m, retries, err, url\n\u001b[0;32m    814\u001b[0m     )\n\u001b[1;32m--> 815\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    816\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    817\u001b[0m \u001b[43m        \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    818\u001b[0m \u001b[43m        \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    819\u001b[0m \u001b[43m        \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    820\u001b[0m \u001b[43m        \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    821\u001b[0m \u001b[43m        \u001b[49m\u001b[43mredirect\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    822\u001b[0m \u001b[43m        \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    823\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    824\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpool_timeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpool_timeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    825\u001b[0m \u001b[43m        \u001b[49m\u001b[43mrelease_conn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrelease_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    826\u001b[0m \u001b[43m        \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    827\u001b[0m \u001b[43m        \u001b[49m\u001b[43mbody_pos\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody_pos\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    828\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mresponse_kw\u001b[49m\n\u001b[0;32m    829\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    831\u001b[0m \u001b[38;5;66;03m# Handle redirect?\u001b[39;00m\n\u001b[0;32m    832\u001b[0m redirect_location \u001b[38;5;241m=\u001b[39m redirect \u001b[38;5;129;01mand\u001b[39;00m response\u001b[38;5;241m.\u001b[39mget_redirect_location()\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\urllib3\\connectionpool.py:815\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[1;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[0;32m    810\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m conn:\n\u001b[0;32m    811\u001b[0m     \u001b[38;5;66;03m# Try again\u001b[39;00m\n\u001b[0;32m    812\u001b[0m     log\u001b[38;5;241m.\u001b[39mwarning(\n\u001b[0;32m    813\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRetrying (\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m) after connection broken by \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m, retries, err, url\n\u001b[0;32m    814\u001b[0m     )\n\u001b[1;32m--> 815\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    816\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    817\u001b[0m \u001b[43m        \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    818\u001b[0m \u001b[43m        \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    819\u001b[0m \u001b[43m        \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    820\u001b[0m \u001b[43m        \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    821\u001b[0m \u001b[43m        \u001b[49m\u001b[43mredirect\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    822\u001b[0m \u001b[43m        \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    823\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    824\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpool_timeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpool_timeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    825\u001b[0m \u001b[43m        \u001b[49m\u001b[43mrelease_conn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrelease_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    826\u001b[0m \u001b[43m        \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    827\u001b[0m \u001b[43m        \u001b[49m\u001b[43mbody_pos\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody_pos\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    828\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mresponse_kw\u001b[49m\n\u001b[0;32m    829\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    831\u001b[0m \u001b[38;5;66;03m# Handle redirect?\u001b[39;00m\n\u001b[0;32m    832\u001b[0m redirect_location \u001b[38;5;241m=\u001b[39m redirect \u001b[38;5;129;01mand\u001b[39;00m response\u001b[38;5;241m.\u001b[39mget_redirect_location()\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\urllib3\\connectionpool.py:787\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[1;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[0;32m    784\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(e, (SocketError, HTTPException)):\n\u001b[0;32m    785\u001b[0m     e \u001b[38;5;241m=\u001b[39m ProtocolError(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mConnection aborted.\u001b[39m\u001b[38;5;124m\"\u001b[39m, e)\n\u001b[1;32m--> 787\u001b[0m retries \u001b[38;5;241m=\u001b[39m \u001b[43mretries\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mincrement\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    788\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43merror\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43me\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_pool\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_stacktrace\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msys\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexc_info\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m2\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[0;32m    789\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    790\u001b[0m retries\u001b[38;5;241m.\u001b[39msleep()\n\u001b[0;32m    792\u001b[0m \u001b[38;5;66;03m# Keep track of the error for the retry warning.\u001b[39;00m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\urllib3\\util\\retry.py:592\u001b[0m, in \u001b[0;36mRetry.increment\u001b[1;34m(self, method, url, response, error, _pool, _stacktrace)\u001b[0m\n\u001b[0;32m    581\u001b[0m new_retry \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnew(\n\u001b[0;32m    582\u001b[0m     total\u001b[38;5;241m=\u001b[39mtotal,\n\u001b[0;32m    583\u001b[0m     connect\u001b[38;5;241m=\u001b[39mconnect,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    588\u001b[0m     history\u001b[38;5;241m=\u001b[39mhistory,\n\u001b[0;32m    589\u001b[0m )\n\u001b[0;32m    591\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m new_retry\u001b[38;5;241m.\u001b[39mis_exhausted():\n\u001b[1;32m--> 592\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m MaxRetryError(_pool, url, error \u001b[38;5;129;01mor\u001b[39;00m ResponseError(cause))\n\u001b[0;32m    594\u001b[0m log\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mIncremented Retry for (url=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m): \u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m\"\u001b[39m, url, new_retry)\n\u001b[0;32m    596\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m new_retry\n",
      "\u001b[1;31mMaxRetryError\u001b[0m: HTTPConnectionPool(host='localhost', port=53740): Max retries exceeded with url: /session/e9b1deeba0f182775fc7c6105969ab2f/elements (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x000001A7610A0610>: Failed to establish a new connection: [WinError 10061] No connection could be made because the target machine actively refused it'))"
     ]
    }
   ],
   "source": [
    "# create an empty DataFrame outside the function\n",
    "df = pd.DataFrame()\n",
    "\n",
    "# loop through postcodes\n",
    "for postcode in codes.loc[1:]['Postcode district']:\n",
    "    # call the function and pass the empty DataFrame as an argument\n",
    "    df1 = zoopla_sales(postcode, 'Sales', 'Zoopla', df)\n",
    "    # append the df1 DataFrame to the empty DataFrame\n",
    "    df = pd.concat([df, df1], ignore_index=True)\n",
    "print('------------------------ WEB SCRAPING COMPLETED: OVERALL TOTAL NUMBER OF PROPERTIES SCRAPED IS ' + str(df.shape[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a76aa12b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('ending_Zoopla_Sales.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7047963d",
   "metadata": {},
   "source": [
    "In case the web scraping breaks while running due to network, we can continue the scrapping process from the last postcode as follows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "42071880",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scraping page 1 from WC1B\n",
      "scraping page 2 from WC1B\n",
      "------------------------------- SCRAPING COMPLETED FOR WC1B\n",
      "Total numbers of properties available in WC1B is 13\n",
      "scraping page 1 from WC1E\n",
      "scraping page 2 from WC1E\n",
      "------------------------------- SCRAPING COMPLETED FOR WC1E\n",
      "Total numbers of properties available in WC1E is 21\n",
      "scraping page 1 from WC1H\n",
      "scraping page 2 from WC1H\n",
      "scraping page 3 from WC1H\n",
      "scraping page 4 from WC1H\n",
      "------------------------------- SCRAPING COMPLETED FOR WC1H\n",
      "Total numbers of properties available in WC1H is 40\n",
      "scraping page 1 from WC1N\n",
      "scraping page 2 from WC1N\n",
      "scraping page 3 from WC1N\n",
      "------------------------------- SCRAPING COMPLETED FOR WC1N\n",
      "Total numbers of properties available in WC1N is 28\n",
      "scraping page 1 from WC1V\n",
      "scraping page 2 from WC1V\n",
      "------------------------------- SCRAPING COMPLETED FOR WC1V\n",
      "Total numbers of properties available in WC1V is 10\n",
      "scraping page 1 from WC1X\n",
      "scraping page 2 from WC1X\n",
      "scraping page 3 from WC1X\n",
      "scraping page 4 from WC1X\n",
      "scraping page 5 from WC1X\n",
      "------------------------------- SCRAPING COMPLETED FOR WC1X\n",
      "Total numbers of properties available in WC1X is 69\n",
      "scraping page 1 from WC2A\n",
      "scraping page 2 from WC2A\n",
      "scraping page 3 from WC2A\n",
      "------------------------------- SCRAPING COMPLETED FOR WC2A\n",
      "Total numbers of properties available in WC2A is 25\n",
      "scraping page 1 from WC2B\n",
      "scraping page 2 from WC2B\n",
      "scraping page 3 from WC2B\n",
      "------------------------------- SCRAPING COMPLETED FOR WC2B\n",
      "Total numbers of properties available in WC2B is 36\n",
      "scraping page 1 from WC2E\n",
      "scraping page 2 from WC2E\n",
      "------------------------------- SCRAPING COMPLETED FOR WC2E\n",
      "Total numbers of properties available in WC2E is 19\n",
      "scraping page 1 from WC2H\n",
      "scraping page 2 from WC2H\n",
      "------------------------------- SCRAPING COMPLETED FOR WC2H\n",
      "Total numbers of properties available in WC2H is 17\n",
      "scraping page 1 from WC2N\n",
      "scraping page 2 from WC2N\n",
      "------------------------------- SCRAPING COMPLETED FOR WC2N\n",
      "Total numbers of properties available in WC2N is 22\n",
      "scraping page 1 from WC2R\n",
      "scraping page 2 from WC2R\n",
      "scraping page 3 from WC2R\n",
      "scraping page 4 from WC2R\n",
      "------------------------------- SCRAPING COMPLETED FOR WC2R\n",
      "Total numbers of properties available in WC2R is 53\n",
      "------------------------ WEB SCRAPING COMPLETED: OVERALL TOTAL NUMBER OF PROPERTIES SCRAPED IS 353\n"
     ]
    }
   ],
   "source": [
    "# create an empty DataFrame outside the function\n",
    "df = pd.DataFrame()\n",
    "\n",
    "# loop through postcodes\n",
    "for postcode in codes.loc[256:]['Postcode district']:\n",
    "    # call the function and pass the empty DataFrame as an argument\n",
    "    df1 = zoopla_sales(postcode, 'Sales', 'Zoopla', df)\n",
    "    # append the df1 DataFrame to the empty DataFrame\n",
    "    df = pd.concat([df, df1], ignore_index=True)\n",
    "print('------------------------ WEB SCRAPING COMPLETED: OVERALL TOTAL NUMBER OF PROPERTIES SCRAPED IS ' + str(df.shape[0]))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f06866a2",
   "metadata": {},
   "source": [
    "Saving the dataframe into a csv file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "e44cbdd6",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('Zoopla_Sales_data.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0985bcc9",
   "metadata": {},
   "source": [
    "Loading the dataset and removing an unwanted column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3d96b546",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ZP_sales = pd.read_csv('Zoopla_Sales_data.csv')\n",
    "df_ZP_sales = df_ZP_sales.drop('Unnamed: 0', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f6995853",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(49521, 13)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check the shape of the dataset\n",
    "df_ZP_sales.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6e582ecc",
   "metadata": {},
   "source": [
    "Almost 50,000 properties were scrapped"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "eee88d19",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unique_Id</th>\n",
       "      <th>Location</th>\n",
       "      <th>Tansaction_Type</th>\n",
       "      <th>Property_Type</th>\n",
       "      <th>Address</th>\n",
       "      <th>Bedrooms</th>\n",
       "      <th>Bathrooms</th>\n",
       "      <th>Price</th>\n",
       "      <th>Description</th>\n",
       "      <th>Listing_Date</th>\n",
       "      <th>Agent</th>\n",
       "      <th>Listing_Source</th>\n",
       "      <th>listing_URL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BR1S00001ZP</td>\n",
       "      <td>BR1</td>\n",
       "      <td>Sales</td>\n",
       "      <td>Bishops Avenue, Bromley, Kent BR1</td>\n",
       "      <td>6 bed semi-detached house for sale</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>£1,250,000</td>\n",
       "      <td>Chain free and newly rennovated to an exceptio...</td>\n",
       "      <td>Listed on 18th May 2023</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Zoopla</td>\n",
       "      <td>https://www.zoopla.co.uk/for-sale/property/br1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BR1S00002ZP</td>\n",
       "      <td>BR1</td>\n",
       "      <td>Sales</td>\n",
       "      <td>College Road, Bromley BR1</td>\n",
       "      <td>1 bed flat for sale</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>£255,000</td>\n",
       "      <td>Escape the city hustle with this stunning one-...</td>\n",
       "      <td>Listed on 18th May 2023</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Zoopla</td>\n",
       "      <td>https://www.zoopla.co.uk/for-sale/property/br1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>BR1S00003ZP</td>\n",
       "      <td>BR1</td>\n",
       "      <td>Sales</td>\n",
       "      <td>Marian Gardens, Bromley, Kent BR1</td>\n",
       "      <td>2 bed flat for sale</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>£550,000</td>\n",
       "      <td>Located in the prestigious Hampton Grange deve...</td>\n",
       "      <td>Listed on 18th May 2023</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Zoopla</td>\n",
       "      <td>https://www.zoopla.co.uk/for-sale/property/br1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>BR1S00004ZP</td>\n",
       "      <td>BR1</td>\n",
       "      <td>Sales</td>\n",
       "      <td>Forde Avenue, Bromley, Kent BR1</td>\n",
       "      <td>3 bed terraced house for sale</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>£600,000</td>\n",
       "      <td>A very generously proportioned mid terraced th...</td>\n",
       "      <td>Listed on 18th May 2023</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Zoopla</td>\n",
       "      <td>https://www.zoopla.co.uk/for-sale/property/br1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>BR1S00005ZP</td>\n",
       "      <td>BR1</td>\n",
       "      <td>Sales</td>\n",
       "      <td>Ellen Close, Bickley, Bromley BR1</td>\n",
       "      <td>3 bed semi-detached house for sale</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>£525,000</td>\n",
       "      <td>*Guide Price £525,000 - £550,000* A spacious t...</td>\n",
       "      <td>Listed on 18th May 2023</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Zoopla</td>\n",
       "      <td>https://www.zoopla.co.uk/for-sale/property/br1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49516</th>\n",
       "      <td>WC2RS00049ZP</td>\n",
       "      <td>WC2R</td>\n",
       "      <td>Sales</td>\n",
       "      <td>Strand, Covent Garden, London WC2R</td>\n",
       "      <td>1 bed flat for sale</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>£1,500,000</td>\n",
       "      <td>A one bedroom flat for sale in the luxurious 1...</td>\n",
       "      <td>Listed on 30th Nov 2020</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Zoopla</td>\n",
       "      <td>https://www.zoopla.co.uk/for-sale/property/WC2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49517</th>\n",
       "      <td>WC2RS00050ZP</td>\n",
       "      <td>WC2R</td>\n",
       "      <td>Sales</td>\n",
       "      <td>Strand, Covent Garden WC2R</td>\n",
       "      <td>1 bed flat for sale</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>£1,500,000</td>\n",
       "      <td>A stunning one bedroom, one bathroom apartment...</td>\n",
       "      <td>Listed on 19th Nov 2020</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Zoopla</td>\n",
       "      <td>https://www.zoopla.co.uk/for-sale/property/WC2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49518</th>\n",
       "      <td>WC2RS00051ZP</td>\n",
       "      <td>WC2R</td>\n",
       "      <td>Sales</td>\n",
       "      <td>Wren House, 190 The Strand, London WC2R</td>\n",
       "      <td>Studio for sale</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>£900,000</td>\n",
       "      <td>This incredibly finished apartment, set in a q...</td>\n",
       "      <td>Listed on 17th Nov 2020</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Zoopla</td>\n",
       "      <td>https://www.zoopla.co.uk/for-sale/property/WC2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49519</th>\n",
       "      <td>WC2RS00052ZP</td>\n",
       "      <td>WC2R</td>\n",
       "      <td>Sales</td>\n",
       "      <td>190 Strand, Strand WC2R</td>\n",
       "      <td>4 bed flat for sale</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>£21,300,000</td>\n",
       "      <td>Beautiful Large Penthouse. A New Build Develop...</td>\n",
       "      <td>Listed on 11th Feb 2020</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Zoopla</td>\n",
       "      <td>https://www.zoopla.co.uk/for-sale/property/WC2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49520</th>\n",
       "      <td>WC2RS00053ZP</td>\n",
       "      <td>WC2R</td>\n",
       "      <td>Sales</td>\n",
       "      <td>13 Arundel Street, London City WC2R</td>\n",
       "      <td>2 bed flat for sale</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>£2,100,000</td>\n",
       "      <td>Located on the world famous Strand, 190 Strand...</td>\n",
       "      <td>Listed on 30th Jan 2020</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Zoopla</td>\n",
       "      <td>https://www.zoopla.co.uk/for-sale/property/WC2...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>49521 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          Unique_Id Location Tansaction_Type  \\\n",
       "0       BR1S00001ZP      BR1           Sales   \n",
       "1       BR1S00002ZP      BR1           Sales   \n",
       "2       BR1S00003ZP      BR1           Sales   \n",
       "3       BR1S00004ZP      BR1           Sales   \n",
       "4       BR1S00005ZP      BR1           Sales   \n",
       "...             ...      ...             ...   \n",
       "49516  WC2RS00049ZP     WC2R           Sales   \n",
       "49517  WC2RS00050ZP     WC2R           Sales   \n",
       "49518  WC2RS00051ZP     WC2R           Sales   \n",
       "49519  WC2RS00052ZP     WC2R           Sales   \n",
       "49520  WC2RS00053ZP     WC2R           Sales   \n",
       "\n",
       "                                 Property_Type  \\\n",
       "0            Bishops Avenue, Bromley, Kent BR1   \n",
       "1                    College Road, Bromley BR1   \n",
       "2            Marian Gardens, Bromley, Kent BR1   \n",
       "3              Forde Avenue, Bromley, Kent BR1   \n",
       "4            Ellen Close, Bickley, Bromley BR1   \n",
       "...                                        ...   \n",
       "49516       Strand, Covent Garden, London WC2R   \n",
       "49517               Strand, Covent Garden WC2R   \n",
       "49518  Wren House, 190 The Strand, London WC2R   \n",
       "49519                  190 Strand, Strand WC2R   \n",
       "49520      13 Arundel Street, London City WC2R   \n",
       "\n",
       "                                  Address  Bedrooms  Bathrooms        Price  \\\n",
       "0      6 bed semi-detached house for sale       6.0        2.0   £1,250,000   \n",
       "1                     1 bed flat for sale       1.0        1.0     £255,000   \n",
       "2                     2 bed flat for sale       2.0        1.0     £550,000   \n",
       "3           3 bed terraced house for sale       3.0        1.0     £600,000   \n",
       "4      3 bed semi-detached house for sale       3.0        2.0     £525,000   \n",
       "...                                   ...       ...        ...          ...   \n",
       "49516                 1 bed flat for sale       1.0        1.0   £1,500,000   \n",
       "49517                 1 bed flat for sale       1.0        1.0   £1,500,000   \n",
       "49518                     Studio for sale       1.0        1.0     £900,000   \n",
       "49519                 4 bed flat for sale       4.0        1.0  £21,300,000   \n",
       "49520                 2 bed flat for sale       2.0        1.0   £2,100,000   \n",
       "\n",
       "                                             Description  \\\n",
       "0      Chain free and newly rennovated to an exceptio...   \n",
       "1      Escape the city hustle with this stunning one-...   \n",
       "2      Located in the prestigious Hampton Grange deve...   \n",
       "3      A very generously proportioned mid terraced th...   \n",
       "4      *Guide Price £525,000 - £550,000* A spacious t...   \n",
       "...                                                  ...   \n",
       "49516  A one bedroom flat for sale in the luxurious 1...   \n",
       "49517  A stunning one bedroom, one bathroom apartment...   \n",
       "49518  This incredibly finished apartment, set in a q...   \n",
       "49519  Beautiful Large Penthouse. A New Build Develop...   \n",
       "49520  Located on the world famous Strand, 190 Strand...   \n",
       "\n",
       "                  Listing_Date  Agent Listing_Source  \\\n",
       "0      Listed on 18th May 2023    NaN         Zoopla   \n",
       "1      Listed on 18th May 2023    NaN         Zoopla   \n",
       "2      Listed on 18th May 2023    NaN         Zoopla   \n",
       "3      Listed on 18th May 2023    NaN         Zoopla   \n",
       "4      Listed on 18th May 2023    NaN         Zoopla   \n",
       "...                        ...    ...            ...   \n",
       "49516  Listed on 30th Nov 2020    NaN         Zoopla   \n",
       "49517  Listed on 19th Nov 2020    NaN         Zoopla   \n",
       "49518  Listed on 17th Nov 2020    NaN         Zoopla   \n",
       "49519  Listed on 11th Feb 2020    NaN         Zoopla   \n",
       "49520  Listed on 30th Jan 2020    NaN         Zoopla   \n",
       "\n",
       "                                             listing_URL  \n",
       "0      https://www.zoopla.co.uk/for-sale/property/br1...  \n",
       "1      https://www.zoopla.co.uk/for-sale/property/br1...  \n",
       "2      https://www.zoopla.co.uk/for-sale/property/br1...  \n",
       "3      https://www.zoopla.co.uk/for-sale/property/br1...  \n",
       "4      https://www.zoopla.co.uk/for-sale/property/br1...  \n",
       "...                                                  ...  \n",
       "49516  https://www.zoopla.co.uk/for-sale/property/WC2...  \n",
       "49517  https://www.zoopla.co.uk/for-sale/property/WC2...  \n",
       "49518  https://www.zoopla.co.uk/for-sale/property/WC2...  \n",
       "49519  https://www.zoopla.co.uk/for-sale/property/WC2...  \n",
       "49520  https://www.zoopla.co.uk/for-sale/property/WC2...  \n",
       "\n",
       "[49521 rows x 13 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ZP_sales"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e72edbcd",
   "metadata": {},
   "source": [
    "<h1 style='background-color: BLACK; padding: 10px; color: white'> Rent </h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6c5be01",
   "metadata": {},
   "source": [
    "Now, we will create a function called 'zoopla_rent' to scrap properties that are for rent on the Zoopla website"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "91431634",
   "metadata": {},
   "outputs": [],
   "source": [
    "def zoopla_rent(postcodes, Trans_type, website, df):\n",
    "    \"\"\"\n",
    "    This function scrapes property data from Rightmove for a given list of postcodes and transaction type,\n",
    "    and returns the data as a pandas DataFrame.\n",
    "\n",
    "    Arguments:\n",
    "    postcodes -- a list of postcodes for which to scrape property data\n",
    "    Trans_type -- the transaction type of the properties to be scraped ('sales' or 'rent')\n",
    "    website -- the name of the website being scraped (in this case, 'Rightmove')\n",
    "    df -- an empty pandas DataFrame to store the scraped data\n",
    "\n",
    "    Returns:\n",
    "    df1 -- a pandas DataFrame containing the scraped property data\n",
    "    \n",
    "    \"\"\"    \n",
    "    # Set the path to the ChromeDriver executable\n",
    "    s = Service(\"C:\\\\Users\\\\user\\\\Downloads\\\\Set ups\\\\chromedriver_win32\\\\chromedriver.exe\")\n",
    "    \n",
    "    # Launch the ChromeDriver with the specified service\n",
    "    driver = webdriver.Chrome(service= s)\n",
    "    \n",
    "    # Navigate to the specified URL \n",
    "    driver.get('https://www.zoopla.co.uk/') \n",
    "\n",
    "    # Maximize the window\n",
    "    time.sleep(2)\n",
    "    driver.maximize_window()\n",
    "    \n",
    "    #Click the rent button\n",
    "    time.sleep(1.5)   \n",
    "    driver.find_element(By.XPATH, \"//button[contains(@id,'radix-:Reckt6:-trigger-to-rent')]\").click()\n",
    "\n",
    "    # Find the search bar  \n",
    "    time.sleep(2.2)\n",
    "    #search = driver.find_element(By.XPATH, '/html/body/div[3]/div/div[1]/main/div[1]/div/div/div[1]/div[2]/div/div/div[2]/div/form/div/div[1]/div/div/div/div/div/div/input')\n",
    "    try:\n",
    "        search = driver.find_element(By.XPATH, \"//input[contains(@id,'downshift')]\")\n",
    "        search.send_keys(postcode)\n",
    "        time.sleep(1.3)\n",
    "    except:\n",
    "        search = driver.find_element(By.XPATH, \"//input[contains(@class,'_1qzmny55 _1ftx2fq8')]\")\n",
    "        search.send_keys(postcode)\n",
    "        time.sleep(1.3)\n",
    "    \n",
    "    \n",
    "    # Click the search button\n",
    "    time.sleep(1.5)   \n",
    "    driver.find_element(By.XPATH, \"//button[@class='x8jo560 x8jo562 x8jo56a _1ftx2fq8'][1]\").click()\n",
    "    \n",
    "    \n",
    "    \n",
    "    # Initialize empty lists for storing scraped data\n",
    "    Trans_type = []\n",
    "    address = []\n",
    "    types = []\n",
    "    bedrooms = []\n",
    "    bathrooms = []\n",
    "    prices = []\n",
    "    desc = []\n",
    "    date_added = []\n",
    "    agent_list = []\n",
    "    property_url = []\n",
    "    website = []\n",
    "    \n",
    "    # Create an empty dataframe to store the scraped data\n",
    "    df = pd.DataFrame()\n",
    "    \n",
    "     # Setting the page number to be 1 \n",
    "    i = 1\n",
    "    while True: \n",
    "        # Scrapping data for the required features in the first page\n",
    "        time.sleep(1) \n",
    "        print(\"{} {} {} {}\".format('scraping page', i,'from', postcode ))\n",
    "        \n",
    "        time.sleep(2.1)\n",
    "        try: \n",
    "            driver.find_element(By.XPATH, \"//*[contains(@class,'u94mg1 u94mg4 u94mg9')]\").click()\n",
    "            time.sleep(1.5)\n",
    "            \n",
    "        except:\n",
    "            time.sleep(1.0)\n",
    "            \n",
    "        time.sleep(1.4)    \n",
    "        address_list =driver.find_elements(By.XPATH, \"//h2[contains(@class,'_1ankud51 _1ftx2fq8')]\")\n",
    "        type_list =driver.find_elements(By.XPATH, \"//h3[contains(@class,'_1ankud52 _1ftx2fq9')]\")\n",
    "        bedroom_list =driver.find_elements(By.XPATH, \"//div[contains(@id,'listing')]//li[1]//span[2]\")\n",
    "        bathroom_list =driver.find_elements(By.XPATH, \"//div[contains(@id,'listing')]//li[3]//span[2]\")\n",
    "        price_list =driver.find_elements(By.XPATH, \"//p[contains(@class,'_170k6632 _1ftx2fq6')]\")\n",
    "        desc_list =driver.find_elements(By.XPATH, \"//p[contains(@class,'_1ankud53 _1ftx2fq9')]\")\n",
    "        date_added_list =driver.find_elements(By.XPATH, \"//li[contains(@class,'_18cib8e1')]\")\n",
    "        agent_list_list =driver.find_elements(By.XPATH, \"//img[contains(@class,'_12bxhf70')]\")\n",
    "        property_url_list = driver.current_url\n",
    "        Trans_type_list = Trans_type\n",
    "        website_list = website\n",
    "        for address_item, type_item, bedroom_item, bathroom_item, price_item, desc_item, date_added_item, agent_list_item in zip(address_list, type_list, bedroom_list, bathroom_list, price_list, desc_list, date_added_list, agent_list_list):\n",
    "            address.append(address_item.text)\n",
    "            types.append(type_item.text)\n",
    "            bedrooms.append(bedroom_item.text)\n",
    "            bathrooms.append(bathroom_item.text)\n",
    "            prices.append(price_item.text)\n",
    "            desc.append(desc_item.text)\n",
    "            date_added.append(date_added_item.text)\n",
    "            agent_list.append(agent_list_item.get_attribute('alt'))\n",
    "            property_url.append(property_url_list)\n",
    "            Trans_type.append(Trans_type_list)\n",
    "            website.append(website_list)\n",
    "\n",
    "\n",
    "        time.sleep(1.3)\n",
    "        # get the height of the page\n",
    "        page_height = driver.execute_script(\"return document.body.scrollHeight;\")\n",
    "\n",
    "        \n",
    "        # Cancel the pop-up window\n",
    "        j= i + 1        \n",
    "        url = f'https://www.zoopla.co.uk/to-rent/property/{postcode}/?price_frequency=per_month&q={postcode}&search_source=home&pn={j}'\n",
    "        if len(address_list) or len(type_list) != 0:\n",
    "            driver.get(url)\n",
    "        else:\n",
    "            print('------------------------------- SCRAPING COMPLETED FOR ' + postcode)\n",
    "            break\n",
    "        \n",
    "            \n",
    "        #Increment the page number\n",
    "        i += 1\n",
    "        time.sleep(1)\n",
    "                \n",
    "        \n",
    "    # Create a dataframe to store data scrapped for each postcode    \n",
    "    df1 = pd.DataFrame({'Location': postcode, 'Tansaction_Type': 'Rent','Property_Type':types, 'Address' :address, 'Bedrooms': bedrooms, 'Bathrooms':bathrooms, 'Price':prices, 'Description': desc, 'Listing_Date':date_added, 'Agent':agent_list, 'Listing_Source': 'Zoopla', 'listing_URL':property_url})\n",
    "    df1.insert(0, 'Unique_Id', [f'{postcode}R{i+1:05d}ZP' for i in range(len(df1))])\n",
    "    print('Total numbers of properties available in ' + postcode + ' is ' + str(df1.shape[0]))\n",
    "\n",
    "     # Concat the dataframe obtain for all postcodes\n",
    "    df = pd.concat([df, df1], ignore_index=True)\n",
    "    \n",
    "     # Return a dataframe\n",
    "    return df1\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff7c7ff6",
   "metadata": {},
   "source": [
    "Now, the data for properties available for rent at Zoopla is scrapped"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "aae6dc33",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scraping page 1 from BR1\n",
      "scraping page 2 from BR1\n",
      "scraping page 3 from BR1\n",
      "scraping page 4 from BR1\n",
      "------------------------------- SCRAPING COMPLETED FOR BR1\n",
      "Total numbers of properties available in BR1 is 54\n",
      "scraping page 1 from BR2\n",
      "scraping page 2 from BR2\n",
      "scraping page 3 from BR2\n",
      "------------------------------- SCRAPING COMPLETED FOR BR2\n",
      "Total numbers of properties available in BR2 is 42\n",
      "scraping page 1 from BR3\n",
      "scraping page 2 from BR3\n",
      "scraping page 3 from BR3\n",
      "------------------------------- SCRAPING COMPLETED FOR BR3\n",
      "Total numbers of properties available in BR3 is 28\n",
      "scraping page 1 from BR4\n",
      "------------------------------- SCRAPING COMPLETED FOR BR4\n",
      "Total numbers of properties available in BR4 is 0\n",
      "scraping page 1 from BR5\n",
      "scraping page 2 from BR5\n",
      "------------------------------- SCRAPING COMPLETED FOR BR5\n",
      "Total numbers of properties available in BR5 is 6\n",
      "scraping page 1 from BR6\n",
      "scraping page 2 from BR6\n",
      "------------------------------- SCRAPING COMPLETED FOR BR6\n",
      "Total numbers of properties available in BR6 is 12\n",
      "scraping page 1 from BR7\n",
      "scraping page 2 from BR7\n",
      "------------------------------- SCRAPING COMPLETED FOR BR7\n",
      "Total numbers of properties available in BR7 is 20\n",
      "scraping page 1 from BR8\n",
      "scraping page 2 from BR8\n",
      "------------------------------- SCRAPING COMPLETED FOR BR8\n",
      "Total numbers of properties available in BR8 is 17\n",
      "scraping page 1 from CR0\n",
      "scraping page 2 from CR0\n",
      "scraping page 3 from CR0\n",
      "scraping page 4 from CR0\n",
      "scraping page 5 from CR0\n",
      "scraping page 6 from CR0\n",
      "scraping page 7 from CR0\n",
      "scraping page 8 from CR0\n",
      "scraping page 9 from CR0\n",
      "scraping page 10 from CR0\n",
      "scraping page 11 from CR0\n",
      "scraping page 12 from CR0\n",
      "scraping page 13 from CR0\n",
      "scraping page 14 from CR0\n",
      "------------------------------- SCRAPING COMPLETED FOR CR0\n",
      "Total numbers of properties available in CR0 is 207\n",
      "scraping page 1 from CR2\n",
      "------------------------------- SCRAPING COMPLETED FOR CR2\n",
      "Total numbers of properties available in CR2 is 0\n"
     ]
    },
    {
     "ename": "NoSuchWindowException",
     "evalue": "Message: no such window: target window already closed\nfrom unknown error: web view not found\n  (Session info: chrome=113.0.5672.127)\nStacktrace:\nBacktrace:\n\tGetHandleVerifier [0x00B18893+48451]\n\t(No symbol) [0x00AAB8A1]\n\t(No symbol) [0x009B5058]\n\t(No symbol) [0x0099D073]\n\t(No symbol) [0x009FDEBB]\n\t(No symbol) [0x00A0BFD3]\n\t(No symbol) [0x009FA0B6]\n\t(No symbol) [0x009D7E08]\n\t(No symbol) [0x009D8F2D]\n\tGetHandleVerifier [0x00D78E3A+2540266]\n\tGetHandleVerifier [0x00DB8959+2801161]\n\tGetHandleVerifier [0x00DB295C+2776588]\n\tGetHandleVerifier [0x00BA2280+612144]\n\t(No symbol) [0x00AB4F6C]\n\t(No symbol) [0x00AB11D8]\n\t(No symbol) [0x00AB12BB]\n\t(No symbol) [0x00AA4857]\n\tBaseThreadInitThunk [0x750A00C9+25]\n\tRtlGetAppContainerNamedObjectPath [0x76FE7B4E+286]\n\tRtlGetAppContainerNamedObjectPath [0x76FE7B1E+238]\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNoSuchWindowException\u001b[0m                     Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[6], line 7\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[38;5;66;03m# loop through postcodes\u001b[39;00m\n\u001b[0;32m      5\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m postcode \u001b[38;5;129;01min\u001b[39;00m codes[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mPostcode district\u001b[39m\u001b[38;5;124m'\u001b[39m]:\n\u001b[0;32m      6\u001b[0m     \u001b[38;5;66;03m# call the function and pass the empty DataFrame as an argument\u001b[39;00m\n\u001b[1;32m----> 7\u001b[0m     df1 \u001b[38;5;241m=\u001b[39m \u001b[43mzoopla_rent\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpostcode\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mRent\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mZoopla\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdf\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m      8\u001b[0m     \u001b[38;5;66;03m# append the df1 DataFrame to the empty DataFrame\u001b[39;00m\n\u001b[0;32m      9\u001b[0m     df \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mconcat([df, df1], ignore_index\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
      "Cell \u001b[1;32mIn[3], line 27\u001b[0m, in \u001b[0;36mzoopla_rent\u001b[1;34m(postcodes, Trans_type, website, df)\u001b[0m\n\u001b[0;32m     25\u001b[0m \u001b[38;5;66;03m# Maximize the window\u001b[39;00m\n\u001b[0;32m     26\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m2\u001b[39m)\n\u001b[1;32m---> 27\u001b[0m \u001b[43mdriver\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmaximize_window\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     29\u001b[0m \u001b[38;5;66;03m#Click the rent button\u001b[39;00m\n\u001b[0;32m     30\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m1.5\u001b[39m)   \n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py:592\u001b[0m, in \u001b[0;36mWebDriver.maximize_window\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    590\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Maximizes the current window that webdriver is using.\"\"\"\u001b[39;00m\n\u001b[0;32m    591\u001b[0m command \u001b[38;5;241m=\u001b[39m Command\u001b[38;5;241m.\u001b[39mW3C_MAXIMIZE_WINDOW\n\u001b[1;32m--> 592\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcommand\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py:440\u001b[0m, in \u001b[0;36mWebDriver.execute\u001b[1;34m(self, driver_command, params)\u001b[0m\n\u001b[0;32m    438\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcommand_executor\u001b[38;5;241m.\u001b[39mexecute(driver_command, params)\n\u001b[0;32m    439\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m response:\n\u001b[1;32m--> 440\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43merror_handler\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcheck_response\u001b[49m\u001b[43m(\u001b[49m\u001b[43mresponse\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    441\u001b[0m     response[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mvalue\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_unwrap_value(response\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mvalue\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m))\n\u001b[0;32m    442\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m response\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\selenium\\webdriver\\remote\\errorhandler.py:245\u001b[0m, in \u001b[0;36mErrorHandler.check_response\u001b[1;34m(self, response)\u001b[0m\n\u001b[0;32m    243\u001b[0m         alert_text \u001b[38;5;241m=\u001b[39m value[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124malert\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m    244\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m exception_class(message, screen, stacktrace, alert_text)  \u001b[38;5;66;03m# type: ignore[call-arg]  # mypy is not smart enough here\u001b[39;00m\n\u001b[1;32m--> 245\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m exception_class(message, screen, stacktrace)\n",
      "\u001b[1;31mNoSuchWindowException\u001b[0m: Message: no such window: target window already closed\nfrom unknown error: web view not found\n  (Session info: chrome=113.0.5672.127)\nStacktrace:\nBacktrace:\n\tGetHandleVerifier [0x00B18893+48451]\n\t(No symbol) [0x00AAB8A1]\n\t(No symbol) [0x009B5058]\n\t(No symbol) [0x0099D073]\n\t(No symbol) [0x009FDEBB]\n\t(No symbol) [0x00A0BFD3]\n\t(No symbol) [0x009FA0B6]\n\t(No symbol) [0x009D7E08]\n\t(No symbol) [0x009D8F2D]\n\tGetHandleVerifier [0x00D78E3A+2540266]\n\tGetHandleVerifier [0x00DB8959+2801161]\n\tGetHandleVerifier [0x00DB295C+2776588]\n\tGetHandleVerifier [0x00BA2280+612144]\n\t(No symbol) [0x00AB4F6C]\n\t(No symbol) [0x00AB11D8]\n\t(No symbol) [0x00AB12BB]\n\t(No symbol) [0x00AA4857]\n\tBaseThreadInitThunk [0x750A00C9+25]\n\tRtlGetAppContainerNamedObjectPath [0x76FE7B4E+286]\n\tRtlGetAppContainerNamedObjectPath [0x76FE7B1E+238]\n"
     ]
    }
   ],
   "source": [
    "# create an empty DataFrame outside the function\n",
    "df = pd.DataFrame()\n",
    "\n",
    "# loop through postcodes\n",
    "for postcode in codes['Postcode district']:\n",
    "    # call the function and pass the empty DataFrame as an argument\n",
    "    df1 = zoopla_rent(postcode, 'Rent', 'Zoopla', df)\n",
    "    # append the df1 DataFrame to the empty DataFrame\n",
    "    df = pd.concat([df, df1], ignore_index=True)\n",
    "print('------------------------ WEB SCRAPING COMPLETED: OVERALL TOTAL NUMBER OF PROPERTIES SCRAPED IS ' + str(df.shape[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1ba2a15",
   "metadata": {},
   "source": [
    "In case the web scraping breaks while running due to network, we can continue the scrapping process from the last postcode as follows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "29b459a5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scraping page 1 from WC1X\n",
      "scraping page 2 from WC1X\n",
      "scraping page 3 from WC1X\n",
      "scraping page 4 from WC1X\n",
      "scraping page 5 from WC1X\n",
      "------------------------------- SCRAPING COMPLETED FOR WC1X\n",
      "Total numbers of properties available in WC1X is 56\n",
      "scraping page 1 from WC2A\n",
      "scraping page 2 from WC2A\n",
      "scraping page 3 from WC2A\n",
      "------------------------------- SCRAPING COMPLETED FOR WC2A\n",
      "Total numbers of properties available in WC2A is 35\n",
      "scraping page 1 from WC2B\n",
      "scraping page 2 from WC2B\n",
      "scraping page 3 from WC2B\n",
      "------------------------------- SCRAPING COMPLETED FOR WC2B\n",
      "Total numbers of properties available in WC2B is 25\n",
      "scraping page 1 from WC2E\n",
      "scraping page 2 from WC2E\n",
      "scraping page 3 from WC2E\n",
      "------------------------------- SCRAPING COMPLETED FOR WC2E\n",
      "Total numbers of properties available in WC2E is 36\n",
      "scraping page 1 from WC2H\n",
      "scraping page 2 from WC2H\n",
      "scraping page 3 from WC2H\n",
      "scraping page 4 from WC2H\n",
      "------------------------------- SCRAPING COMPLETED FOR WC2H\n",
      "Total numbers of properties available in WC2H is 29\n",
      "scraping page 1 from WC2N\n",
      "scraping page 2 from WC2N\n",
      "scraping page 3 from WC2N\n",
      "------------------------------- SCRAPING COMPLETED FOR WC2N\n",
      "Total numbers of properties available in WC2N is 24\n",
      "scraping page 1 from WC2R\n",
      "scraping page 2 from WC2R\n",
      "scraping page 3 from WC2R\n",
      "------------------------------- SCRAPING COMPLETED FOR WC2R\n",
      "Total numbers of properties available in WC2R is 30\n",
      "------------------------ WEB SCRAPING COMPLETED: OVERALL TOTAL NUMBER OF PROPERTIES SCRAPED IS 235\n"
     ]
    }
   ],
   "source": [
    "# create an empty DataFrame outside the function\n",
    "df = pd.DataFrame()\n",
    "\n",
    "# loop through postcodes\n",
    "for postcode in codes.loc[261:]['Postcode district']:\n",
    "    # call the function and pass the empty DataFrame as an argument\n",
    "    df1 = zoopla_rent(postcode, 'Rent', 'Zoopla', df)\n",
    "    # append the df1 DataFrame to the empty DataFrame\n",
    "    df = pd.concat([df, df1], ignore_index=True)\n",
    "print('------------------------ WEB SCRAPING COMPLETED: OVERALL TOTAL NUMBER OF PROPERTIES SCRAPED IS ' + str(df.shape[0]))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4c940ab8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('Zoopla_rent-data2.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "dcd63f68",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Postcode district</th>\n",
       "      <th>Local Areas</th>\n",
       "      <th>Borough</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>261</th>\n",
       "      <td>WC1X</td>\n",
       "      <td>Kings Cross, Finsbury (west)</td>\n",
       "      <td>Camden</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>262</th>\n",
       "      <td>WC2A</td>\n",
       "      <td>Lincoln's Inn Fields, Royal Courts of Justice</td>\n",
       "      <td>Camden</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>263</th>\n",
       "      <td>WC2B</td>\n",
       "      <td>Drury Lane, Aldwych</td>\n",
       "      <td>Camden</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264</th>\n",
       "      <td>WC2E</td>\n",
       "      <td>Covent Garden</td>\n",
       "      <td>Westminster</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>265</th>\n",
       "      <td>WC2H</td>\n",
       "      <td>Leicester Square</td>\n",
       "      <td>Westminster</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266</th>\n",
       "      <td>WC2N</td>\n",
       "      <td>Charing Cross</td>\n",
       "      <td>Westminster</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>267</th>\n",
       "      <td>WC2R</td>\n",
       "      <td>Somerset House</td>\n",
       "      <td>Westminster</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Postcode district                                    Local Areas   \n",
       "261              WC1X                   Kings Cross, Finsbury (west)  \\\n",
       "262              WC2A  Lincoln's Inn Fields, Royal Courts of Justice   \n",
       "263              WC2B                            Drury Lane, Aldwych   \n",
       "264              WC2E                                  Covent Garden   \n",
       "265              WC2H                               Leicester Square   \n",
       "266              WC2N                                  Charing Cross   \n",
       "267              WC2R                                 Somerset House   \n",
       "\n",
       "         Borough  \n",
       "261       Camden  \n",
       "262       Camden  \n",
       "263       Camden  \n",
       "264  Westminster  \n",
       "265  Westminster  \n",
       "266  Westminster  \n",
       "267  Westminster  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = codes.loc[261:]\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "59058ee2",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}