DELHI.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Delhi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import html_to_json\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from selenium.webdriver.common.action_chains import ActionChains\n",
    "import selenium\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "\n",
    "import time\n",
    "import re\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "homepage = 'https://vegetablemarketprice.com/'\n",
    "data = []\n",
    "html_list = []\n",
    "driver = webdriver.Firefox()\n",
    "driver.get(homepage)\n",
    "search_input = driver.find_element(by=By.XPATH, value='//*[@id=\"header-search-input\"]')\n",
    "\n",
    "header_search_input = 'Delhi Vegetable Market Price History'\n",
    "search_input.send_keys(header_search_input)\n",
    "search_input.send_keys(Keys.ENTER)\n",
    "time.sleep(5)\n",
    "driver.find_element(by=By.XPATH,value='/html/body/ul/li[1]/div').click()\n",
    "\n",
    "\n",
    "switch = driver.find_element(By.XPATH, '//*[@id=\"navItemTable\"]')\n",
    "switch.click()\n",
    "\n",
    "change_date = driver.find_element(by = By.XPATH,value = '//*[@id=\"reportrange\"]')\n",
    "\n",
    "change_date.click()\n",
    "custom_range = driver.find_element(By.XPATH, \"//*[contains(text(), 'Custom Range')]\")\n",
    "custom_range.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialzing the dates\n",
    "while True: \n",
    "    check_month = driver.find_element(By.XPATH, \"/html/body/div[4]/div[2]/div[1]/table/thead/tr[1]/th[2]\")\n",
    "    month = check_month.get_attribute(\"innerHTML\")\n",
    "    if month == \"Feb 2022\":\n",
    "        break\n",
    "        \n",
    "    else:\n",
    "        back_button = driver.find_element(By.XPATH,'/html/body/div[4]/div[2]/div[1]/table/thead/tr[1]/th[1]')\n",
    "\n",
    "        back_button.click()\n",
    "        \n",
    "first_date = driver.find_element(By.XPATH, '/html/body/div[4]/div[2]/div[1]/table/tbody/tr[1]/td[3]')\n",
    "first_date.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# selecting date from feb to july\n",
    "i=0\n",
    "while i <= 2:\n",
    "    front_button = driver.find_element(By.XPATH,\"/html/body/div[4]/div[3]/div[1]/table/thead/tr[1]/th[3]\")\n",
    "    front_button.click()\n",
    "    i = i+1\n",
    "\n",
    "last_date = driver.find_element(By.XPATH,\"/html/body/div[4]/div[3]/div[1]/table/tbody/tr[5]/td[7]\")\n",
    "last_date.click()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "apply_button = driver.find_element(By.XPATH, '/html/body/div[4]/div[4]/button[2]')\n",
    "driver.execute_script(\"arguments[0].scrollIntoView();\", apply_button)\n",
    "apply_button.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_list = ['Onion Big', 'Onion Small', 'Tomato', 'Green Chilli', 'Beetroot', 'Potato',\n",
    "    'Raw Banana (Plantain)', 'Amaranth Leaves', 'Amla', 'Ash gourd', 'Baby Corn',\n",
    "    'Banana Flower', 'Capsicum', 'Bitter Gourd', 'Bottle Gourd', 'Butter Beans',\n",
    "    'Broad Beans', 'Cabbage', 'Carrot', 'Cauliflower', 'Cluster beans', 'Coconut',\n",
    "    'Colocasia Leaves', 'Colocasia', 'Coriander Leaves', 'Corn', 'Cucumber',\n",
    "    'Curry Leaves', 'Dill Leaves', 'Drumsticks', 'Brinjal', 'Brinjal (Big)',\n",
    "    'Elephant Yam', 'Fenugreek Leaves', 'French Beans', 'Garlic', 'Ginger',\n",
    "    'Onion Green', 'Green Peas', 'Ivy Gourd', 'Lemon (Lime)', 'Mango Raw',\n",
    "    'Mint Leaves', 'Mushroom', 'Mustard Leaves', 'Ladies Finger', 'Pumpkin',\n",
    "    'Radish', 'Ridge Gourd', 'Shallot (Pearl Onion)', 'Snake Gourd', 'Sorrel Leaves',\n",
    "    'Spinach', 'Sweet Potato']\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Onion Big\n",
      "Onion Small\n",
      "Tomato\n",
      "Green Chilli\n",
      "Beetroot\n",
      "Potato\n",
      "Raw Banana (Plantain)\n",
      "Amaranth Leaves\n",
      "Amla\n",
      "Ash gourd\n",
      "Baby Corn\n",
      "Banana Flower\n",
      "Capsicum\n",
      "Bitter Gourd\n",
      "Bottle Gourd\n",
      "Butter Beans\n",
      "Broad Beans\n",
      "Cabbage\n",
      "Carrot\n",
      "Cauliflower\n",
      "Cluster beans\n",
      "Coconut\n",
      "Colocasia Leaves\n",
      "Colocasia\n",
      "Coriander Leaves\n",
      "Corn\n",
      "Cucumber\n",
      "Curry Leaves\n",
      "Dill Leaves\n",
      "Drumsticks\n",
      "Brinjal\n",
      "Brinjal (Big)\n",
      "Elephant Yam\n",
      "Fenugreek Leaves\n",
      "French Beans\n",
      "Garlic\n",
      "Ginger\n",
      "Onion Green\n",
      "Green Peas\n",
      "Ivy Gourd\n",
      "Lemon (Lime)\n",
      "Mango Raw\n",
      "Mint Leaves\n",
      "Mushroom\n",
      "Mustard Leaves\n",
      "Ladies Finger\n",
      "Pumpkin\n",
      "Radish\n",
      "Ridge Gourd\n",
      "Shallot (Pearl Onion)\n",
      "Snake Gourd\n",
      "Sorrel Leaves\n",
      "Spinach\n",
      "Sweet Potato\n"
     ]
    }
   ],
   "source": [
    "for i in range (0,len(final_list)):\n",
    "    while True:\n",
    "        try:\n",
    "            element = driver.find_element(By.XPATH, '/html/body/div[2]/div[2]/div/div[1]/div[4]/div/div[1]/div/div[1]/button')\n",
    "            element.click()\n",
    "        except:\n",
    "            break\n",
    "    search_input = driver.find_element(By.NAME, 'search_terms')\n",
    "    search_input.send_keys(final_list[i])\n",
    "    time.sleep(3)\n",
    "    search_input.send_keys(Keys.ENTER)\n",
    "    \n",
    "    temp = driver.find_element(By.XPATH, '//*[@id=\"historyDataTable\"]')\n",
    "    time.sleep(3)\n",
    "    temp_html = temp.get_attribute(\"innerHTML\")\n",
    "    html_list.append(temp_html)\n",
    "    time.sleep(3)\n",
    "    print(final_list[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "date = re.findall('<th>(.*?)</th>',html_list[0])\n",
    "dates = []\n",
    "for i in date:\n",
    "    temp = re.findall('\"white-space: nowrap;\">(.*?)</span>',i)\n",
    "    if len(temp) != 0:\n",
    "        dates.append(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "Name = []\n",
    "Prices = []\n",
    "\n",
    "for i in html_list:\n",
    "    temp_n = re.findall('<tbody><tr><td>(.*?)</td><td>',i)\n",
    "    Name.extend(temp_n)\n",
    "    \n",
    "    temp_p = re.findall('<td>₹ (.*?)</td>',i)\n",
    "    Prices.append(temp_p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "#converting into dataframe \n",
    "df = pd.DataFrame(dates,columns = ['Dates'])\n",
    "df1 = pd.DataFrame(Prices)\n",
    "df1 = df1.transpose()\n",
    "df1.dropna(inplace = True)\n",
    "df1.columns = [Name]\n",
    "df = pd.merge(df,df1,left_index=True, right_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Dates</th>\n",
       "      <th>(Onion Big,)</th>\n",
       "      <th>(Onion Small,)</th>\n",
       "      <th>(Tomato,)</th>\n",
       "      <th>(Green Chilli,)</th>\n",
       "      <th>(Beetroot,)</th>\n",
       "      <th>(Potato,)</th>\n",
       "      <th>(Raw Banana (Plantain),)</th>\n",
       "      <th>(Amaranth Leaves,)</th>\n",
       "      <th>(Amla,)</th>\n",
       "      <th>...</th>\n",
       "      <th>(Mustard Leaves,)</th>\n",
       "      <th>(Ladies Finger,)</th>\n",
       "      <th>(Pumpkin,)</th>\n",
       "      <th>(Radish,)</th>\n",
       "      <th>(Ridge Gourd,)</th>\n",
       "      <th>(Shallot (Pearl Onion),)</th>\n",
       "      <th>(Snake Gourd,)</th>\n",
       "      <th>(Sorrel Leaves,)</th>\n",
       "      <th>(Spinach,)</th>\n",
       "      <th>(Sweet Potato,)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-02-01</td>\n",
       "      <td>36</td>\n",
       "      <td>62</td>\n",
       "      <td>19</td>\n",
       "      <td>42</td>\n",
       "      <td>49</td>\n",
       "      <td>22</td>\n",
       "      <td>5</td>\n",
       "      <td>15</td>\n",
       "      <td>60</td>\n",
       "      <td>...</td>\n",
       "      <td>20</td>\n",
       "      <td>59</td>\n",
       "      <td>26</td>\n",
       "      <td>24</td>\n",
       "      <td>30</td>\n",
       "      <td>34</td>\n",
       "      <td>25</td>\n",
       "      <td>13</td>\n",
       "      <td>12</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2022-02-02</td>\n",
       "      <td>39</td>\n",
       "      <td>63</td>\n",
       "      <td>17</td>\n",
       "      <td>36</td>\n",
       "      <td>50</td>\n",
       "      <td>21</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>60</td>\n",
       "      <td>...</td>\n",
       "      <td>15</td>\n",
       "      <td>57</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>30</td>\n",
       "      <td>36</td>\n",
       "      <td>25</td>\n",
       "      <td>11</td>\n",
       "      <td>15</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-02-03</td>\n",
       "      <td>39</td>\n",
       "      <td>63</td>\n",
       "      <td>17</td>\n",
       "      <td>36</td>\n",
       "      <td>50</td>\n",
       "      <td>21</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>60</td>\n",
       "      <td>...</td>\n",
       "      <td>15</td>\n",
       "      <td>57</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>30</td>\n",
       "      <td>36</td>\n",
       "      <td>25</td>\n",
       "      <td>11</td>\n",
       "      <td>15</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2022-02-04</td>\n",
       "      <td>39</td>\n",
       "      <td>63</td>\n",
       "      <td>17</td>\n",
       "      <td>36</td>\n",
       "      <td>50</td>\n",
       "      <td>21</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>60</td>\n",
       "      <td>...</td>\n",
       "      <td>15</td>\n",
       "      <td>57</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>30</td>\n",
       "      <td>36</td>\n",
       "      <td>25</td>\n",
       "      <td>11</td>\n",
       "      <td>15</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2022-02-05</td>\n",
       "      <td>38</td>\n",
       "      <td>63</td>\n",
       "      <td>21</td>\n",
       "      <td>45</td>\n",
       "      <td>48</td>\n",
       "      <td>26</td>\n",
       "      <td>4</td>\n",
       "      <td>12</td>\n",
       "      <td>60</td>\n",
       "      <td>...</td>\n",
       "      <td>17</td>\n",
       "      <td>58</td>\n",
       "      <td>21</td>\n",
       "      <td>21</td>\n",
       "      <td>33</td>\n",
       "      <td>31</td>\n",
       "      <td>28</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>175</th>\n",
       "      <td>2022-07-26</td>\n",
       "      <td>25</td>\n",
       "      <td>34</td>\n",
       "      <td>17</td>\n",
       "      <td>33</td>\n",
       "      <td>45</td>\n",
       "      <td>29</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>60</td>\n",
       "      <td>...</td>\n",
       "      <td>12</td>\n",
       "      <td>33</td>\n",
       "      <td>20</td>\n",
       "      <td>24</td>\n",
       "      <td>30</td>\n",
       "      <td>37</td>\n",
       "      <td>29</td>\n",
       "      <td>13</td>\n",
       "      <td>12</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>176</th>\n",
       "      <td>2022-07-27</td>\n",
       "      <td>25</td>\n",
       "      <td>34</td>\n",
       "      <td>17</td>\n",
       "      <td>33</td>\n",
       "      <td>45</td>\n",
       "      <td>29</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>60</td>\n",
       "      <td>...</td>\n",
       "      <td>12</td>\n",
       "      <td>33</td>\n",
       "      <td>20</td>\n",
       "      <td>24</td>\n",
       "      <td>30</td>\n",
       "      <td>37</td>\n",
       "      <td>29</td>\n",
       "      <td>13</td>\n",
       "      <td>12</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177</th>\n",
       "      <td>2022-07-28</td>\n",
       "      <td>25</td>\n",
       "      <td>34</td>\n",
       "      <td>17</td>\n",
       "      <td>33</td>\n",
       "      <td>45</td>\n",
       "      <td>29</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>60</td>\n",
       "      <td>...</td>\n",
       "      <td>12</td>\n",
       "      <td>33</td>\n",
       "      <td>20</td>\n",
       "      <td>24</td>\n",
       "      <td>30</td>\n",
       "      <td>37</td>\n",
       "      <td>29</td>\n",
       "      <td>13</td>\n",
       "      <td>12</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178</th>\n",
       "      <td>2022-07-29</td>\n",
       "      <td>25</td>\n",
       "      <td>34</td>\n",
       "      <td>17</td>\n",
       "      <td>33</td>\n",
       "      <td>45</td>\n",
       "      <td>29</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>60</td>\n",
       "      <td>...</td>\n",
       "      <td>12</td>\n",
       "      <td>33</td>\n",
       "      <td>20</td>\n",
       "      <td>24</td>\n",
       "      <td>30</td>\n",
       "      <td>37</td>\n",
       "      <td>29</td>\n",
       "      <td>13</td>\n",
       "      <td>12</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>179</th>\n",
       "      <td>2022-07-30</td>\n",
       "      <td>25</td>\n",
       "      <td>34</td>\n",
       "      <td>17</td>\n",
       "      <td>33</td>\n",
       "      <td>45</td>\n",
       "      <td>29</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>60</td>\n",
       "      <td>...</td>\n",
       "      <td>12</td>\n",
       "      <td>33</td>\n",
       "      <td>20</td>\n",
       "      <td>24</td>\n",
       "      <td>30</td>\n",
       "      <td>37</td>\n",
       "      <td>29</td>\n",
       "      <td>13</td>\n",
       "      <td>12</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>180 rows × 55 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          Dates (Onion Big,) (Onion Small,) (Tomato,) (Green Chilli,)  \\\n",
       "0    2022-02-01           36             62        19              42   \n",
       "1    2022-02-02           39             63        17              36   \n",
       "2    2022-02-03           39             63        17              36   \n",
       "3    2022-02-04           39             63        17              36   \n",
       "4    2022-02-05           38             63        21              45   \n",
       "..          ...          ...            ...       ...             ...   \n",
       "175  2022-07-26           25             34        17              33   \n",
       "176  2022-07-27           25             34        17              33   \n",
       "177  2022-07-28           25             34        17              33   \n",
       "178  2022-07-29           25             34        17              33   \n",
       "179  2022-07-30           25             34        17              33   \n",
       "\n",
       "    (Beetroot,) (Potato,) (Raw Banana (Plantain),) (Amaranth Leaves,) (Amla,)  \\\n",
       "0            49        22                        5                 15      60   \n",
       "1            50        21                        4                 17      60   \n",
       "2            50        21                        4                 17      60   \n",
       "3            50        21                        4                 17      60   \n",
       "4            48        26                        4                 12      60   \n",
       "..          ...       ...                      ...                ...     ...   \n",
       "175          45        29                        7                 10      60   \n",
       "176          45        29                        7                 10      60   \n",
       "177          45        29                        7                 10      60   \n",
       "178          45        29                        7                 10      60   \n",
       "179          45        29                        7                 10      60   \n",
       "\n",
       "     ... (Mustard Leaves,) (Ladies Finger,) (Pumpkin,) (Radish,)  \\\n",
       "0    ...                20               59         26        24   \n",
       "1    ...                15               57         24        24   \n",
       "2    ...                15               57         24        24   \n",
       "3    ...                15               57         24        24   \n",
       "4    ...                17               58         21        21   \n",
       "..   ...               ...              ...        ...       ...   \n",
       "175  ...                12               33         20        24   \n",
       "176  ...                12               33         20        24   \n",
       "177  ...                12               33         20        24   \n",
       "178  ...                12               33         20        24   \n",
       "179  ...                12               33         20        24   \n",
       "\n",
       "    (Ridge Gourd,) (Shallot (Pearl Onion),) (Snake Gourd,) (Sorrel Leaves,)  \\\n",
       "0               30                       34             25               13   \n",
       "1               30                       36             25               11   \n",
       "2               30                       36             25               11   \n",
       "3               30                       36             25               11   \n",
       "4               33                       31             28               13   \n",
       "..             ...                      ...            ...              ...   \n",
       "175             30                       37             29               13   \n",
       "176             30                       37             29               13   \n",
       "177             30                       37             29               13   \n",
       "178             30                       37             29               13   \n",
       "179             30                       37             29               13   \n",
       "\n",
       "    (Spinach,) (Sweet Potato,)  \n",
       "0           12              38  \n",
       "1           15              43  \n",
       "2           15              43  \n",
       "3           15              43  \n",
       "4           10              39  \n",
       "..         ...             ...  \n",
       "175         12              37  \n",
       "176         12              37  \n",
       "177         12              37  \n",
       "178         12              37  \n",
       "179         12              37  \n",
       "\n",
       "[180 rows x 55 columns]"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "# exporting the data into csv file\n",
    "path = 'C:\\\\Users\\\\91982\\\\Downloads\\\\project\\\\FEB_JULY_Prices_Delhi.csv'\n",
    "df.to_csv(path, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extracting data from aug to dec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "#visiting to noida market price history from main page\n",
    "homepage = 'https://vegetablemarketprice.com/'\n",
    "data = []\n",
    "html_list = []\n",
    "driver = webdriver.Firefox()\n",
    "driver.get(homepage)\n",
    "search_input = driver.find_element(by=By.XPATH, value='//*[@id=\"header-search-input\"]')\n",
    "\n",
    "header_search_input = 'Delhi Vegetable Market Price History'\n",
    "search_input.send_keys(header_search_input)\n",
    "search_input.send_keys(Keys.ENTER)\n",
    "time.sleep(5)\n",
    "driver.find_element(by=By.XPATH,value='/html/body/ul/li[1]/div').click()\n",
    "\n",
    "#selecting table in dashboard\n",
    "switch = driver.find_element(By.XPATH, '//*[@id=\"navItemTable\"]')\n",
    "switch.click()\n",
    "\n",
    "#selecting custom range from the date \n",
    "change_date = driver.find_element(by = By.XPATH,value = '//*[@id=\"reportrange\"]')\n",
    "change_date.click()\n",
    "custom_range = driver.find_element(By.XPATH, \"//*[contains(text(), 'Custom Range')]\")\n",
    "custom_range.click()\n",
    "\n",
    "\n",
    "# Initialzing the dates\n",
    "while True: \n",
    "    check_month = driver.find_element(By.XPATH, \"/html/body/div[4]/div[2]/div[1]/table/thead/tr[1]/th[2]\")\n",
    "    month = check_month.get_attribute(\"innerHTML\")\n",
    "    if month == \"Aug 2022\":\n",
    "        break\n",
    "        \n",
    "    else:\n",
    "        back_button = driver.find_element(By.XPATH,'/html/body/div[4]/div[2]/div[1]/table/thead/tr[1]/th[1]')\n",
    "\n",
    "        back_button.click()\n",
    "        \n",
    "first_date = driver.find_element(By.XPATH, '/html/body/div[4]/div[2]/div[1]/table/tbody/tr[1]/td[2]')\n",
    "first_date.click()\n",
    "\n",
    "#selecting date from Aug to Dec\n",
    "i=0\n",
    "while i <= 2:\n",
    "    front_button = driver.find_element(By.XPATH,\"/html/body/div[4]/div[3]/div[1]/table/thead/tr[1]/th[3]\")\n",
    "    front_button.click()\n",
    "    i = i+1\n",
    "\n",
    "last_date = driver.find_element(By.XPATH,\"/html/body/div[4]/div[3]/div[1]/table/tbody/tr[5]/td[7]\")\n",
    "last_date.click()\n",
    "\n",
    "apply_button = driver.find_element(By.XPATH,\"/html/body/div[4]/div[4]/button[2]\")\n",
    "apply_button.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Onion Big\n",
      "Onion Small\n",
      "Tomato\n",
      "Green Chilli\n",
      "Beetroot\n",
      "Potato\n",
      "Raw Banana (Plantain)\n",
      "Amaranth Leaves\n",
      "Amla\n",
      "Ash gourd\n",
      "Baby Corn\n",
      "Banana Flower\n",
      "Capsicum\n",
      "Bitter Gourd\n",
      "Bottle Gourd\n",
      "Butter Beans\n",
      "Broad Beans\n",
      "Cabbage\n",
      "Carrot\n",
      "Cauliflower\n",
      "Cluster beans\n",
      "Coconut\n",
      "Colocasia Leaves\n",
      "Colocasia\n",
      "Coriander Leaves\n",
      "Corn\n",
      "Cucumber\n",
      "Curry Leaves\n",
      "Dill Leaves\n",
      "Drumsticks\n",
      "Brinjal\n",
      "Brinjal (Big)\n",
      "Elephant Yam\n",
      "Fenugreek Leaves\n",
      "French Beans\n",
      "Garlic\n",
      "Ginger\n",
      "Onion Green\n",
      "Green Peas\n",
      "Ivy Gourd\n",
      "Lemon (Lime)\n",
      "Mango Raw\n",
      "Mint Leaves\n",
      "Mushroom\n",
      "Mustard Leaves\n",
      "Ladies Finger\n",
      "Pumpkin\n",
      "Radish\n",
      "Ridge Gourd\n",
      "Shallot (Pearl Onion)\n",
      "Snake Gourd\n",
      "Sorrel Leaves\n",
      "Spinach\n",
      "Sweet Potato\n"
     ]
    }
   ],
   "source": [
    "#Extracting  the names of vegetables \n",
    "for i in range (0,len(final_list)):\n",
    "    while True:\n",
    "        try:\n",
    "            element = driver.find_element(By.XPATH, '/html/body/div[2]/div[2]/div/div[1]/div[4]/div/div[1]/div/div[1]/button')\n",
    "            element.click()\n",
    "        except:\n",
    "            break\n",
    "    search_input = driver.find_element(By.NAME, 'search_terms')\n",
    "    search_input.send_keys(final_list[i])\n",
    "    time.sleep(3)\n",
    "    search_input.send_keys(Keys.ENTER)\n",
    "    \n",
    "    temp = driver.find_element(By.XPATH, '//*[@id=\"historyDataTable\"]')\n",
    "    time.sleep(3)\n",
    "    temp_html = temp.get_attribute(\"innerHTML\")\n",
    "    html_list.append(temp_html)\n",
    "    time.sleep(3)\n",
    "    print(final_list[i])\n",
    "\n",
    "#extracting names of vegetables and prices\n",
    "Name = []\n",
    "Prices = []\n",
    "\n",
    "for i in html_list:\n",
    "    temp_n = re.findall('<tbody><tr><td>(.*?)</td><td>',i)\n",
    "    Name.extend(temp_n)\n",
    "    \n",
    "    temp_p = re.findall('<td>₹ (.*?)</td>',i)\n",
    "    Prices.append(temp_p)\n",
    "\n",
    "#extracting date \n",
    "date = re.findall('<th>(.*?)</th>',html_list[0])\n",
    "dates = []\n",
    "for i in date:\n",
    "    temp = re.findall('\"white-space: nowrap;\">(.*?)</span>',i)\n",
    "    if len(temp) != 0:\n",
    "        dates.append(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# converting into Dataframe\n",
    "df = pd.DataFrame(dates,columns = ['Dates'])\n",
    "df1 = pd.DataFrame(Prices).transpose()\n",
    "df1.dropna(inplace = True)\n",
    "df1.columns = [Name]\n",
    "df = pd.merge(df,df1,left_index=True, right_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Dates</th>\n",
       "      <th>(Onion Big,)</th>\n",
       "      <th>(Onion Small,)</th>\n",
       "      <th>(Tomato,)</th>\n",
       "      <th>(Green Chilli,)</th>\n",
       "      <th>(Beetroot,)</th>\n",
       "      <th>(Potato,)</th>\n",
       "      <th>(Raw Banana (Plantain),)</th>\n",
       "      <th>(Amaranth Leaves,)</th>\n",
       "      <th>(Amla,)</th>\n",
       "      <th>...</th>\n",
       "      <th>(Mustard Leaves,)</th>\n",
       "      <th>(Ladies Finger,)</th>\n",
       "      <th>(Pumpkin,)</th>\n",
       "      <th>(Radish,)</th>\n",
       "      <th>(Ridge Gourd,)</th>\n",
       "      <th>(Shallot (Pearl Onion),)</th>\n",
       "      <th>(Snake Gourd,)</th>\n",
       "      <th>(Sorrel Leaves,)</th>\n",
       "      <th>(Spinach,)</th>\n",
       "      <th>(Sweet Potato,)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-08-01</td>\n",
       "      <td>21</td>\n",
       "      <td>38</td>\n",
       "      <td>17</td>\n",
       "      <td>37</td>\n",
       "      <td>43</td>\n",
       "      <td>31</td>\n",
       "      <td>11</td>\n",
       "      <td>10</td>\n",
       "      <td>50</td>\n",
       "      <td>...</td>\n",
       "      <td>17</td>\n",
       "      <td>37</td>\n",
       "      <td>26</td>\n",
       "      <td>30</td>\n",
       "      <td>26</td>\n",
       "      <td>30</td>\n",
       "      <td>30</td>\n",
       "      <td>12</td>\n",
       "      <td>14</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2022-08-02</td>\n",
       "      <td>21</td>\n",
       "      <td>38</td>\n",
       "      <td>17</td>\n",
       "      <td>37</td>\n",
       "      <td>43</td>\n",
       "      <td>31</td>\n",
       "      <td>11</td>\n",
       "      <td>10</td>\n",
       "      <td>50</td>\n",
       "      <td>...</td>\n",
       "      <td>17</td>\n",
       "      <td>37</td>\n",
       "      <td>26</td>\n",
       "      <td>30</td>\n",
       "      <td>26</td>\n",
       "      <td>30</td>\n",
       "      <td>30</td>\n",
       "      <td>12</td>\n",
       "      <td>14</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-08-03</td>\n",
       "      <td>21</td>\n",
       "      <td>38</td>\n",
       "      <td>17</td>\n",
       "      <td>37</td>\n",
       "      <td>43</td>\n",
       "      <td>31</td>\n",
       "      <td>11</td>\n",
       "      <td>10</td>\n",
       "      <td>50</td>\n",
       "      <td>...</td>\n",
       "      <td>17</td>\n",
       "      <td>37</td>\n",
       "      <td>26</td>\n",
       "      <td>30</td>\n",
       "      <td>26</td>\n",
       "      <td>30</td>\n",
       "      <td>30</td>\n",
       "      <td>12</td>\n",
       "      <td>14</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2022-08-04</td>\n",
       "      <td>25</td>\n",
       "      <td>38</td>\n",
       "      <td>16</td>\n",
       "      <td>32</td>\n",
       "      <td>45</td>\n",
       "      <td>34</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>50</td>\n",
       "      <td>...</td>\n",
       "      <td>13</td>\n",
       "      <td>35</td>\n",
       "      <td>23</td>\n",
       "      <td>26</td>\n",
       "      <td>28</td>\n",
       "      <td>36</td>\n",
       "      <td>30</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2022-08-05</td>\n",
       "      <td>25</td>\n",
       "      <td>38</td>\n",
       "      <td>16</td>\n",
       "      <td>32</td>\n",
       "      <td>45</td>\n",
       "      <td>34</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>50</td>\n",
       "      <td>...</td>\n",
       "      <td>13</td>\n",
       "      <td>35</td>\n",
       "      <td>23</td>\n",
       "      <td>26</td>\n",
       "      <td>28</td>\n",
       "      <td>36</td>\n",
       "      <td>30</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>148</th>\n",
       "      <td>2022-12-27</td>\n",
       "      <td>29</td>\n",
       "      <td>39</td>\n",
       "      <td>15</td>\n",
       "      <td>36</td>\n",
       "      <td>35</td>\n",
       "      <td>27</td>\n",
       "      <td>7</td>\n",
       "      <td>13</td>\n",
       "      <td>60</td>\n",
       "      <td>...</td>\n",
       "      <td>16</td>\n",
       "      <td>30</td>\n",
       "      <td>29</td>\n",
       "      <td>24</td>\n",
       "      <td>30</td>\n",
       "      <td>35</td>\n",
       "      <td>30</td>\n",
       "      <td>14</td>\n",
       "      <td>10</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>2022-12-28</td>\n",
       "      <td>29</td>\n",
       "      <td>39</td>\n",
       "      <td>15</td>\n",
       "      <td>36</td>\n",
       "      <td>35</td>\n",
       "      <td>27</td>\n",
       "      <td>7</td>\n",
       "      <td>13</td>\n",
       "      <td>60</td>\n",
       "      <td>...</td>\n",
       "      <td>16</td>\n",
       "      <td>30</td>\n",
       "      <td>29</td>\n",
       "      <td>24</td>\n",
       "      <td>30</td>\n",
       "      <td>35</td>\n",
       "      <td>30</td>\n",
       "      <td>14</td>\n",
       "      <td>10</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150</th>\n",
       "      <td>2022-12-29</td>\n",
       "      <td>27</td>\n",
       "      <td>35</td>\n",
       "      <td>15</td>\n",
       "      <td>36</td>\n",
       "      <td>40</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>60</td>\n",
       "      <td>...</td>\n",
       "      <td>18</td>\n",
       "      <td>28</td>\n",
       "      <td>25</td>\n",
       "      <td>22</td>\n",
       "      <td>28</td>\n",
       "      <td>29</td>\n",
       "      <td>30</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151</th>\n",
       "      <td>2022-12-30</td>\n",
       "      <td>27</td>\n",
       "      <td>35</td>\n",
       "      <td>15</td>\n",
       "      <td>36</td>\n",
       "      <td>40</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>60</td>\n",
       "      <td>...</td>\n",
       "      <td>18</td>\n",
       "      <td>28</td>\n",
       "      <td>25</td>\n",
       "      <td>22</td>\n",
       "      <td>28</td>\n",
       "      <td>29</td>\n",
       "      <td>30</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>152</th>\n",
       "      <td>2022-12-31</td>\n",
       "      <td>27</td>\n",
       "      <td>35</td>\n",
       "      <td>15</td>\n",
       "      <td>36</td>\n",
       "      <td>40</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>60</td>\n",
       "      <td>...</td>\n",
       "      <td>18</td>\n",
       "      <td>28</td>\n",
       "      <td>25</td>\n",
       "      <td>22</td>\n",
       "      <td>28</td>\n",
       "      <td>29</td>\n",
       "      <td>30</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>153 rows × 55 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          Dates (Onion Big,) (Onion Small,) (Tomato,) (Green Chilli,)  \\\n",
       "0    2022-08-01           21             38        17              37   \n",
       "1    2022-08-02           21             38        17              37   \n",
       "2    2022-08-03           21             38        17              37   \n",
       "3    2022-08-04           25             38        16              32   \n",
       "4    2022-08-05           25             38        16              32   \n",
       "..          ...          ...            ...       ...             ...   \n",
       "148  2022-12-27           29             39        15              36   \n",
       "149  2022-12-28           29             39        15              36   \n",
       "150  2022-12-29           27             35        15              36   \n",
       "151  2022-12-30           27             35        15              36   \n",
       "152  2022-12-31           27             35        15              36   \n",
       "\n",
       "    (Beetroot,) (Potato,) (Raw Banana (Plantain),) (Amaranth Leaves,) (Amla,)  \\\n",
       "0            43        31                       11                 10      50   \n",
       "1            43        31                       11                 10      50   \n",
       "2            43        31                       11                 10      50   \n",
       "3            45        34                        9                 10      50   \n",
       "4            45        34                        9                 10      50   \n",
       "..          ...       ...                      ...                ...     ...   \n",
       "148          35        27                        7                 13      60   \n",
       "149          35        27                        7                 13      60   \n",
       "150          40        28                       10                 10      60   \n",
       "151          40        28                       10                 10      60   \n",
       "152          40        28                       10                 10      60   \n",
       "\n",
       "     ... (Mustard Leaves,) (Ladies Finger,) (Pumpkin,) (Radish,)  \\\n",
       "0    ...                17               37         26        30   \n",
       "1    ...                17               37         26        30   \n",
       "2    ...                17               37         26        30   \n",
       "3    ...                13               35         23        26   \n",
       "4    ...                13               35         23        26   \n",
       "..   ...               ...              ...        ...       ...   \n",
       "148  ...                16               30         29        24   \n",
       "149  ...                16               30         29        24   \n",
       "150  ...                18               28         25        22   \n",
       "151  ...                18               28         25        22   \n",
       "152  ...                18               28         25        22   \n",
       "\n",
       "    (Ridge Gourd,) (Shallot (Pearl Onion),) (Snake Gourd,) (Sorrel Leaves,)  \\\n",
       "0               26                       30             30               12   \n",
       "1               26                       30             30               12   \n",
       "2               26                       30             30               12   \n",
       "3               28                       36             30               12   \n",
       "4               28                       36             30               12   \n",
       "..             ...                      ...            ...              ...   \n",
       "148             30                       35             30               14   \n",
       "149             30                       35             30               14   \n",
       "150             28                       29             30               12   \n",
       "151             28                       29             30               12   \n",
       "152             28                       29             30               12   \n",
       "\n",
       "    (Spinach,) (Sweet Potato,)  \n",
       "0           14              38  \n",
       "1           14              38  \n",
       "2           14              38  \n",
       "3           13              37  \n",
       "4           13              37  \n",
       "..         ...             ...  \n",
       "148         10              38  \n",
       "149         10              38  \n",
       "150         12              39  \n",
       "151         12              39  \n",
       "152         12              39  \n",
       "\n",
       "[153 rows x 55 columns]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# exporting the data into csv file\n",
    "p='C:\\\\Users\\\\91982\\\\Downloads\\\\\\\\AUG_DEC_Prices_DELHI.csv'\n",
    "df.to_csv(p, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extracting data of delhi from Jan to March "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#visiting the page\n",
    "homepage = 'https://vegetablemarketprice.com/'\n",
    "data = []\n",
    "html_list = []\n",
    "driver = webdriver.Firefox()\n",
    "driver.get(homepage)\n",
    "search_input = driver.find_element(by=By.XPATH, value='//*[@id=\"header-search-input\"]')\n",
    "\n",
    "# #Searching Noida Price History\n",
    "header_search_input = 'Delhi Vegetable Market Price History'\n",
    "search_input.send_keys(header_search_input)\n",
    "search_input.send_keys(Keys.ENTER)\n",
    "time.sleep(5)\n",
    "driver.find_element(by=By.XPATH,value='/html/body/ul/li[1]/div').click()\n",
    "\n",
    "#switching from chart to table in dashboard\n",
    "switch = driver.find_element(By.XPATH, '//*[@id=\"navItemTable\"]')\n",
    "switch.click()\n",
    "\n",
    "# #selecting the date \n",
    "change_date = driver.find_element(by = By.XPATH,value = '//*[@id=\"reportrange\"]')\n",
    "change_date.click()\n",
    "custom_range = driver.find_element(By.XPATH, \"//*[contains(text(), 'Custom Range')]\")\n",
    "custom_range.click()\n",
    "\n",
    "# Initialzing starting date and end date\n",
    "while True: \n",
    "    check_month = driver.find_element(By.XPATH, \"/html/body/div[4]/div[2]/div[1]/table/thead/tr[1]/th[2]\")\n",
    "    month = check_month.get_attribute(\"innerHTML\")\n",
    "    if month == \"Jan 2023\":\n",
    "        break\n",
    "        \n",
    "    else:\n",
    "        back_button = driver.find_element(By.XPATH,'/html/body/div[4]/div[2]/div[1]/table/thead/tr[1]/th[1]')\n",
    "\n",
    "        back_button.click()\n",
    "        \n",
    "first_date = driver.find_element(By.XPATH, '/html/body/div[4]/div[2]/div[1]/table/tbody/tr[2]/td[1]')\n",
    "first_date.click()\n",
    "\n",
    "i=0                  #switching page of calendar \n",
    "while i < 1:\n",
    "    front_button = driver.find_element(By.XPATH,\"/html/body/div[4]/div[3]/div[1]/table/thead/tr[1]/th[3]\")\n",
    "    front_button.click()\n",
    "    i = i+1\n",
    "\n",
    "last_date = driver.find_element(By.XPATH,\"/html/body/div[4]/div[3]/div[1]/table/tbody/tr[5]/td[6]\")  # selecting end date \n",
    "last_date.click()\n",
    "\n",
    "#applyig the dates \n",
    "wait = WebDriverWait(driver, 60)  # Adjust the timeout as needed\n",
    "apply_button = wait.until(EC.element_to_be_clickable((By.XPATH, \"/html/body/div[4]/div[4]/button[2]\")))\n",
    "apply_button.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extracing the names of vegetables\n",
    "for i in range (0,len(final_list)):\n",
    "    while True:\n",
    "        try:\n",
    "            element = driver.find_element(By.XPATH, '/html/body/div[2]/div[2]/div/div[1]/div[4]/div/div[1]/div/div[1]/button')\n",
    "            element.click()\n",
    "        except:\n",
    "            break\n",
    "    search_input = driver.find_element(By.NAME, 'search_terms')\n",
    "    search_input.send_keys(final_list[i])\n",
    "    time.sleep(3)\n",
    "    search_input.send_keys(Keys.ENTER)\n",
    "    \n",
    "    temp = driver.find_element(By.XPATH, '//*[@id=\"historyDataTable\"]')\n",
    "    time.sleep(3)\n",
    "    temp_html = temp.get_attribute(\"innerHTML\")\n",
    "    html_list.append(temp_html)\n",
    "    time.sleep(3)\n",
    "    print(final_list[i])\n",
    "\n",
    "# extracting the dates from html code\n",
    "date = re.findall('<th>(.*?)</th>',html_list[0])\n",
    "dates = []\n",
    "for i in date:\n",
    "    temp = re.findall('\"white-space: nowrap;\">(.*?)</span>',i)\n",
    "    if len(temp) != 0:\n",
    "        dates.append(temp)\n",
    "\n",
    "\n",
    "# Extracting names and prices of the vegetables \n",
    "Name = []\n",
    "Prices = []\n",
    "\n",
    "for i in html_list:\n",
    "    temp_n = re.findall('<tbody><tr><td>(.*?)</td><td>',i)\n",
    "    Name.extend(temp_n)\n",
    "    \n",
    "    temp_p = re.findall('<td>₹ (.*?)</td>',i)\n",
    "    Prices.append(temp_p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Dates</th>\n",
       "      <th>(Onion Big,)</th>\n",
       "      <th>(Onion Small,)</th>\n",
       "      <th>(Tomato,)</th>\n",
       "      <th>(Green Chilli,)</th>\n",
       "      <th>(Beetroot,)</th>\n",
       "      <th>(Potato,)</th>\n",
       "      <th>(Raw Banana (Plantain),)</th>\n",
       "      <th>(Amaranth Leaves,)</th>\n",
       "      <th>(Amla,)</th>\n",
       "      <th>...</th>\n",
       "      <th>(Mustard Leaves,)</th>\n",
       "      <th>(Ladies Finger,)</th>\n",
       "      <th>(Pumpkin,)</th>\n",
       "      <th>(Radish,)</th>\n",
       "      <th>(Ridge Gourd,)</th>\n",
       "      <th>(Shallot (Pearl Onion),)</th>\n",
       "      <th>(Snake Gourd,)</th>\n",
       "      <th>(Sorrel Leaves,)</th>\n",
       "      <th>(Spinach,)</th>\n",
       "      <th>(Sweet Potato,)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2023-01-01</td>\n",
       "      <td>27</td>\n",
       "      <td>35</td>\n",
       "      <td>15</td>\n",
       "      <td>36</td>\n",
       "      <td>40</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>60</td>\n",
       "      <td>...</td>\n",
       "      <td>18</td>\n",
       "      <td>28</td>\n",
       "      <td>25</td>\n",
       "      <td>22</td>\n",
       "      <td>28</td>\n",
       "      <td>29</td>\n",
       "      <td>30</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2023-01-02</td>\n",
       "      <td>30</td>\n",
       "      <td>35</td>\n",
       "      <td>16</td>\n",
       "      <td>36</td>\n",
       "      <td>34</td>\n",
       "      <td>33</td>\n",
       "      <td>7</td>\n",
       "      <td>13</td>\n",
       "      <td>60</td>\n",
       "      <td>...</td>\n",
       "      <td>18</td>\n",
       "      <td>30</td>\n",
       "      <td>29</td>\n",
       "      <td>24</td>\n",
       "      <td>29</td>\n",
       "      <td>36</td>\n",
       "      <td>28</td>\n",
       "      <td>15</td>\n",
       "      <td>11</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2023-01-03</td>\n",
       "      <td>30</td>\n",
       "      <td>35</td>\n",
       "      <td>16</td>\n",
       "      <td>36</td>\n",
       "      <td>34</td>\n",
       "      <td>33</td>\n",
       "      <td>7</td>\n",
       "      <td>13</td>\n",
       "      <td>60</td>\n",
       "      <td>...</td>\n",
       "      <td>18</td>\n",
       "      <td>30</td>\n",
       "      <td>29</td>\n",
       "      <td>24</td>\n",
       "      <td>29</td>\n",
       "      <td>36</td>\n",
       "      <td>28</td>\n",
       "      <td>15</td>\n",
       "      <td>11</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2023-01-04</td>\n",
       "      <td>24</td>\n",
       "      <td>38</td>\n",
       "      <td>17</td>\n",
       "      <td>37</td>\n",
       "      <td>31</td>\n",
       "      <td>32</td>\n",
       "      <td>7</td>\n",
       "      <td>11</td>\n",
       "      <td>70</td>\n",
       "      <td>...</td>\n",
       "      <td>14</td>\n",
       "      <td>32</td>\n",
       "      <td>24</td>\n",
       "      <td>27</td>\n",
       "      <td>28</td>\n",
       "      <td>32</td>\n",
       "      <td>31</td>\n",
       "      <td>13</td>\n",
       "      <td>11</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2023-01-05</td>\n",
       "      <td>24</td>\n",
       "      <td>38</td>\n",
       "      <td>17</td>\n",
       "      <td>37</td>\n",
       "      <td>31</td>\n",
       "      <td>32</td>\n",
       "      <td>7</td>\n",
       "      <td>11</td>\n",
       "      <td>70</td>\n",
       "      <td>...</td>\n",
       "      <td>14</td>\n",
       "      <td>32</td>\n",
       "      <td>24</td>\n",
       "      <td>27</td>\n",
       "      <td>28</td>\n",
       "      <td>32</td>\n",
       "      <td>31</td>\n",
       "      <td>13</td>\n",
       "      <td>11</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>2023-03-27</td>\n",
       "      <td>19</td>\n",
       "      <td>49</td>\n",
       "      <td>24</td>\n",
       "      <td>38</td>\n",
       "      <td>30</td>\n",
       "      <td>32</td>\n",
       "      <td>7</td>\n",
       "      <td>12</td>\n",
       "      <td>50</td>\n",
       "      <td>...</td>\n",
       "      <td>18</td>\n",
       "      <td>41</td>\n",
       "      <td>20</td>\n",
       "      <td>25</td>\n",
       "      <td>27</td>\n",
       "      <td>31</td>\n",
       "      <td>27</td>\n",
       "      <td>13</td>\n",
       "      <td>11</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>2023-03-28</td>\n",
       "      <td>19</td>\n",
       "      <td>49</td>\n",
       "      <td>24</td>\n",
       "      <td>38</td>\n",
       "      <td>30</td>\n",
       "      <td>32</td>\n",
       "      <td>7</td>\n",
       "      <td>12</td>\n",
       "      <td>50</td>\n",
       "      <td>...</td>\n",
       "      <td>18</td>\n",
       "      <td>41</td>\n",
       "      <td>20</td>\n",
       "      <td>25</td>\n",
       "      <td>27</td>\n",
       "      <td>31</td>\n",
       "      <td>27</td>\n",
       "      <td>13</td>\n",
       "      <td>11</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>2023-03-29</td>\n",
       "      <td>22</td>\n",
       "      <td>56</td>\n",
       "      <td>21</td>\n",
       "      <td>38</td>\n",
       "      <td>29</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "      <td>15</td>\n",
       "      <td>60</td>\n",
       "      <td>...</td>\n",
       "      <td>16</td>\n",
       "      <td>50</td>\n",
       "      <td>24</td>\n",
       "      <td>21</td>\n",
       "      <td>27</td>\n",
       "      <td>32</td>\n",
       "      <td>29</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>2023-03-30</td>\n",
       "      <td>22</td>\n",
       "      <td>56</td>\n",
       "      <td>21</td>\n",
       "      <td>38</td>\n",
       "      <td>29</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "      <td>15</td>\n",
       "      <td>60</td>\n",
       "      <td>...</td>\n",
       "      <td>16</td>\n",
       "      <td>50</td>\n",
       "      <td>24</td>\n",
       "      <td>21</td>\n",
       "      <td>27</td>\n",
       "      <td>32</td>\n",
       "      <td>29</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>2023-03-31</td>\n",
       "      <td>22</td>\n",
       "      <td>56</td>\n",
       "      <td>21</td>\n",
       "      <td>38</td>\n",
       "      <td>29</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "      <td>15</td>\n",
       "      <td>60</td>\n",
       "      <td>...</td>\n",
       "      <td>16</td>\n",
       "      <td>50</td>\n",
       "      <td>24</td>\n",
       "      <td>21</td>\n",
       "      <td>27</td>\n",
       "      <td>32</td>\n",
       "      <td>29</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>90 rows × 55 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         Dates (Onion Big,) (Onion Small,) (Tomato,) (Green Chilli,)  \\\n",
       "0   2023-01-01           27             35        15              36   \n",
       "1   2023-01-02           30             35        16              36   \n",
       "2   2023-01-03           30             35        16              36   \n",
       "3   2023-01-04           24             38        17              37   \n",
       "4   2023-01-05           24             38        17              37   \n",
       "..         ...          ...            ...       ...             ...   \n",
       "85  2023-03-27           19             49        24              38   \n",
       "86  2023-03-28           19             49        24              38   \n",
       "87  2023-03-29           22             56        21              38   \n",
       "88  2023-03-30           22             56        21              38   \n",
       "89  2023-03-31           22             56        21              38   \n",
       "\n",
       "   (Beetroot,) (Potato,) (Raw Banana (Plantain),) (Amaranth Leaves,) (Amla,)  \\\n",
       "0           40        28                       10                 10      60   \n",
       "1           34        33                        7                 13      60   \n",
       "2           34        33                        7                 13      60   \n",
       "3           31        32                        7                 11      70   \n",
       "4           31        32                        7                 11      70   \n",
       "..         ...       ...                      ...                ...     ...   \n",
       "85          30        32                        7                 12      50   \n",
       "86          30        32                        7                 12      50   \n",
       "87          29        28                       10                 15      60   \n",
       "88          29        28                       10                 15      60   \n",
       "89          29        28                       10                 15      60   \n",
       "\n",
       "    ... (Mustard Leaves,) (Ladies Finger,) (Pumpkin,) (Radish,)  \\\n",
       "0   ...                18               28         25        22   \n",
       "1   ...                18               30         29        24   \n",
       "2   ...                18               30         29        24   \n",
       "3   ...                14               32         24        27   \n",
       "4   ...                14               32         24        27   \n",
       "..  ...               ...              ...        ...       ...   \n",
       "85  ...                18               41         20        25   \n",
       "86  ...                18               41         20        25   \n",
       "87  ...                16               50         24        21   \n",
       "88  ...                16               50         24        21   \n",
       "89  ...                16               50         24        21   \n",
       "\n",
       "   (Ridge Gourd,) (Shallot (Pearl Onion),) (Snake Gourd,) (Sorrel Leaves,)  \\\n",
       "0              28                       29             30               12   \n",
       "1              29                       36             28               15   \n",
       "2              29                       36             28               15   \n",
       "3              28                       32             31               13   \n",
       "4              28                       32             31               13   \n",
       "..            ...                      ...            ...              ...   \n",
       "85             27                       31             27               13   \n",
       "86             27                       31             27               13   \n",
       "87             27                       32             29               13   \n",
       "88             27                       32             29               13   \n",
       "89             27                       32             29               13   \n",
       "\n",
       "   (Spinach,) (Sweet Potato,)  \n",
       "0          12              39  \n",
       "1          11              39  \n",
       "2          11              39  \n",
       "3          11              37  \n",
       "4          11              37  \n",
       "..        ...             ...  \n",
       "85         11              36  \n",
       "86         11              36  \n",
       "87         13              37  \n",
       "88         13              37  \n",
       "89         13              37  \n",
       "\n",
       "[90 rows x 55 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(dates,columns = ['Dates'])\n",
    "df1 = pd.DataFrame(Prices).transpose()\n",
    "df1.dropna(inplace = True)\n",
    "df1.columns = [Name]\n",
    "df = pd.merge(df,df1,left_index=True, right_index=True)\n",
    "\n",
    "# exporting csv file \n",
    "path = 'C:\\\\Users\\\\91982\\\\Downloads\\\\project\\\\JAN_MARCH_Prices_Delhi.csv'\n",
    "df.to_csv(path, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in final_list:\n",
    "    if i not in Name:\n",
    "        print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}