Amazon Web Scraping(Python) .ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "545f0489",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import  the necessary libraries \n",
    "\n",
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "import time\n",
    "import datetime\n",
    "import csv\n",
    "import pandas as pd\n",
    "\n",
    "import smtplib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cab9e290",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "            Lenovo V15 (2021) Intel Core i3 10th Gen 39.63cm (15.6\") FHD Thin and Light Laptop (4GB RAM/ 256GB SSD/ Windows 10 Home/ Iron Grey/ 1.85 kg), 82C500XXIH\n",
      "           \n",
      "\n",
      "                          38,609\n",
      "                          \n",
      "                           .\n",
      "                          \n",
      "\n",
      "\n",
      "             23 ratings\n",
      "            \n"
     ]
    }
   ],
   "source": [
    "# connecting to the amazon website\n",
    "\n",
    "URL = 'https://www.amazon.in/Lenovo-inches-Business-Windows-82C500XXIH/dp/B0979M4T4X/ref=sr_1_13?brr=1&qid=1680180682&rd=1&sr=8-13'\n",
    "\n",
    "\n",
    "headers = {\"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36\", \"Accept-Encoding\":\"gzip, deflate, br\", \"Accept\":\"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7\", \"DNT\":\"1\",\"Connection\":\"close\", \"Upgrade-Insecure-Requests\":\"1\"}\n",
    "\n",
    "\n",
    "page = requests.get(URL, headers=headers)\n",
    "\n",
    "#Pulling all HTML data\n",
    "soup1 = BeautifulSoup(page.content, \"html.parser\")\n",
    "\n",
    "#Prettifying the data\n",
    "soup2 = BeautifulSoup(soup1.prettify(), \"html.parser\")\n",
    "\n",
    "#Pulling the title text\n",
    "title = soup2.find(id='productTitle').get_text()\n",
    "\n",
    "\n",
    "# As we are not able to extract 's-id' directly, we will use below method to take out the price(By using class comparison)\n",
    "\n",
    "\n",
    "price = soup2.find('span',class_='a-price-whole').get_text()\n",
    "\n",
    "\n",
    "# Pull out the total number of ratings given on the product\n",
    "\n",
    "\n",
    "total_rate = soup2.find(id='acrCustomerReviewText').get_text()\n",
    "\n",
    "\n",
    "print(title)\n",
    "print(price)\n",
    "print(total_rate)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "33388f25",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "38,609\n",
      "Lenovo V15 (2021) Intel Core i3 10th Gen 39.63cm (15.6\") FHD Thin and Light Laptop (4GB RAM/ 256GB SSD/ Windows 10 Home/ Iron Grey/ 1.85 kg), 82C500XXIH\n",
      "23 ratings\n"
     ]
    }
   ],
   "source": [
    "# Cleaning the data\n",
    "\n",
    "price = price.strip()[0:6]\n",
    "title = title.strip()\n",
    "total_rate = total_rate.strip()\n",
    "\n",
    "\n",
    "\n",
    "print(price)\n",
    "print(title)\n",
    "print(total_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "eebb234b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# insert the timestamp on which data was inserted in the csv\n",
    "\n",
    "today = datetime.date.today()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "8b3ee92f",
   "metadata": {},
   "outputs": [],
   "source": [
    "header = ['Title', 'Price', 'TotalRatings', 'Date']\n",
    "\n",
    "data = [title, price, total_rate, today]\n",
    "\n",
    "with open('AmazonDataset.csv', 'w', newline='', encoding='UTF8') as f:\n",
    "    writer = csv.writer(f)\n",
    "    writer.writerow(header)\n",
    "    writer.writerow(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0f42aa5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def send_mail():\n",
    "    server = smtplib.SMTP_SSL('smtp.gmail.com',465)\n",
    "    server.ehlo()\n",
    "    #server.starttls()\n",
    "    server.ehlo()\n",
    "    server.login('xxxx-xxxx@gmail.com','xxxxxxxxxxxxxx')\n",
    "    \n",
    "    subject = \"Be aware about the product\"\n",
    "    body = \"We have recieved lots of suspicion about this current product's related products, and as this laptop has fewer comments, please be aware about this product.\"\n",
    "   \n",
    "    msg = f\"Subject: {subject}\\n\\n{body}\"\n",
    "    \n",
    "    server.sendmail(\n",
    "        'xxxx-xxxx@gmail.com',\n",
    "        msg\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c04869a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# If a product has less than 10 ratings, send mail to the customer, kind of giving a caution to him/her about product \n",
    "\n",
    "if(total_rate < 10):\n",
    "    send_mail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c2952f2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Viewing the csv file in python only using pandas\n",
    "\n",
    "df = pd.read_csv(r\"C:\\Users\\gaurav\\AmazonDataset.csv\")\n",
    "\n",
    "print(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "c56eebdf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Start automating the process of appending data\n",
    "\n",
    "def auto_data():\n",
    "    \n",
    "    URL = 'https://www.amazon.in/Lenovo-inches-Business-Windows-82C500XXIH/dp/B0979M4T4X/ref=sr_1_13?brr=1&qid=1680180682&rd=1&sr=8-13'\n",
    "\n",
    "    headers = {\"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36\", \"Accept-Encoding\":\"gzip, deflate, br\", \"Accept\":\"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7\", \"DNT\":\"1\",\"Connection\":\"close\", \"Upgrade-Insecure-Requests\":\"1\"}\n",
    "\n",
    "    page = requests.get(URL, headers=headers)\n",
    "\n",
    "    soup1 = BeautifulSoup(page.content, \"html.parser\")\n",
    "    \n",
    "    soup2 = BeautifulSoup(soup1.prettify(), \"html.parser\")\n",
    "    \n",
    "    title = soup2.find(id='productTitle').get_text()\n",
    "\n",
    "    price = soup2.find('span',class_='a-price-whole').get_text()\n",
    "\n",
    "    total_rate = soup2.find(id='acrCustomerReviewText').get_text()\n",
    "\n",
    "    today = datetime.date.today()\n",
    "    \n",
    "    header = ['Title', 'Price', 'TotalRatings', 'Date']\n",
    "\n",
    "    data = [title, price, total_rate, today]\n",
    "    \n",
    "    with open('AmazonDataset.csv', 'w', newline='', encoding='UTF8') as f:\n",
    "        writer = csv.writer(f)\n",
    "        writer.writerow(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1efaf3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# calling the function\n",
    "\n",
    "while(True):\n",
    "    auto_data()\n",
    "    time.sleep(86400)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aeac7c0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Viewing the dataset\n",
    "\n",
    "print(df)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}