AMAZON_WEB_SCRAPING.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyOc1n9d0rd0iq4pXRkB1bU/",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/gerardbullock/gerardbullock.github.io/blob/main/AMAZON_WEB_SCRAPING.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from IPython.display import Image\n",
        "Image(url=\"https://designcoral.com/wp-content/uploads/2019/10/What-is-Web-Scraping.jpg\", width=700, height=400)"
      ],
      "metadata": {
        "id": "hAkm1b_Abp0z",
        "outputId": "5c87ade9-47b9-444c-bf05-8b2d1adffc5a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 421
        }
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<img src=\"https://designcoral.com/wp-content/uploads/2019/10/What-is-Web-Scraping.jpg\" width=\"700\" height=\"400\"/>"
            ],
            "text/plain": [
              "<IPython.core.display.Image object>"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# AMAZON WEB SCRAPING"
      ],
      "metadata": {
        "id": "3izaH7uQI7lr"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1UdZ7kG9HpI_"
      },
      "outputs": [],
      "source": [
        "# import libraries \n",
        "\n",
        "from bs4 import BeautifulSoup\n",
        "import pandas as pd\n",
        "import requests\n",
        "import time\n",
        "import datetime\n",
        "import smtplib\n",
        "import csv"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Connect to Website and pull in data\n",
        "\n",
        "url = 'https://www.amazon.com/Funny-Data-Systems-Business-Analyst/dp/B07FNW9FGJ/ref=sr_1_6?crid=3A75AAJNOU5X0&keywords=data+analyst+shirt&qid=1667376203&qu=eyJxc2MiOiIzLjkyIiwicXNhIjoiMy42OCIsInFzcCI6IjEuOTIifQ%3D%3D&sprefix=data+analyst+shirt%2Caps%2C327&sr=8-6'\n",
        " \n",
        "headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0',\n",
        "           'Accept-Language': 'en-US, en;q=0.5'\n",
        "}\n"
      ],
      "metadata": {
        "id": "QkULTGAYKIGI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "page = requests.get(url, headers = headers)\n",
        "page"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "O00nkiVSKq7-",
        "outputId": "ca29f117-2238-4931-9f17-5f40f803574b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<Response [200]>"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "soup1 = BeautifulSoup(page.content, \"html.parser\")\n",
        "soup2 = BeautifulSoup(soup1.prettify(), \"html.parser\")"
      ],
      "metadata": {
        "id": "QdK8VQV2K23V"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "title = soup2.find(id='productTitle').get_text()\n",
        "price = soup2.find('span', class_=\"a-offscreen\").get_text()"
      ],
      "metadata": {
        "id": "EDIDmRQwK7fJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(title)\n",
        "print(price)"
      ],
      "metadata": {
        "id": "TOZ0iPieLEQ8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7e499bf0-9c60-4d54-ead6-90a57651c939"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "                   Funny Got Data MIS Data Systems Business Analyst T-Shirt\n",
            "                  \n",
            "\n",
            "                  $16.99\n",
            "                 \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Clean up the data\n",
        "\n",
        "price = price.strip()[1:]\n",
        "title = title.strip()\n",
        "\n",
        "print(title)\n",
        "print(price)"
      ],
      "metadata": {
        "id": "GeBFLvVFLJGc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "409da40a-5112-48b3-cf13-5934afc20693"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Funny Got Data MIS Data Systems Business Analyst T-Shirt\n",
            "16.99\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Create a Timestamp for your output to track when data was collected\n",
        "\n",
        "import datetime\n",
        "\n",
        "today = datetime.date.today()\n",
        "\n",
        "print(today)"
      ],
      "metadata": {
        "id": "gUGRKdDHLPN-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c7b26ba3-17c1-4ce9-9f3d-5fe2e7864e3a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2022-12-14\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Create CSV and write headers and data into the file\n",
        "\n",
        "header = ['Title', 'Price', 'Date']\n",
        "data = [title, price, today]\n",
        "\n",
        "\n",
        "with open('AmazonWebScraperDataset.csv', 'w', newline='', encoding='UTF8') as f:\n",
        "    writer = csv.writer(f)\n",
        "    writer.writerow(header)\n",
        "    writer.writerow(data)"
      ],
      "metadata": {
        "id": "z2vZru5nLZcj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "df = pd.read_csv('/content/AmazonWebScraperDataset.csv')\n",
        "\n",
        "print(df)"
      ],
      "metadata": {
        "id": "KLonNmwsLbnc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "615d324d-9a81-4880-d452-39bb45eaeccc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "                                               Title  Price        Date\n",
            "0  Funny Got Data MIS Data Systems Business Analy...  16.99  2022-12-14\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Now we are appending data to the csv\n",
        "\n",
        "with open('AmazonWebScraperDataset.csv', 'a+', newline='', encoding='UTF8') as f:\n",
        "    writer = csv.writer(f)\n",
        "    writer.writerow(data)"
      ],
      "metadata": {
        "id": "b4hFFIITY2Ez"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Combine all of the above code into one function\n",
        "\n",
        "\n",
        "def check_price():\n",
        "    URL = 'https://www.amazon.com/Funny-Data-Systems-Business-Analyst/dp/B07FNW9FGJ/ref=sr_1_3?dchild=1&keywords=data%2Banalyst%2Btshirt&qid=1626655184&sr=8-3&customId=B0752XJYNL&th=1'\n",
        "\n",
        "    headers = {\"User-Agent\":'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36 Edg/108.0.1462.46'}\n",
        "    #headers = {\"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36\", \"Accept-Encoding\":\"gzip, deflate\", \"Accept\":\"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\", \"DNT\":\"1\",\"Connection\":\"close\", \"Upgrade-Insecure-Requests\":\"1\"}\n",
        "\n",
        "    page = requests.get(URL, headers=headers)\n",
        "\n",
        "    soup1 = BeautifulSoup(page.content, \"html.parser\")\n",
        "\n",
        "    soup2 = BeautifulSoup(soup1.prettify(), \"html.parser\")\n",
        "\n",
        "    title = soup2.find(id='productTitle').get_text()\n",
        "\n",
        "    #price = soup2.find(id='priceblock_ourprice').get_text()\n",
        "    price = soup2.find('span', class_=\"a-offscreen\").get_text()\n",
        "\n",
        "    price = price.strip()[1:]\n",
        "    title = title.strip()\n",
        "\n",
        "    import datetime\n",
        "\n",
        "    today = datetime.date.today()\n",
        "    \n",
        "    import csv \n",
        "\n",
        "    header = ['Title', 'Price', 'Date']\n",
        "    data = [title, price, today]\n",
        "\n",
        "    with open('AmazonWebScraperDataset.csv', 'a+', newline='', encoding='UTF8') as f:\n",
        "        writer = csv.writer(f)\n",
        "        writer.writerow(data)"
      ],
      "metadata": {
        "id": "5Dvz9R9wY9bt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Runs check_price after a set time and inputs data into your CSV\n",
        "\n",
        "while(True):\n",
        "    price\n",
        "    time.sleep(100)"
      ],
      "metadata": {
        "id": "fizP3E0DbB3I"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "import pandas as pd\n",
        "\n",
        "df = pd.read_csv('/content/AmazonWebScraperDataset.csv')\n",
        "\n",
        "print(df)"
      ],
      "metadata": {
        "id": "SLr-ru94n8_Q"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# If uou want to try sending yourself an email (just for fun) when a price hits below a certain level you can try it\n",
        "# out with this script\n",
        "\n",
        "def send_mail():\n",
        "    server = smtplib.SMTP_SSL('smtp.gmail.com',465)\n",
        "    server.ehlo()\n",
        "    #server.starttls()\n",
        "    server.ehlo()\n",
        "    server.login('gbullock48@gmail.com','xxxxxxxxxxxxxx')\n",
        "    \n",
        "    subject = \"The Shirt you want is below $15! Now is your chance to buy!\"\n",
        "    body = \"Gerard, This is the moment we have been waiting for. Now is your chance to pick up the shirt of your dreams. Don't mess it up! Link here: https://www.amazon.com/Funny-Data-Systems-Business-Analyst/dp/B07FNW9FGJ/ref=sr_1_3?dchild=1&keywords=data+analyst+tshirt&qid=1626655184&sr=8-3\"\n",
        "   \n",
        "    msg = f\"Subject: {subject}\\n\\n{body}\"\n",
        "    \n",
        "    server.sendmail(\n",
        "        'gbullock48@gmail.com',\n",
        "        msg\n",
        "     \n",
        "    )"
      ],
      "metadata": {
        "id": "PCkEWUsSoNcr"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}