-
Notifications
You must be signed in to change notification settings - Fork 0
/
scraper.py
100 lines (78 loc) · 2.82 KB
/
scraper.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
from selenium.webdriver.common.by import By
from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support.ui import WebDriverWait
import ssl
from email.message import EmailMessage
import datetime
import smtplib
import pandas as pd
import requests
from bs4 import BeautifulSoup
from price_parser import Price
import schedule
import time
# config
SAVE_TO_CSV = True
SEND_MAIL = True
waitlist_CSV = "waitlist.csv"
aftertracking_CSV = "aftertracking.csv"
# Edit your email address
mail_user = ‘’
mail_pass = ‘’
mail_to = ‘’
def get_urls(csv_file):
# Use pandas to read csv data
dataframe = pd.read_csv(csv_file)
return dataframe
def process_products(dataframe):
updated_products = []
# converting the DataFrame into a list of dictionaries.
for product in dataframe.to_dict("records"):
# automating web broser by using selenium...click functions to be added..
driver = webdriver.Chrome(ChromeDriverManager().install())
driver.get(product["url"])
page = driver.page_source
product["price"] = get_price(page)
product["result"] = product["price"] < product["expected_price"]
updated_products.append(product)
return pd.DataFrame(updated_products)
def get_response(url):
response = requests.get(url)
return response.text
def get_price(page_html):
soup = BeautifulSoup(page_html, "lxml")
try:
el = soup.select_one(".item-price__normal")
price = Price.fromstring(el.text)
except:
el = soup.select_one(".item-price__price-amount")
price = Price.fromstring(el.text)
return price.amount_float
def send_mail(dataframe):
subject = 'Msg from you Indigo Tracker!'
if dataframe[dataframe["result"]].to_string() == 'true':
body = f"Good News! The price is lower than your expected price! BUY IT NOW! \n{datetime.datetime(2021, 9, 1)}\nThank you for trying INDIGO tracker~\nBest regards,\nXinyi"
else:
body = f"OOPS! It seems too expensive \n{datetime.datetime(2021, 9, 1)}\nThank you for trying INDIGO tracker~\nBest regards,\nXinyi"
em = EmailMessage()
em['Form'] = mail_user
em["To"] = mail_to
em['subject'] = subject
em.set_content(body)
context = ssl.create_default_context()
with smtplib.SMTP_SSL('smtp.gmail.com', 465, context=context) as smtp:
smtp.login(mail_user, mail_pass)
smtp.sendmail(mail_user, mail_to, em.as_string())
def main():
dataframe = get_urls(waitlist_CSV)
dataframe_updated = process_products(dataframe)
if SAVE_TO_CSV:
dataframe_updated.to_csv(aftertracking_CSV, index=False, mode="a")
if SEND_MAIL:
send_mail(dataframe_updated)
main()
# schedule.every().day.at("10:30").do(main)
# while True:
# schedule.run_pending()
# time.sleep(1)