-
Notifications
You must be signed in to change notification settings - Fork 0
/
webscraping_beautifulsoup4
98 lines (71 loc) · 3.24 KB
/
webscraping_beautifulsoup4
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
"""
@author: Engy Tawadros
# Project Dependency :
# pip install selenium
# pip install webdriver_manager
# pip install beautifulsoup4
"""
import time
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from bs4 import BeautifulSoup
# Chrome Options -------------------------------------:
chrome_options = Options()
chrome_options.add_argument('incognito') ## Opens Chrome in incognito mode
# chrome_options.add_experimental_option("detach", False) ## chromedriver will stay open afterward
# chrome_options.add_argument('--headless') ## chrome window will not pop up
#### ----------------------------------------->
driver = webdriver.Chrome(options=chrome_options)
# driver.get('https://www.google.com/') # testing
amazon_base_url = "https://www.amazon.com"
#product = 'B07BNNQJSL' # lindt
product = 'B00NGKWYS2' # Cadbury Dairy Milk
amazon_product_url = amazon_base_url + "/dp/" + product
driver.get(amazon_product_url)
time.sleep(3) # imp to sleep
page_title = driver.title # get webpage title
print('\n'+page_title)
#################### {page_source} ##################
html_page_source = driver.page_source
# print(html_page_source)
################## Beautiful Soup #############
## Beautiful Soup Documentation
# https://www.crummy.com/software/BeautifulSoup/bs4/doc/
soup = BeautifulSoup(html_page_source, "html.parser")
# print(soup)
# print(soup.prettify())
# print(soup.get_text())
################## 2 - Get Review Link #################
review_link = soup.find("a", {'data-hook': "see-all-reviews-link-foot"})
review_link = review_link['href']
print(review_link)
################## 3- Open Reviews page #################
reviews_url = amazon_base_url + review_link
driver.get(reviews_url)
# # print(reviews_url)
html2 = driver.page_source
soup = BeautifulSoup(html2, "html.parser")
################################################################################################
print('2 ways to pasrse the data:')
# print(soup.find('i',{'data-hook': "review-star-rating"} ))
print(soup.find("a", {'data-hook': "product-link"}).text)
print(driver.find_element("xpath", '//*[@id="cm_cr-product_info"]/div/div[2]/div/div/div[2]/div[1]/h1/a').text)
print('-------------------------------------------------------------')
product_link = soup.find("a", {'data-hook': "product-link"}).text.strip("\n")
reviews_title = soup.find("span", {'data-hook': "review-title"}).text.strip("\n")
reviews_body = soup.find("span", {'data-hook': "review-body"}).text.strip("\n")
reviews_date = soup.find("span", {'data-hook': "review-date"}).text.strip("\n")
reviews_stars = soup.find("i", {'data-hook': "review-star-rating"}).text.strip("\n")
helpful = soup.find("div", {'class': "cr-helpful-text"})
print(f'product-link: {product_link}')
print(f'reviews_title: {reviews_title}')
print(f'reviews_body: {reviews_body}')
print(f'reviews_date: {reviews_date}')
print(f'reviews_stars: {reviews_stars}')
print(f'helpful: {helpful}')
import pandas as pd
df = pd.DataFrame({"product-link": [product_link], "reviews_title": [reviews_title], "reviews_body": [reviews_body],
"reviews_date": [reviews_date], "reviews_stars": [reviews_stars]})
print(df)
df.to_csv("amazon_product.csv", index=False)