-
Notifications
You must be signed in to change notification settings - Fork 0
/
data_extraction.py
103 lines (87 loc) · 2.49 KB
/
data_extraction.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
from bs4 import BeautifulSoup
import numpy as np
import pandas as pd
# Opening the whole html file and reading it
with open('flipkart_laptops.html', 'r', encoding='utf-8') as f:
laptop_html = f.read()
soup = BeautifulSoup(laptop_html, 'html.parser')
container = soup.find_all('div', {'class': '_2kHMtA'})
# Running a FOR loop and creating lists to store various data elements
name = []
price = []
rating = []
no_of_ratings_nd_reviews = []
core = []
rom = []
os = []
ram = []
display = []
waranty_period = []
for i in container:
# Name column
try:
title = i.find('div', {'class': '_4rR01T'})
name.append(title.text)
except:
name.append(np.nan)
# Price column
try:
kimat = i.find('div', {'class': '_30jeq3 _1_WHN1'})
price.append(kimat.text)
except:
price.append(np.nan)
# Stars column
try:
stars = i.find('div', {'class': '_3LWZlK'})
rating.append(stars.text)
except:
rating.append(np.nan)
# No. of ratings and reviews column
try:
ratings_nd_reviews = i.find('span', {'class': '_2_R_DZ'})
no_of_ratings_nd_reviews.append(ratings_nd_reviews.find('span').text)
except:
no_of_ratings_nd_reviews.append(np.nan)
x = i.find('ul', {'class': '_1xgFaf'}).find_all('li') # For detailed specs
# Core column
try:
core.append(x[0].text)
except:
core.append(np.nan)
# ROM column
try:
rom.append(x[1].text)
except:
rom.append(np.nan)
# OS column
try:
os.append(x[2].text)
except:
os.append(np.nan)
# RAM column
try:
ram.append(x[3].text)
except:
ram.append(np.nan)
# Display column
try:
display.append(x[4].text)
except:
display.append(np.nan)
# Warranty period
try:
waranty_period.append(x[5].text)
except:
waranty_period.append(np.nan)
# Creating a dictionary and putting it in a variable
data_to_add = {'Name': name, 'Price': price, 'Stars': rating, 'Total ratings and reviews': no_of_ratings_nd_reviews,
'Core': core, 'Read Only Memory (ROM)': rom, 'Operating System': os, 'Random Access Memory (RAM)': ram,
'Display': display,
'Warranty Period': waranty_period}
# Appending the dictionary into a DataFrame
df = pd.DataFrame(data_to_add)
# Setting the index right
df.index +=1
print(df)
# Exporting the created DataFrame to a CSV file
df.to_csv('flipkart_laptops_february.csv')