-
Notifications
You must be signed in to change notification settings - Fork 0
/
qooveeBot.py
142 lines (120 loc) · 5.53 KB
/
qooveeBot.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
import requests, bs4
import json
from fuzzywuzzy import fuzz
import re
from credentials.credentials import Credentials
import platform
from datetime import datetime
import time
class FetchProductData:
def __init__(self):
self.product_list = []
def product_details(self, link, unit_of_measurement, min_order_weight, product_origin, product_description, prices,currency,converted_min_price,converted_max_price):
min_price = None
max_price = None
if len(prices) == 1:
min_price = prices[0] #0=sameprice
max_price = prices[0] #0=sameprice
else:
min_price = prices[0] #0=oldprice
max_price = prices[1] #0=newprice
self.product_list.append(
{'description': product_description,
'min_price': prices,
'max_price': prices,
'currency': currency,
'weight': min_order_weight,
'unit': unit_of_measurement,
'origin': product_origin,
'link': link,
'website': "qoovee.com",
'converted_min_price':converted_min_price,
'converted_max_price':converted_max_price})
def find_data(self, description, origin,credentials):
data = None
api_result = None
web_soup = None
check = True
while(check):
try:
base_url = 'https://www.qoovee.com/en/search/?q=' #qoovee search
params = credentials.get_proxycrawl_credentails(url = base_url + description)
params['token']='dB35Y2RoUcjbCxqapH1w0w'
params['page_wait']=3000
proxy_url = credentials.get_proxycrawl_url() #proxy
api_result = requests.get(proxy_url, params)
web_soup = bs4.BeautifulSoup(api_result.content, "html.parser")
check = False
except Exception as e:
print(e)
check = True
pass
if len(web_soup.findAll('div',{'id':'goods'})) != 0: #class
data = self.fetch_data(web_soup, description, origin)
print(data)
else:
data = []
def fetch_data(self, web_soup, description, origin):
list_of_products = web_soup.findAll('div', {'class':'market_item-inner'}) #class
price_list = web_soup.findAll('div',{'itemprop':'offers'})
index=0
try:
for Data_from_websites in list_of_products:
price_from_website = price_list[index].text
dullprice = '\nSpecify the price\n'
if price_from_website is dullprice:
price_from_website = "Price Unavailable"
else:
price_from_website = price_list[index].text
min_order_weight = None
min_order_unit = None
link = None
product_description = None
prices = None
currency = None
convertedprice1 = None
convertedprice2 = None
if price_from_website is not None and price_from_website != '':
try:
link = Data_from_websites.find('h4')
product_description = Data_from_websites.find('span',{'itemprop':'name'}).text
P1= price_from_website
currency ="USD"
unit_of_measurement = None
prices=str(P1).replace("USD",'').split('/')
min_order_weight = 1
description = description.lower()
product_description = product_description.lower()
convertedprice1=0
convertedprice2=0
except Exception as ex:
print(ex)
print("line 201 EXCEPTION")
product_description = product_description.lower()
product_origin = "USA"
description = description.lower()
reCheck = re.search(r'(\D*)?(\d+)\s*/?(\w+)$', product_description)
if reCheck is not None and len(reCheck.groups()) != 0:
if len(reCheck.groups()) > 1:
unit_of_measurement= reCheck.groups()[-1]
min_order_weight = reCheck.groups()[-2]
if description in product_description:
self.product_details(link, unit_of_measurement, min_order_weight, product_origin,
product_description, prices,currency,convertedprice1,convertedprice2)
elif fuzz.token_set_ratio(description, product_description) >= 85:
self.product_details(link, min_order_unit, min_order_weight, product_origin,
product_description, prices,currency,convertedprice1,convertedprice2)
else:
print("price not fetched")
index = index + 1
except:
pass
return self.product_list
if __name__ == "__main__":
origin = ''
description = 'acid' #search
credentials=Credentials()
fetch_products = FetchProductData()
fetch_products.find_data(description, origin, credentials)
#qoovee.com
#end