-
Notifications
You must be signed in to change notification settings - Fork 0
/
scraper.py
executable file
·67 lines (51 loc) · 2.57 KB
/
scraper.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
import sys
import re
import os
import pandas as pd
from pandas import ExcelWriter
import urllib.request
from bs4 import BeautifulSoup
from extractVendors import ExtractVendors
from extractProducts import ExtractProducts
from extractCategories import ExtractCategories
from extractIndustries import ExtractIndustries
from postProcess import PostProcess
base_url = 'https://dir.indiamart.com'
def scrape(base_url,out_dir='./out',log_dir=None):
industries = ExtractIndustries(base_url)
g_categories = pd.DataFrame(columns={'Name','URL','Industry'})
for industry in industries.itertuples():
categories = ExtractCategories(base_url, industry)
g_categories = g_categories.append(categories,ignore_index=True,sort=False)
cat_found = len(g_categories.index)
print(f"##########\n#########\nFound {cat_found} categories TOTAL\nFetching products ...\n##############\n##############\n")
i=1
g_products = pd.DataFrame(columns={'Name','URL','subCategory','Category','Industry'})
for category in g_categories.itertuples():
products = ExtractProducts(base_url, category)
g_products = g_products.append(products,ignore_index=True,sort=False)
print(f'\n\n~~~~~~~~~~\n{i*100/cat_found}% of categories scanned\n~~~~~~~~~~~~~~\n\n')
i += 1
prod_found = len(g_products.index)
print(f"##########\n#########\nFound {prod_found} products TOTAL\nFetching vendors ...\n##############\n##############\n")
i=1
g_vendors = pd.DataFrame(columns={'Name','URL','Phone','Address','Category','Industry'})
for product in g_products.itertuples():
vendors = ExtractVendors(base_url, product)
g_vendors = g_vendors.append(products,ignore_index=True,sort=False)
print(f'\n\n@@@@@@@@@@@@@\n{i*100/prod_found}% of products scanned\n@@@@@@@@@@@@@@@\n\n')
i += 1
writer = ExcelWriter(out_dir + '/all_vendors.xlsx') # TO DO : adapt script to write multiple sheets per file, one industry per file
g_vendors.sort_values('Name',inplace=True)
g_vendors.drop_duplicates('URL',inplace=True)
g_vendors.to_excel(writer, index=False)
writer.save()
writer.close()
print(f"Found {len(g_vendors.index)} products TOTAL !!\nSaved data at {out_dir}/all_vendors.xlsx\n")
return 0
scrape(base_url)
# categories = pd.DataFrame(columns={'Name', 'URL','Industry'})
# industries = pd.DataFrame(columns={'Name', 'URL'})
# sellers = pd.DataFrame(columns=['Name', 'URL', 'Phone', 'Address','Category','Industry'])
# sellers, categories, industries = scrape(sellers, categories, industries)
# PostProcess()