forked from nayangupta824/disease_Scraper
-
Notifications
You must be signed in to change notification settings - Fork 1
/
scraper.py
80 lines (63 loc) · 1.84 KB
/
scraper.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
from bs4 import BeautifulSoup #Beautiful soup library used to pull data from html ot xml
import urllib2 #To fetch source of the webpage ... just thinking .. xD
#Now fetching the source
response = urllib2.urlopen("http://www.malacards.org/categories")
html = response.read()
soup = BeautifulSoup(html,'html.parser')
base = 'http://www.malacards.org'
diseases_list_link = []
dic = {}
for a in soup.findAll('a'):
if 'diseases' in a.text:
#goto diseases list under this category
diseases_list_link.append(base + a.get('href'))
#now we have the link to to diseases from sub-category of them
leng = 0
for sub_disease in diseases_list_link:
#goto first sub-category
print sub_disease
response1 = urllib2.urlopen(sub_disease)
html1 = response1.read()
soup1 = BeautifulSoup(html1,'html.parser')
#now we are at diseases list page
#html may have multiple tables so we store the list of those table
tables = soup1.findChildren('table')
#get the first table
my_table = tables[0]
rows = my_table.findChildren(['tr'])
r = 1
for row in rows:
cnt = 0
cells = row.findChildren('td')
inc = 0
for cell in cells:
cnt = cnt + 1
if cnt == 1:
c = cell.string
c = int(c)
if c == r:
r = r+1
inc = 1
if cnt == 4 and inc == 1:
value = cell.string
#print value
#value contains the name of the disease
link = base + cell.a.get('href')
#link contains the link to the desp of disease
dic[value] = link
leng = leng+1
#print leng
#print dic
print leng
for dis in dic:
print dis
response2 = urllib2.urlopen(dic[dis])
html2 = response2.read()
soup2 = BeautifulSoup(html2,'html.parser')
table = soup2.find('table',{'class':'borderless symptoms_table'})
sympt = []
if table != None:
lists = table.findChildren('li')
for li in lists:
sympt.append(li.string)
dic[dis] = sympt