-
Notifications
You must be signed in to change notification settings - Fork 5
/
minispider.py
81 lines (57 loc) · 1.64 KB
/
minispider.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
#!/usr/bin/python
import httplib
import sys
import re
from HTMLParser import HTMLParser
class miniHTMLParser( HTMLParser ):
viewedQueue = []
instQueue = []
def get_next_link( self ):
if self.instQueue == []:
return ''
else:
return self.instQueue.pop(0)
def gethtmlfile( self, site, page ):
try:
httpconn = httplib.HTTPConnection(site)
httpconn.request("GET", page)
resp = httpconn.getresponse()
resppage = resp.read()
except:
resppage = ""
return resppage
def handle_starttag( self, tag, attrs ):
if tag == 'a':
newstr = str(attrs[0][1])
if re.search('http', newstr) == None:
if re.search('mailto', newstr) == None:
if re.search('htm', newstr) != None:
if (newstr in self.viewedQueue) == False:
print " adding", newstr
self.instQueue.append( newstr )
self.viewedQueue.append( newstr )
else:
print " ignoring", newstr
else:
print " ignoring", newstr
else:
print " ignoring", newstr
def main():
if sys.argv[1] == '':
print "usage is ./minispider.py site link"
sys.exit(2)
mySpider = miniHTMLParser()
link = sys.argv[2]
while link != '':
print "\nChecking link ", link
# Get the file from the site and link
retfile = mySpider.gethtmlfile( sys.argv[1], link )
# Feed the file into the HTML parser
mySpider.feed(retfile)
# Search the retfile here
# Get the next link in level traversal order
link = mySpider.get_next_link()
mySpider.close()
print "\ndone\n"
if __name__ == "__main__":
main()