-
Notifications
You must be signed in to change notification settings - Fork 0
/
douban_gallary_downloader.py
executable file
·97 lines (79 loc) · 2.31 KB
/
douban_gallary_downloader.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
#!/usr/bin/python
#it's ugly ,but it works :-)
from BeautifulSoup import BeautifulSoup
import urllib2
import urllib
import sys
import os
import time
#proxy = urllib2.ProxyHandler({'http': '172.19.1.2:8217'})
#opener = urllib2.build_opener(proxy)
#urllib2.install_opener(opener)
if len(sys.argv) != 4:
print "usage: ./xxx douban_url out_path pic_cnt"
sys.exit(1)
input_url = sys.argv[1]
output_path = sys.argv[2]
all_cnt = int(sys.argv[3])
if os.path.isdir(output_path):
os.chdir(output_path)
else:
print "dir not exist ,mkdir "
os.mkdir(output_path)
os.chdir(output_path)
if input_url[:7]!="http://":
input_url = "http://"+input_url
headers = {'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Charset':'GB2312,utf-8;q=0.7,*;q=0.7',
'Accept-Language':'zh-cn,zh;q=0.5',
'Cache-Control':'max-age=0',
'Connection':'keep-alive',
'Keep-Alive':'115',
'Referer':"http://www.renren.com",
'User-Agent':'Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.14) Gecko/20110221 Ubuntu/10.10 (maverick) Firefox/3.6.14'
}
now_url = input_url
pic_cnt = 0
soup=None
pic_list=None
try:
output_path = output_path.decode("utf-8")
except Exception:
try:
output_path = output_path.decode("gbk")
except Exception:
print "can not decode path "
sys.exit(0)
print "input url is "+ input_url
print "output path is " + output_path
while True:
try:
req = urllib2.Request(now_url, headers=headers)
response = urllib2.urlopen(req)
doc = response.read()
soup = BeautifulSoup(doc)
pic_list = soup.findAll(attrs={"class":"mainphoto"})
print "html markup is "+str(pic_list)
next_url = soup.findAll(attrs={"class":"mainphoto"})[0]["href"]
if next_url.find("douban.com") == -1:
next_url = "http://www.douban.com"+next_url
print "next url is "+next_url
pic_url = soup.findAll(attrs={"class":"mainphoto"})[0].contents[1]["src"]
print "pic url is "+pic_url
pic_content = urllib.urlopen(pic_url).read()
pic_name = pic_url.split("/")[-1]
pic_file = open(pic_name,"wb")
pic_file.write(pic_content)
pic_file.close()
pic_cnt += 1
print "save pic to "+pic_name +"in dir "+output_path
time.sleep(6)
if next_url == input_url:
break
else:
now_url = next_url
except Exception,err:
print err
print pic_list
break
print "downloaded "+ str(pic_cnt)+" in all :-)"