add single downloader for single ivod

ivod_downloader.py

@@ -102,6 +102,7 @@ def get_date_list(comt, start_date=None, end_date=None):
             return False
     except:
         sys.stderr.write('get_date_list web error, comtid: %s\n' % comt)
+        reset_cookie()
         return False
 
 def get_movie_by_date(comit, date, page=1):
@@ -117,6 +118,7 @@ def get_movie_by_date(comit, date, page=1):
         web = urllib2.urlopen(req)
     except:
         sys.stderr.write('get_movie_by_date web error, comit: %s, date: %s\n' % (comit, date))
+        reset_cookie()
         return False
     if web.getcode() == 200:
         html_result = web.read()
@@ -153,6 +155,7 @@ def get_movie_url(wzs_id, t, quality='w'):
     except:
         random_sleep()
         sys.stderr.write('get_movie_url web error, wzs_id: %s, t: %s\n' % (wzs_id, t))
+        reset_cookie()
         return False
     #print web.getcode()
     if web.getcode() == 200:

ivod_single_downloader.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import os, urllib, urllib2, cookielib, sys, random, time, datetime, subprocess
+from BeautifulSoup import BeautifulSoup, SoupStrainer
+from optparse import OptionParser
+
+if __name__ == '__main__':
+    os.chdir(os.path.dirname(__file__))
+
+reload(sys)
+sys.setdefaultencoding('utf-8')
+
+currect_time = 0
+
+def check_url(url):
+    req = None
+    try:
+        req = urllib.urlopen(url)
+        return req.getcode() == 200
+    except:
+        return False
+
+def init_cookie():
+    cookie=cookielib.CookieJar()
+    opener=urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie))
+    urllib2.install_opener(opener)
+    reset_cookie()
+
+def reset_cookie():
+    global currect_time
+    #if time lagger then 15 min, will reset.
+    if time.time() - currect_time > 900:
+        http_header = {'User-Agent': 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)', 'Host': 'ivod.ly.gov.tw'}
+        req = urllib2.Request('http://ivod.ly.gov.tw/', None, http_header)
+        try:
+            web = urllib2.urlopen(req)
+            result = web.read()
+            currect_time = time.time()
+        except:
+            sys.stderr.write('reset cookie error\n')
+            return False
+        #print result
+
+def test_php():
+    cmd = 'php test_ext.php'
+    result = os.system(cmd)
+    return result == 0
+
+def get_movie_url(url):
+    http_header = {'Referer': 'http://ivod.ly.gov.tw/Committee', 
+        'User-Agent': 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)', 
+        'Host': 'ivod.ly.gov.tw',
+        'Connection': 'keep-alive'}
+
+    url = url.replace('300K', '1M')
+    req = urllib2.Request(url, None, http_header)
+    try:
+        web = urllib2.urlopen(req)
+    except:
+        sys.stderr.write('get_movie_url web error')
+        reset_cookie()
+        return False
+    #print web.getcode()
+    if web.getcode() == 200:
+        html_result = web.read()
+        #print html_result
+        xml = BeautifulSoup(html_result)
+        text_block = xml.find('div', {'class': 'movie_box clearfix'}).find('div', {'class':'text'})
+        meet = text_block.find('h4').text.replace(u'會議別 ：', u'').replace(u'委員會', u'')
+        name = text_block.findAll('p')[1].text.replace(u'委  員  名  稱：', u'')
+        date = text_block.findAll('p')[4].text.replace(u'會  議  時  間：', u'').split(' ')[0]
+        filename = '%s %s %s' % (date, meet, name)
+
+        div_movie = xml.find('div', {'class': 'movie'})
+        if not div_movie:
+            div_movie = xml.find('div', {'class': 'movie_large'})
+        #print div_movie
+        if div_movie:
+            #print div_movie
+            script_text = div_movie.find('script').text
+            script_text = script_text.replace("readyPlayer('http://ivod.ly.gov.tw/public/scripts/','", '')
+            script_text = script_text.replace("');", '')
+            #print script_text
+            return_code1 = subprocess.call(['php', 'AdobeHDS.php', '--quality', 'high', '--delete', '--manifest', script_text, '--outfile', filename])
+            #return script_text
+            return return_code1
+        #return xml
+    else:
+        sys.stderr.write('get_movie_url content error')
+        return False
+
+def main():
+    usage = "usage: %prog [options]"
+    parser = OptionParser(usage)
+    parser.add_option("-u", "--url", dest="url",
+                      help='IVOD Url')
+    (options, args) = parser.parse_args()
+
+    if not test_php():
+        print "Please check PHP extensions."
+        sys.exit(1)
+    reset_cookie()
+    if not options.url:
+        print 'Please input url.'
+        sys.exit(1)
+    get_movie_url(options.url)
+
+
+if __name__ == '__main__':
+    main()