Merge pull request #4 from qrtt1/master

重構：調整專案結構與整理共用程式碼

downloader.py → ivod/downloader.py

@@ -21,34 +21,10 @@ def download_adobe_hds(manifest_url, filename, **kwargs):
     return 1
 
 
-def _deprecated_download_adobe_hds(manifest_url, filename, **kwargs):
-
-    cmd = ['php',
-           'AdobeHDS.php',
-           '--quality',
-           'high',
-           '--delete',
-           '--manifest',
-           manifest_url,
-           '--outfile',
-           filename]
-
-    if "outdir" in kwargs:
-        cmd.append("--outdir")
-        cmd.append(kwargs["outdir"])
-
-    if "maxspeed" in kwargs:
-        cmd.append("--maxspeed")
-        cmd.append(kwargs["maxspeed"])
-
-    ret = subprocess.call(cmd)
-    return ret
-
-
 class AdobeHDS():
 
     def download(self, url, filename):
-        from youtube_dl.downloader.f4m import *
+        from youtube_dl.downloader.f4m import F4mFD
         from youtube_dl import YoutubeDL
 
         ydl = YoutubeDL()

ivod/function_test.py

@@ -0,0 +1,61 @@
+import unittest
+from origin import ivod_downloader as origin_ivod_downloader
+from origin import ivod_single_downloader as origin_ivod_single_downloader
+
+import ivod_downloader
+import ivod_single_downloader
+
+
+class TestVODResults(unittest.TestCase):
+
+    def test_get_movie_by_date(self):
+        origin = origin_ivod_downloader.get_movie_by_date(8, '2014-10-09')
+        current = ivod_downloader.get_movie_by_date(8, '2014-10-09')
+
+        self.assertTrue(current)
+        self.assertEqual(origin, current)
+
+    def test_get_date_list(self):
+        origin = origin_ivod_downloader.get_date_list(
+            8,
+            '2014-03-01',
+            '2014-10-01')
+        current = ivod_downloader.get_date_list(8, '2014-03-01', '2014-10-01')
+
+        self.assertTrue(current)
+        self.assertEqual(origin, current)
+
+    def test_get_movie_by_date(self):
+        origin = origin_ivod_downloader.get_movie_by_date(8, '2014-10-01')
+        current = ivod_downloader.get_movie_by_date(8, '2014-10-01')
+
+        self.assertTrue(current)
+        self.assertEqual(origin, current)
+
+    def test_get_movie_url(self):
+        origin = origin_ivod_downloader.get_movie_url('74035', 'clip')
+        current = ivod_downloader.get_movie_url('74035', 'clip')
+
+        self.assertTrue(current)
+        self.assertEqual(origin, current)
+
+
+class TestVODSingleResults(unittest.TestCase):
+
+    def setUp(self):
+        self.download_url_args = []
+
+    def _mock_adobe_hds_downloader(self, manifest, filename):
+        self.download_url_args += [(manifest, filename)]
+
+    def test_single_downloader(self):
+        origin_ivod_single_downloader.download_adobe_hds = self._mock_adobe_hds_downloader
+        ivod_single_downloader.download_adobe_hds = self._mock_adobe_hds_downloader
+
+        origin_ivod_single_downloader.download_from_url(
+            'http://ivod.ly.gov.tw/Play/VOD/76394/300K')
+        ivod_single_downloader.download_from_url(
+            'http://ivod.ly.gov.tw/Play/VOD/76394/300K')
+
+        self.assertTrue(self.download_url_args[0])
+        self.assertEqual(self.download_url_args[0], self.download_url_args[1])

ivod/ivod_parser.py

@@ -0,0 +1,88 @@
+# -*- coding: utf-8 -*-
+import urllib2
+from request import build_request
+from BeautifulSoup import BeautifulSoup
+
+
+_SCRIPT_PLAYER_DELIMITER = "readyPlayer('http://ivod.ly.gov.tw/public/scripts/','"
+
+
+def _extract_manifest_from_player_script(content):
+    content = content.replace(_SCRIPT_PLAYER_DELIMITER, '')
+    content = content.replace("');", '')
+
+    return content
+
+
+def _extract_text_info_from_player_page(url, content):
+    meet = None
+    name = None
+    date = None
+    filename = None
+
+    if 'VOD' in url:
+        meet = content.find('h4').text.replace(
+            u'會議別 ：',
+            u'').replace(
+            u'委員會',
+            u'')
+        name = content.findAll('p')[1].text.replace(u'委  員  名  稱：', u'')
+        date = content.findAll('p')[4].text.replace(
+            u'會  議  時  間：',
+            u'').split(' ')[0]
+        filename = '%s_%s_%s.flv' % (date, meet, name)
+    elif 'FULL' in url:
+        meet = content.find('h4').text.replace(
+            u'會議別 ：',
+            u'').replace(
+            u'委員會',
+            u'')
+        date = content.findAll('p')[1].text.replace(
+            u'會  議  時  間：',
+            u'').split(' ')[0]
+        filename = '%s_%s.flv' % (date, meet)
+
+    return {'filename': filename, 'date': date, 'name': name, 'meet': meet}
+
+def _supported_url(url):
+    return 'http://ivod.ly.gov.tw/Play/' in url
+
+
+def extract_manifest_from_player_page(url, ensure_high_quality_video=False):
+
+    if not _supported_url(url):
+        return None
+
+    if ensure_high_quality_video:
+        url = url.replace('300K', '1M')
+
+    req = build_request(url, None)
+    try:
+        resp = urllib2.urlopen(req)
+        if resp.code != 200:
+            return None
+
+        content = resp.read()
+        xml = BeautifulSoup(content)
+        div_movie = xml.find('div', {'class': 'movie'})
+
+        movie_info = xml.find(
+            'div', {
+                'class': 'movie_box clearfix'}).find(
+            'div', {
+                'class': 'text'})
+        info = _extract_text_info_from_player_page(url, movie_info)
+
+        if not div_movie:
+            div_movie = xml.find('div', {'class': 'movie_large'})
+
+        if div_movie:
+            script_text = div_movie.find('script').text
+            info['manifest'] = _extract_manifest_from_player_script(
+                script_text)
+            return info
+    except Exception as e:
+        raise e
+        pass
+
+    return None

ivod/request.py

@@ -0,0 +1,64 @@
+import json
+import urllib
+import urllib2
+
+
+class PresetURL(object):
+
+    COMMS_DATE = 'http://ivod.ly.gov.tw/Committee/CommsDate'
+    MOVIE_BY_DATE = 'http://ivod.ly.gov.tw/Committee/MovieByDate'
+
+
+def _default_header():
+    header = {'Referer': 'http://ivod.ly.gov.tw/Committee',
+              'Accept': '*/*',
+              'User-Agent': 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)',
+              'Host': 'ivod.ly.gov.tw',
+              'Connection': 'keep-alive',
+              'X-Requested-With': 'XMLHttpRequest',
+              'Pragma': 'no-cache'}
+
+    return header
+
+
+def enocded_request_body(data=None):
+    if data:
+        return urllib.urlencode(data)
+    return None
+
+
+def build_header(header):
+    if header:
+        return header
+    return _default_header()
+
+
+def build_request(url, data=None, header=None):
+    req = urllib2.Request(
+        url,
+        enocded_request_body(data),
+        build_header(header))
+    return req
+
+
+def get_content(url_or_alias, data=None):
+    url = url_or_alias
+    if url_or_alias in PresetURL.__dict__:
+        url = PresetURL.__dict__[url_or_alias]
+    req = build_request(url, data)
+
+    try:
+        resp = urllib2.urlopen(req)
+        if resp.code == 200:
+            return resp.read().decode('utf-8-sig')
+    except Exception as e:
+        pass
+
+    return None
+
+
+def get_json_content(url_or_alias, data=None):
+    content = get_content(url_or_alias, data)
+    if content:
+        return json.loads(content)
+    return None