Merge pull request ytdl-org#4 from ytdl-org/master

kobiburnley · May 9, 2019 · aa0bccf · aa0bccf
2 parents 574be1b + a5b92d3
commit aa0bccf
Show file tree

Hide file tree

Showing 4 changed files with 54 additions and 22 deletions.
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
@@ -1033,7 +1033,10 @@
     SkyNewsArabiaIE,
     SkyNewsArabiaArticleIE,
 )
-from .skysports import SkySportsIE
+from .sky import (
+    SkyNewsIE,
+    SkySportsIE,
+)
 from .slideshare import SlideshareIE
 from .slideslive import SlidesLiveIE
 from .slutload import SlutloadIE

diff --git a/youtube_dl/extractor/skysports.py → youtube_dl/extractor/sky.py b/youtube_dl/extractor/skysports.py → youtube_dl/extractor/sky.py
@@ -10,34 +10,25 @@
 )
 
 
-class SkySportsIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?skysports\.com/watch/video/(?P<id>[0-9]+)'
-    _TEST = {
-        'url': 'http://www.skysports.com/watch/video/10328419/bale-its-our-time-to-shine',
-        'md5': '77d59166cddc8d3cb7b13e35eaf0f5ec',
-        'info_dict': {
-            'id': '10328419',
-            'ext': 'mp4',
-            'title': 'Bale: It\'s our time to shine',
-            'description': 'md5:e88bda94ae15f7720c5cb467e777bb6d',
-        },
-        'add_ie': ['Ooyala'],
-    }
-
+class SkyBaseIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         video_data = extract_attributes(self._search_regex(
-            r'(<div.+?class="sdc-article-video__media-ooyala"[^>]+>)', webpage, 'video data'))
+            r'(<div.+?class="[^"]*sdc-article-video__media-ooyala[^"]*"[^>]+>)',
+            webpage, 'video data'))
 
         video_url = 'ooyala:%s' % video_data['data-video-id']
         if video_data.get('data-token-required') == 'true':
-            token_fetch_options = self._parse_json(video_data.get('data-token-fetch-options', '{}'), video_id, fatal=False) or {}
+            token_fetch_options = self._parse_json(video_data.get(
+                'data-token-fetch-options', '{}'), video_id, fatal=False) or {}
             token_fetch_url = token_fetch_options.get('url')
             if token_fetch_url:
-                embed_token = self._download_webpage(urljoin(url, token_fetch_url), video_id, fatal=False)
+                embed_token = self._download_webpage(urljoin(
+                    url, token_fetch_url), video_id, fatal=False)
                 if embed_token:
-                    video_url = smuggle_url(video_url, {'embed_token': embed_token.strip('"')})
+                    video_url = smuggle_url(
+                        video_url, {'embed_token': embed_token.strip('"')})
 
         return {
             '_type': 'url_transparent',
@@ -47,3 +38,33 @@ def _real_extract(self, url):
             'description': strip_or_none(self._og_search_description(webpage)),
             'ie_key': 'Ooyala',
         }
+
+
+class SkySportsIE(SkyBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?skysports\.com/watch/video/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://www.skysports.com/watch/video/10328419/bale-its-our-time-to-shine',
+        'md5': '77d59166cddc8d3cb7b13e35eaf0f5ec',
+        'info_dict': {
+            'id': 'o3eWJnNDE6l7kfNO8BOoBlRxXRQ4ANNQ',
+            'ext': 'mp4',
+            'title': 'Bale: It\'s our time to shine',
+            'description': 'md5:e88bda94ae15f7720c5cb467e777bb6d',
+        },
+        'add_ie': ['Ooyala'],
+    }
+
+
+class SkyNewsIE(SkyBaseIE):
+    _VALID_URL = r'https?://news\.sky\.com/video/[0-9a-z-]+-(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'https://news.sky.com/video/russian-plane-inspected-after-deadly-fire-11712962',
+        'md5': 'd6327e581473cea9976a3236ded370cd',
+        'info_dict': {
+            'id': '1ua21xaDE6lCtZDmbYfl8kwsKLooJbNM',
+            'ext': 'mp4',
+            'title': 'Russian plane inspected after deadly fire',
+            'description': 'The Russian Investigative Committee has released video of the wreckage of a passenger plane which caught fire near Moscow.',
+        },
+        'add_ie': ['Ooyala'],
+    }
diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
@@ -526,7 +526,7 @@ def _real_extract(self, url):
 
 class YahooGyaOIE(InfoExtractor):
     IE_NAME = 'yahoo:gyao'
-    _VALID_URL = r'https?://(?:gyao\.yahoo\.co\.jp/p|streaming\.yahoo\.co\.jp/p/y)/(?P<id>\d+/v\d+)'
+    _VALID_URL = r'https?://(?:gyao\.yahoo\.co\.jp/(?:p|title/[^/]+)|streaming\.yahoo\.co\.jp/p/y)/(?P<id>\d+/v\d+|[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
     _TESTS = [{
         'url': 'https://gyao.yahoo.co.jp/p/00449/v03102/',
         'info_dict': {
@@ -536,6 +536,9 @@ class YahooGyaOIE(InfoExtractor):
     }, {
         'url': 'https://streaming.yahoo.co.jp/p/y/01034/v00133/',
         'only_matching': True,
+    }, {
+        'url': 'https://gyao.yahoo.co.jp/title/%E3%81%97%E3%82%83%E3%81%B9%E3%81%8F%E3%82%8A007/5b025a49-b2e5-4dc7-945c-09c6634afacf',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
@@ -2100,8 +2100,13 @@ def _extract_filesize(media_url):
         else:
             self._downloader.report_warning('unable to extract uploader nickname')
 
-        channel_id = self._html_search_meta(
-            'channelId', video_webpage, 'channel id')
+        channel_id = (
+            str_or_none(video_details.get('channelId')) or
+            self._html_search_meta(
+                'channelId', video_webpage, 'channel id', default=None) or
+            self._search_regex(
+                r'data-channel-external-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
+                video_webpage, 'channel id', default=None, group='id'))
         channel_url = 'http://www.youtube.com/channel/%s' % channel_id if channel_id else None
 
         # thumbnail image