Merge pull request #177 from blackjack4494/master

Release 2020.10.09

.gitignore

@@ -60,3 +60,5 @@ venv/
 
 # VS Code related files
 .vscode
+
+cookies.txt

youtube_dlc/extractor/bandcamp.py

@@ -99,7 +99,6 @@ def _real_extract(self, url):
             webpage, 'track info', default='{}')
 
         track_info = self._parse_json(trackinfo_block, title)
-
         if track_info:
             file_ = track_info.get('file')
             if isinstance(file_, dict):
@@ -115,7 +114,7 @@ def _real_extract(self, url):
                         'acodec': ext,
                         'abr': int_or_none(abr_str),
                     })
-            track = track_info.get('title')
+
             track_id = str_or_none(track_info.get('track_id') or track_info.get('id'))
             track_number = int_or_none(track_info.get('track_num'))
             duration = float_or_none(track_info.get('duration'))
@@ -126,6 +125,7 @@ def extract(key):
                 webpage, key, default=None, group='value')
             return data.replace(r'\"', '"').replace('\\\\', '\\') if data else data
 
+        track = extract('title')
         artist = extract('artist')
         album = extract('album_title')
         timestamp = unified_timestamp(

youtube_dlc/extractor/bet.py

@@ -3,6 +3,8 @@
 from .mtv import MTVServicesInfoExtractor
 from ..utils import unified_strdate
 
+# TODO Remove - Reason: Outdated Site
+
 
 class BetIE(MTVServicesInfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?bet\.com/(?:[^/]+/)+(?P<id>.+?)\.html'

youtube_dlc/extractor/cmt.py

@@ -2,6 +2,8 @@
 
 from .mtv import MTVIE
 
+# TODO Remove - Reason: Outdated Site
+
 
 class CMTIE(MTVIE):
     IE_NAME = 'cmt.com'
@@ -39,7 +41,7 @@ class CMTIE(MTVIE):
         'only_matching': True,
     }]
 
-    def _extract_mgid(self, webpage):
+    def _extract_mgid(self, webpage, url):
         mgid = self._search_regex(
             r'MTVN\.VIDEO\.contentUri\s*=\s*([\'"])(?P<mgid>.+?)\1',
             webpage, 'mgid', group='mgid', default=None)
@@ -50,5 +52,5 @@ def _extract_mgid(self, webpage):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        mgid = self._extract_mgid(webpage)
+        mgid = self._extract_mgid(webpage, url)
         return self.url_result('http://media.mtvnservices.com/embed/%s' % mgid)

youtube_dlc/extractor/comedycentral.py

@@ -48,7 +48,7 @@ class ComedyCentralFullEpisodesIE(MTVServicesInfoExtractor):
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
         webpage = self._download_webpage(url, playlist_id)
-        mgid = self._extract_triforce_mgid(webpage, data_zone='t2_lc_promo1')
+        mgid = self._extract_mgid(webpage, url, data_zone='t2_lc_promo1')
         videos_info = self._get_videos_info(mgid)
         return videos_info
 

youtube_dlc/extractor/expressen.py

@@ -15,7 +15,7 @@
 class ExpressenIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                     https?://
-                        (?:www\.)?expressen\.se/
+                        (?:www\.)?(?:expressen|di)\.se/
                         (?:(?:tvspelare/video|videoplayer/embed)/)?
                         tv/(?:[^/]+/)*
                         (?P<id>[^/?#&]+)
@@ -42,13 +42,16 @@ class ExpressenIE(InfoExtractor):
     }, {
         'url': 'https://www.expressen.se/videoplayer/embed/tv/ditv/ekonomistudion/experterna-har-ar-fragorna-som-avgor-valet/?embed=true&external=true&autoplay=true&startVolume=0&partnerId=di',
         'only_matching': True,
+    }, {
+        'url': 'https://www.di.se/videoplayer/embed/tv/ditv/borsmorgon/implantica-rusar-70--under-borspremiaren-hor-styrelsemedlemmen/?embed=true&external=true&autoplay=true&startVolume=0&partnerId=di',
+        'only_matching': True,
     }]
 
     @staticmethod
     def _extract_urls(webpage):
         return [
             mobj.group('url') for mobj in re.finditer(
-                r'<iframe[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//(?:www\.)?expressen\.se/(?:tvspelare/video|videoplayer/embed)/tv/.+?)\1',
+                r'<iframe[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//(?:www\.)?(?:expressen|di)\.se/(?:tvspelare/video|videoplayer/embed)/tv/.+?)\1',
                 webpage)]
 
     def _real_extract(self, url):

youtube_dlc/extractor/iprima.py

@@ -86,7 +86,8 @@ def _real_extract(self, url):
             (r'<iframe[^>]+\bsrc=["\'](?:https?:)?//(?:api\.play-backend\.iprima\.cz/prehravac/embedded|prima\.iprima\.cz/[^/]+/[^/]+)\?.*?\bid=(p\d+)',
              r'data-product="([^"]+)">',
              r'id=["\']player-(p\d+)"',
-             r'playerId\s*:\s*["\']player-(p\d+)'),
+             r'playerId\s*:\s*["\']player-(p\d+)',
+             r'\bvideos\s*=\s*["\'](p\d+)'),
             webpage, 'real id')
 
         playerpage = self._download_webpage(

youtube_dlc/extractor/mtv.py

@@ -7,6 +7,7 @@
 from ..compat import (
     compat_str,
     compat_xpath,
+    compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
@@ -22,6 +23,7 @@
     unescapeHTML,
     update_url_query,
     url_basename,
+    get_domain,
     xpath_text,
 )
 
@@ -253,7 +255,42 @@ def _extract_triforce_mgid(self, webpage, data_zone=None, video_id=None):
 
         return try_get(feed, lambda x: x['result']['data']['id'], compat_str)
 
-    def _extract_mgid(self, webpage):
+    def _extract_new_triforce_mgid(self, webpage, url='', video_id=None):
+        # print(compat_urlparse.urlparse(url).netloc)
+        if url == '':
+            return
+        domain = get_domain(url)
+        if domain is None:
+            raise ExtractorError(
+                '[%s] could not get domain' % self.IE_NAME,
+                expected=True)
+        url = url.replace("https://", "http://")
+        enc_url = compat_urlparse.quote(url, safe='')
+        _TRIFORCE_V8_TEMPLATE = 'https://%s/feeds/triforce/manifest/v8?url=%s'
+        triforce_manifest_url = _TRIFORCE_V8_TEMPLATE % (domain, enc_url)
+
+        manifest = self._download_json(triforce_manifest_url, video_id, fatal=False)
+        if manifest:
+            if manifest.get('manifest').get('type') == 'redirect':
+                self.to_screen('Found a redirect. Downloading manifest from new location')
+                new_loc = manifest.get('manifest').get('newLocation')
+                new_loc = new_loc.replace("https://", "http://")
+                enc_new_loc = compat_urlparse.quote(new_loc, safe='')
+                triforce_manifest_new_loc = _TRIFORCE_V8_TEMPLATE % (domain, enc_new_loc)
+                manifest = self._download_json(triforce_manifest_new_loc, video_id, fatal=False)
+
+        item_id = try_get(manifest, lambda x: x['manifest']['reporting']['itemId'], compat_str)
+        if not item_id:
+            self.to_screen('Found no id!')
+            return
+
+        # 'episode' can be anything. 'content' is used often as well
+        _MGID_TEMPLATE = 'mgid:arc:episode:%s:%s'
+        mgid = _MGID_TEMPLATE % (domain, item_id)
+
+        return mgid
+
+    def _extract_mgid(self, webpage, url, data_zone=None):
         try:
             # the url can be http://media.mtvnservices.com/fb/{mgid}.swf
             # or http://media.mtvnservices.com/{mgid}
@@ -276,14 +313,17 @@ def _extract_mgid(self, webpage):
                 r'embed/(mgid:.+?)["\'&?/]', sm4_embed, 'mgid', default=None)
 
         if not mgid:
-            mgid = self._extract_triforce_mgid(webpage)
+            mgid = self._extract_new_triforce_mgid(webpage, url)
+
+        if not mgid:
+            mgid = self._extract_triforce_mgid(webpage, data_zone)
 
         return mgid
 
     def _real_extract(self, url):
         title = url_basename(url)
         webpage = self._download_webpage(url, title)
-        mgid = self._extract_mgid(webpage)
+        mgid = self._extract_mgid(webpage, url)
         videos_info = self._get_videos_info(mgid)
         return videos_info
 

youtube_dlc/extractor/nick.py

@@ -245,5 +245,5 @@ class NickRuIE(MTVServicesInfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        mgid = self._extract_mgid(webpage)
+        mgid = self._extract_mgid(webpage, url)
         return self.url_result('http://media.mtvnservices.com/embed/%s' % mgid)

youtube_dlc/extractor/spike.py

@@ -20,8 +20,18 @@ class BellatorIE(MTVServicesInfoExtractor):
     _FEED_URL = 'http://www.bellator.com/feeds/mrss/'
     _GEO_COUNTRIES = ['US']
 
-    def _extract_mgid(self, webpage):
-        return self._extract_triforce_mgid(webpage)
+    def _extract_mgid(self, webpage, url):
+        mgid = None
+
+        if not mgid:
+            mgid = self._extract_triforce_mgid(webpage)
+
+        if not mgid:
+            mgid = self._extract_new_triforce_mgid(webpage, url)
+
+        return mgid
+
+# TODO Remove - Reason: Outdated Site
 
 
 class ParamountNetworkIE(MTVServicesInfoExtractor):
@@ -43,7 +53,7 @@ class ParamountNetworkIE(MTVServicesInfoExtractor):
     _FEED_URL = 'http://www.paramountnetwork.com/feeds/mrss/'
     _GEO_COUNTRIES = ['US']
 
-    def _extract_mgid(self, webpage):
+    def _extract_mgid(self, webpage, url):
         root_data = self._parse_json(self._search_regex(
             r'window\.__DATA__\s*=\s*({.+})',
             webpage, 'data'), None)

youtube_dlc/extractor/vh1.py

@@ -3,6 +3,8 @@
 
 from .mtv import MTVServicesInfoExtractor
 
+# TODO Remove - Reason: Outdated Site
+
 
 class VH1IE(MTVServicesInfoExtractor):
     IE_NAME = 'vh1.com'