[cleanup] Misc (for 2024.01 release)

.gitignore

@@ -33,20 +33,23 @@ cookies
 *.gif
 *.jpeg
 *.jpg
+*.lrc
 *.m4a
 *.m4v
 *.mhtml
 *.mkv
 *.mov
 *.mp3
 *.mp4
+*.mpg
 *.mpga
 *.oga
 *.ogg
 *.opus
 *.png
 *.sbv
 *.srt
+*.ssa
 *.swf
 *.swp
 *.tt

Makefile

@@ -17,8 +17,8 @@ pypi-files: AUTHORS Changelog.md LICENSE README.md README.txt supportedsites \
 clean-test:
 	rm -rf test/testdata/sigs/player-*.js tmp/ *.annotations.xml *.aria2 *.description *.dump *.frag \
 	*.frag.aria2 *.frag.urls *.info.json *.live_chat.json *.meta *.part* *.tmp *.temp *.unknown_video *.ytdl \
-	*.3gp *.ape *.ass *.avi *.desktop *.f4v *.flac *.flv *.gif *.jpeg *.jpg *.m4a *.m4v *.mhtml *.mkv *.mov *.mp3 \
-	*.mp4 *.mpga *.oga *.ogg *.opus *.png *.sbv *.srt *.swf *.swp *.tt *.ttml *.url *.vtt *.wav *.webloc *.webm *.webp
+	*.3gp *.ape *.ass *.avi *.desktop *.f4v *.flac *.flv *.gif *.jpeg *.jpg *.lrc *.m4a *.m4v *.mhtml *.mkv *.mov *.mp3 *.mp4 \
+	*.mpg *.mpga *.oga *.ogg *.opus *.png *.sbv *.srt *.ssa *.swf *.swp *.tt *.ttml *.url *.vtt *.wav *.webloc *.webm *.webp
 clean-dist:
 	rm -rf yt-dlp.1.temp.md yt-dlp.1 README.txt MANIFEST build/ dist/ .coverage cover/ yt-dlp.tar.gz completions/ \
 	yt_dlp/extractor/lazy_extractors.py *.spec CONTRIBUTING.md.tmp yt-dlp yt-dlp.exe yt_dlp.egg-info/ AUTHORS

bundle/__init__.py

@@ -1 +0,0 @@
-# Empty file

bundle/py2exe.py

@@ -20,7 +20,7 @@ def main():
         'py2exe builds do not support pycryptodomex and needs VC++14 to run. '
         'It is recommended to run "pyinst.py" to build using pyinstaller instead')
 
-    return freeze(
+    freeze(
         console=[{
             'script': './yt_dlp/__main__.py',
             'dest_base': 'yt-dlp',

devscripts/__init__.py

@@ -1 +0,0 @@
-# Empty file needed to make devscripts.utils properly importable from outside

devscripts/changelog_override.json

@@ -120,5 +120,11 @@
         "when": "15f22b4880b6b3f71f350c64d70976ae65b9f1ca",
         "short": "[webvtt] Allow spaces before newlines for CueBlock (#7681)",
         "authors": ["TSRBerry"]
+    },
+    {
+        "action": "change",
+        "when": "4ce57d3b873c2887814cbec03d029533e82f7db5",
+        "short": "[ie] Support multi-period MPD streams (#6654)",
+        "authors": ["alard", "pukkandan"]
     }
 ]

devscripts/make_changelog.py

@@ -253,7 +253,7 @@ class CommitRange:
         ''', re.VERBOSE | re.DOTALL)
     EXTRACTOR_INDICATOR_RE = re.compile(r'(?:Fix|Add)\s+Extractors?', re.IGNORECASE)
     REVERT_RE = re.compile(r'(?:\[[^\]]+\]\s+)?(?i:Revert)\s+([\da-f]{40})')
-    FIXES_RE = re.compile(r'(?i:Fix(?:es)?(?:\s+bugs?)?(?:\s+in|\s+for)?|Revert)\s+([\da-f]{40})')
+    FIXES_RE = re.compile(r'(?i:Fix(?:es)?(?:\s+bugs?)?(?:\s+in|\s+for)?|Revert|Improve)\s+([\da-f]{40})')
     UPSTREAM_MERGE_RE = re.compile(r'Update to ytdl-commit-([\da-f]+)')
 
     def __init__(self, start, end, default_author=None):

test/test_networking.py

@@ -69,7 +69,7 @@ def do_GET(self):
             self.send_response(200)
             self.send_header('Content-Type', 'text/plain; charset=utf-8')
             self.end_headers()
-            self.wfile.write('{self.proxy_name}: {self.path}'.format(self=self).encode())
+            self.wfile.write(f'{self.proxy_name}: {self.path}'.encode())
     return HTTPTestRequestHandler
 
 

yt_dlp/YoutubeDL.py

@@ -575,7 +575,7 @@ class YoutubeDL:
         'url', 'manifest_url', 'manifest_stream_number', 'ext', 'format', 'format_id', 'format_note',
         'width', 'height', 'aspect_ratio', 'resolution', 'dynamic_range', 'tbr', 'abr', 'acodec', 'asr', 'audio_channels',
         'vbr', 'fps', 'vcodec', 'container', 'filesize', 'filesize_approx', 'rows', 'columns',
-        'player_url', 'protocol', 'fragment_base_url', 'fragments', 'is_from_start',
+        'player_url', 'protocol', 'fragment_base_url', 'fragments', 'is_from_start', 'is_dash_periods', 'request_data',
         'preference', 'language', 'language_preference', 'quality', 'source_preference', 'cookies',
         'http_headers', 'stretched_ratio', 'no_resume', 'has_drm', 'extra_param_to_segment_url', 'hls_aes', 'downloader_options',
         'page_url', 'app', 'play_path', 'tc_url', 'flash_version', 'rtmp_live', 'rtmp_conn', 'rtmp_protocol', 'rtmp_real_time'

yt_dlp/dependencies/__init__.py

@@ -53,9 +53,7 @@
 
 try:
     import websockets
-except (ImportError, SyntaxError):
-    # websockets 3.10 on python 3.6 causes SyntaxError
-    # See https://github.com/yt-dlp/yt-dlp/issues/2633
+except ImportError:
     websockets = None
 
 try:

yt_dlp/extractor/abematv.py

@@ -53,7 +53,7 @@ def __init__(self, ie: 'AbemaTVIE'):
         # the protocol that this should really handle is 'abematv-license://'
         # abematv_license_open is just a placeholder for development purposes
         # ref. https://github.com/python/cpython/blob/f4c03484da59049eb62a9bf7777b963e2267d187/Lib/urllib/request.py#L510
-        setattr(self, 'abematv-license_open', getattr(self, 'abematv_license_open'))
+        setattr(self, 'abematv-license_open', getattr(self, 'abematv_license_open', None))
         self.ie = ie
 
     def _get_videokey_from_ticket(self, ticket):

yt_dlp/extractor/adultswim.py

@@ -107,7 +107,6 @@ def _real_extract(self, url):
       title
       tvRating
     }''' % episode_path
-            ['getVideoBySlug']
         else:
             query = query % '''metaDescription
     title

yt_dlp/extractor/altcensored.py

@@ -4,6 +4,7 @@
 from .common import InfoExtractor
 from ..utils import (
     InAdvancePagedList,
+    clean_html,
     int_or_none,
     orderedSet,
     str_to_int,
@@ -32,23 +33,23 @@ class AltCensoredIE(InfoExtractor):
             'duration': 926.09,
             'thumbnail': 'https://archive.org/download/youtube-k0srjLSkga8/youtube-k0srjLSkga8.thumbs/k0srjLSkga8_000925.jpg',
             'view_count': int,
-            'categories': ['News & Politics'],  # FIXME
+            'categories': ['News & Politics'],
         }
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
+        category = clean_html(self._html_search_regex(
+            r'<a href="/category/\d+">([^<]+)</a>', webpage, 'category', default=None))
 
         return {
             '_type': 'url_transparent',
             'url': f'https://archive.org/details/youtube-{video_id}',
             'ie_key': ArchiveOrgIE.ie_key(),
             'view_count': str_to_int(self._html_search_regex(
                 r'YouTube Views:(?:\s|&nbsp;)*([\d,]+)', webpage, 'view count', default=None)),
-            'categories': self._html_search_regex(
-                r'<a href="/category/\d+">\s*\n?\s*([^<]+)</a>',
-                webpage, 'category', default='').split() or None,
+            'categories': [category] if category else None,
         }
 
 

yt_dlp/extractor/antenna.py

@@ -67,7 +67,7 @@ def _real_extract(self, url):
         webpage = self._download_webpage(url, video_id)
         info = self._download_and_extract_api_data(video_id, netloc)
         info['description'] = self._og_search_description(webpage, default=None)
-        info['_old_archive_ids'] = [make_archive_id('Ant1NewsGrWatch', video_id)],
+        info['_old_archive_ids'] = [make_archive_id('Ant1NewsGrWatch', video_id)]
         return info
 
 

yt_dlp/extractor/arte.py

@@ -142,10 +142,10 @@ class ArteTVIE(ArteTVBaseIE):
     def _fix_accessible_subs_locale(subs):
         updated_subs = {}
         for lang, sub_formats in subs.items():
-            for format in sub_formats:
-                if format.get('url', '').endswith('-MAL.m3u8'):
+            for fmt in sub_formats:
+                if fmt.get('url', '').endswith('-MAL.m3u8'):
                     lang += '-acc'
-                updated_subs.setdefault(lang, []).append(format)
+                updated_subs.setdefault(lang, []).append(fmt)
         return updated_subs
 
     def _real_extract(self, url):

yt_dlp/extractor/beatbump.py

@@ -3,7 +3,7 @@
 
 
 class BeatBumpVideoIE(InfoExtractor):
-    _VALID_URL = r'https://beatbump\.(?:ml|io)/listen\?id=(?P<id>[\w-]+)'
+    _VALID_URL = r'https?://beatbump\.(?:ml|io)/listen\?id=(?P<id>[\w-]+)'
     _TESTS = [{
         'url': 'https://beatbump.ml/listen?id=MgNrAu2pzNs',
         'md5': '5ff3fff41d3935b9810a9731e485fe66',
@@ -48,7 +48,7 @@ def _real_extract(self, url):
 
 
 class BeatBumpPlaylistIE(InfoExtractor):
-    _VALID_URL = r'https://beatbump\.(?:ml|io)/(?:release\?id=|artist/|playlist/)(?P<id>[\w-]+)'
+    _VALID_URL = r'https?://beatbump\.(?:ml|io)/(?:release\?id=|artist/|playlist/)(?P<id>[\w-]+)'
     _TESTS = [{
         'url': 'https://beatbump.ml/release?id=MPREb_gTAcphH99wE',
         'playlist_count': 50,

yt_dlp/extractor/bilibili.py

@@ -1965,6 +1965,7 @@ class BiliIntlIE(BiliIntlBaseIE):
         'only_matching': True,
     }]
 
+    @staticmethod
     def _make_url(video_id, series_id=None):
         if series_id:
             return f'https://www.bilibili.tv/en/play/{series_id}/{video_id}'

yt_dlp/extractor/cableav.py

@@ -2,7 +2,7 @@
 
 
 class CableAVIE(InfoExtractor):
-    _VALID_URL = r'https://cableav\.tv/(?P<id>[a-zA-Z0-9]+)'
+    _VALID_URL = r'https?://cableav\.tv/(?P<id>[a-zA-Z0-9]+)'
     _TESTS = [{
         'url': 'https://cableav.tv/lS4iR9lWjN8/',
         'md5': '7e3fe5e49d61c4233b7f5b0f69b15e18',

yt_dlp/extractor/camfm.py

@@ -13,7 +13,7 @@
 
 
 class CamFMShowIE(InfoExtractor):
-    _VALID_URL = r'https://(?:www\.)?camfm\.co\.uk/shows/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?camfm\.co\.uk/shows/(?P<id>[^/]+)'
     _TESTS = [{
         'playlist_mincount': 5,
         'url': 'https://camfm.co.uk/shows/soul-mining/',
@@ -42,7 +42,7 @@ def _real_extract(self, url):
 
 
 class CamFMEpisodeIE(InfoExtractor):
-    _VALID_URL = r'https://(?:www\.)?camfm\.co\.uk/player/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?camfm\.co\.uk/player/(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'https://camfm.co.uk/player/43336',
         'skip': 'Episode will expire - don\'t actually know when, but it will go eventually',

yt_dlp/extractor/cineverse.py

@@ -13,7 +13,7 @@
 
 
 class CineverseBaseIE(InfoExtractor):
-    _VALID_URL_BASE = r'https://www\.(?P<host>%s)' % '|'.join(map(re.escape, (
+    _VALID_URL_BASE = r'https?://www\.(?P<host>%s)' % '|'.join(map(re.escape, (
         'cineverse.com',
         'asiancrush.com',
         'dovechannel.com',

yt_dlp/extractor/common.py

@@ -747,7 +747,7 @@ def extract(self, url):
             raise
         except ExtractorError as e:
             e.video_id = e.video_id or self.get_temp_id(url)
-            e.ie = e.ie or self.IE_NAME,
+            e.ie = e.ie or self.IE_NAME
             e.traceback = e.traceback or sys.exc_info()[2]
             raise
         except IncompleteRead as e:
@@ -1339,7 +1339,10 @@ def _get_netrc_login_info(self, netrc_machine=None):
         else:
             return None, None
         if not info:
-            raise netrc.NetrcParseError(f'No authenticators for {netrc_machine}')
+            self.to_screen(f'No authenticators for {netrc_machine}')
+            return None, None
+
+        self.write_debug(f'Using netrc for {netrc_machine} authentication')
         return info[0], info[2]
 
     def _get_login_info(self, username_option='username', password_option='password', netrc_machine=None):

yt_dlp/extractor/cybrary.py

@@ -110,7 +110,7 @@ def _real_extract(self, url):
 
 
 class CybraryCourseIE(CybraryBaseIE):
-    _VALID_URL = r'https://app\.cybrary\.it/browse/course/(?P<id>[\w-]+)/?(?:$|[#?])'
+    _VALID_URL = r'https?://app\.cybrary\.it/browse/course/(?P<id>[\w-]+)/?(?:$|[#?])'
     _TESTS = [{
         'url': 'https://app.cybrary.it/browse/course/az-500-microsoft-azure-security-technologies',
         'info_dict': {

yt_dlp/extractor/duoplay.py

@@ -13,7 +13,7 @@
 
 
 class DuoplayIE(InfoExtractor):
-    _VALID_URL = r'https://duoplay\.ee/(?P<id>\d+)/[\w-]+/?(?:\?(?:[^#]+&)?ep=(?P<ep>\d+))?'
+    _VALID_URL = r'https?://duoplay\.ee/(?P<id>\d+)/[\w-]+/?(?:\?(?:[^#]+&)?ep=(?P<ep>\d+))?'
     _TESTS = [{
         'note': 'Siberi võmm S02E12',
         'url': 'https://duoplay.ee/4312/siberi-vomm?ep=24',

yt_dlp/extractor/egghead.py

@@ -19,7 +19,7 @@ def _call_api(self, path, video_id, resource, fatal=True):
 class EggheadCourseIE(EggheadBaseIE):
     IE_DESC = 'egghead.io course'
     IE_NAME = 'egghead:course'
-    _VALID_URL = r'https://(?:app\.)?egghead\.io/(?:course|playlist)s/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:app\.)?egghead\.io/(?:course|playlist)s/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'https://egghead.io/courses/professor-frisby-introduces-composable-functional-javascript',
         'playlist_count': 29,
@@ -65,7 +65,7 @@ def _real_extract(self, url):
 class EggheadLessonIE(EggheadBaseIE):
     IE_DESC = 'egghead.io lesson'
     IE_NAME = 'egghead:lesson'
-    _VALID_URL = r'https://(?:app\.)?egghead\.io/(?:api/v1/)?lessons/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:app\.)?egghead\.io/(?:api/v1/)?lessons/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'https://egghead.io/lessons/javascript-linear-data-flow-with-container-style-types-box',
         'info_dict': {

yt_dlp/extractor/gamejolt.py

@@ -88,7 +88,7 @@ def _parse_post(self, post_data):
             'uploader_id': user_data.get('username'),
             'uploader_url': format_field(user_data, 'url', 'https://gamejolt.com%s'),
             'categories': [try_get(category, lambda x: '%s - %s' % (x['community']['name'], x['channel'].get('display_title') or x['channel']['title']))
-                           for category in post_data.get('communities' or [])],
+                           for category in post_data.get('communities') or []],
             'tags': traverse_obj(
                 lead_content, ('content', ..., 'content', ..., 'marks', ..., 'attrs', 'tag'), expected_type=str_or_none),
             'like_count': int_or_none(post_data.get('like_count')),

yt_dlp/extractor/getcourseru.py

@@ -160,9 +160,8 @@ def _real_extract(self, url):
             self._login(hostname, username, password)
 
         display_id = self._match_id(url)
-        # NB: 404 is returned due to yt-dlp not properly following redirects #9020
-        webpage, urlh = self._download_webpage_handle(url, display_id, expected_status=404)
-        if self._LOGIN_URL_PATH in urlh.url or urlh.status == 404:
+        webpage, urlh = self._download_webpage_handle(url, display_id)
+        if self._LOGIN_URL_PATH in urlh.url:
             raise ExtractorError(
                 f'This video is only available for registered users. {self._login_hint("any", netrc=hostname)}',
                 expected=True)

yt_dlp/extractor/itprotv.py

@@ -31,7 +31,7 @@ def _check_if_logged_in(self, webpage):
 
 
 class ITProTVIE(ITProTVBaseIE):
-    _VALID_URL = r'https://app\.itpro\.tv/course/(?P<course>[\w-]+)/(?P<id>[\w-]+)'
+    _VALID_URL = r'https?://app\.itpro\.tv/course/(?P<course>[\w-]+)/(?P<id>[\w-]+)'
     _TESTS = [{
         'url': 'https://app.itpro.tv/course/guided-tour/introductionitprotv',
         'md5': 'bca4a28c2667fd1a63052e71a94bb88c',

yt_dlp/extractor/kommunetv.py

@@ -3,7 +3,7 @@
 
 
 class KommunetvIE(InfoExtractor):
-    _VALID_URL = r'https://\w+\.kommunetv\.no/archive/(?P<id>\w+)'
+    _VALID_URL = r'https?://\w+\.kommunetv\.no/archive/(?P<id>\w+)'
     _TEST = {
         'url': 'https://oslo.kommunetv.no/archive/921',
         'md5': '5f102be308ee759be1e12b63d5da4bbc',

yt_dlp/extractor/lecturio.py

@@ -172,7 +172,7 @@ def _real_extract(self, url):
 
 
 class LecturioCourseIE(LecturioBaseIE):
-    _VALID_URL = r'https://app\.lecturio\.com/(?:[^/]+/(?P<nt>[^/?#&]+)\.course|(?:#/)?course/c/(?P<id>\d+))'
+    _VALID_URL = r'https?://app\.lecturio\.com/(?:[^/]+/(?P<nt>[^/?#&]+)\.course|(?:#/)?course/c/(?P<id>\d+))'
     _TESTS = [{
         'url': 'https://app.lecturio.com/medical-courses/microbiology-introduction.course#/',
         'info_dict': {
@@ -209,7 +209,7 @@ def _real_extract(self, url):
 
 
 class LecturioDeCourseIE(LecturioBaseIE):
-    _VALID_URL = r'https://(?:www\.)?lecturio\.de/[^/]+/(?P<id>[^/?#&]+)\.kurs'
+    _VALID_URL = r'https?://(?:www\.)?lecturio\.de/[^/]+/(?P<id>[^/?#&]+)\.kurs'
     _TEST = {
         'url': 'https://www.lecturio.de/jura/grundrechte.kurs',
         'only_matching': True,

yt_dlp/extractor/medaltv.py

@@ -9,7 +9,6 @@
     int_or_none,
     str_or_none,
     traverse_obj,
-    update_url_query,
 )
 
 
@@ -82,7 +81,7 @@ class MedalTVIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(update_url_query(url, {'mobilebypass': 'true'}), video_id)
+        webpage = self._download_webpage(url, video_id, query={'mobilebypass': 'true'})
 
         hydration_data = self._search_json(
             r'<script[^>]*>[^<]*\bhydrationData\s*=', webpage,

yt_dlp/extractor/megaphone.py

@@ -5,7 +5,7 @@
 class MegaphoneIE(InfoExtractor):
     IE_NAME = 'megaphone.fm'
     IE_DESC = 'megaphone.fm embedded players'
-    _VALID_URL = r'https://player\.megaphone\.fm/(?P<id>[A-Z0-9]+)'
+    _VALID_URL = r'https?://player\.megaphone\.fm/(?P<id>[A-Z0-9]+)'
     _EMBED_REGEX = [rf'<iframe[^>]*?\ssrc=["\'](?P<url>{_VALID_URL})']
     _TEST = {
         'url': 'https://player.megaphone.fm/GLT9749789991',

yt_dlp/extractor/minoto.py

@@ -21,7 +21,7 @@ def _real_extract(self, url):
                 continue
             container = fmt.get('container')
             if container == 'hls':
-                formats.extend(fmt_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
+                formats.extend(self._extract_m3u8_formats(fmt_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
             else:
                 fmt_profile = fmt.get('profile') or {}
                 formats.append({

yt_dlp/extractor/monstercat.py

@@ -16,7 +16,7 @@
 
 
 class MonstercatIE(InfoExtractor):
-    _VALID_URL = r'https://www\.monstercat\.com/release/(?P<id>\d+)'
+    _VALID_URL = r'https?://www\.monstercat\.com/release/(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://www.monstercat.com/release/742779548009',
         'playlist_count': 20,