Merge branch 'master' of https://github.com/yt-dlp/yt-dlp into ytdlp

* 'master' of https://github.com/yt-dlp/yt-dlp:
  [extractor/bilibili] Extract `flac` with premium account (#4759)
  [Build] Update pyinstaller
  [extractor/eurosport] Add extractor (#4613)
  [extractor/mediaset] Fix embed extraction
  [extractor/epoch] Add extractor (#4772)
  [extractor/stripchat] Don't modify input URL (#4781)
  [jsinterp] Add `charcodeAt` and bitwise overflow (#4706)
  [extractor/newspicks] Add extractor (#4725)
  [cookies] Support firefox container in `--cookies-from-browser` (#4753)
  [extractor/crunchyroll:beta] Use anonymous access (#4704)
  Restore LD_LIBRARY_PATH when using PyInstaller (#4666)
  [utils] Add `deprecation_warning`

README.md

@@ -775,13 +775,14 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                     and dump cookie jar in
     --no-cookies                    Do not read/dump cookies from/to file
                                     (default)
-    --cookies-from-browser BROWSER[+KEYRING][:PROFILE]
+    --cookies-from-browser BROWSER[+KEYRING][:PROFILE[:CONTAINER]]
                                     The name of the browser and (optionally) the
                                     name/path of the profile to load cookies
-                                    from, separated by a ":". Currently
-                                    supported browsers are: brave, chrome,
-                                    chromium, edge, firefox, opera, safari,
-                                    vivaldi. By default, the most recently
+                                    from (and container name if Firefox)
+                                    separated by a ":". Currently supported
+                                    browsers are: brave, chrome, chromium, edge,
+                                    firefox, opera, safari, vivaldi. By default,
+                                    the default container of the most recently
                                     accessed profile is used. The keyring used
                                     for decrypting Chromium cookies on Linux can
                                     be (optionally) specified after the browser

test/test_jsinterp.py

@@ -352,6 +352,22 @@ def test_regex(self):
         ''')
         self.assertEqual(jsi.call_function('x').flags & re.I, re.I)
 
+    def test_char_code_at(self):
+        jsi = JSInterpreter('function x(i){return "test".charCodeAt(i)}')
+        self.assertEqual(jsi.call_function('x', 0), 116)
+        self.assertEqual(jsi.call_function('x', 1), 101)
+        self.assertEqual(jsi.call_function('x', 2), 115)
+        self.assertEqual(jsi.call_function('x', 3), 116)
+        self.assertEqual(jsi.call_function('x', 4), None)
+        self.assertEqual(jsi.call_function('x', 'not_a_number'), 116)
+
+    def test_bitwise_operators_overflow(self):
+        jsi = JSInterpreter('function x(){return -524999584 << 5}')
+        self.assertEqual(jsi.call_function('x'), 379882496)
+
+        jsi = JSInterpreter('function x(){return 1236566549 << 5}')
+        self.assertEqual(jsi.call_function('x'), 915423904)
+
 
 if __name__ == '__main__':
     unittest.main()

yt_dlp/YoutubeDL.py

@@ -90,6 +90,7 @@
     args_to_str,
     bug_reports_message,
     date_from_str,
+    deprecation_warning,
     determine_ext,
     determine_protocol,
     dig_object_type,
@@ -329,8 +330,9 @@ class YoutubeDL:
                        should act on each input URL as opposed to for the entire queue
     cookiefile:        File name or text stream from where cookies should be read and dumped to
     cookiesfrombrowser:  A tuple containing the name of the browser, the profile
-                       name/path from where cookies are loaded, and the name of the
-                       keyring, e.g. ('chrome', ) or ('vivaldi', 'default', 'BASICTEXT')
+                       name/path from where cookies are loaded, the name of the keyring,
+                       and the container name, e.g. ('chrome', ) or
+                       ('vivaldi', 'default', 'BASICTEXT') or ('firefox', 'default', None, 'Meta')
     legacyserverconnect: Explicitly allow HTTPS connection to servers that do not
                        support RFC 5746 secure renegotiation
     nocheckcertificate:  Do not verify SSL certificates
@@ -686,7 +688,7 @@ def check_deprecated(param, option, suggestion):
         for msg in self.params.get('_warnings', []):
             self.report_warning(msg)
         for msg in self.params.get('_deprecation_warnings', []):
-            self.deprecation_warning(msg)
+            self.deprecated_feature(msg)
 
         self.params['compat_opts'] = set(self.params.get('compat_opts', ()))
         if 'list-formats' in self.params['compat_opts']:
@@ -890,9 +892,11 @@ def _write_string(self, message, out=None, only_once=False):
     def to_stdout(self, message, skip_eol=False, quiet=None):
         """Print message to stdout"""
         if quiet is not None:
-            self.deprecation_warning('"YoutubeDL.to_stdout" no longer accepts the argument quiet. Use "YoutubeDL.to_screen" instead')
+            self.deprecation_warning('"YoutubeDL.to_stdout" no longer accepts the argument quiet. '
+                                     'Use "YoutubeDL.to_screen" instead')
         if skip_eol is not False:
-            self.deprecation_warning('"YoutubeDL.to_stdout" no longer accepts the argument skip_eol. Use "YoutubeDL.to_screen" instead')
+            self.deprecation_warning('"YoutubeDL.to_stdout" no longer accepts the argument skip_eol. '
+                                     'Use "YoutubeDL.to_screen" instead')
         self._write_string(f'{self._bidi_workaround(message)}\n', self._out_files.out)
 
     def to_screen(self, message, skip_eol=False, quiet=None):
@@ -1031,11 +1035,14 @@ def report_warning(self, message, only_once=False):
                 return
             self.to_stderr(f'{self._format_err("WARNING:", self.Styles.WARNING)} {message}', only_once)
 
-    def deprecation_warning(self, message):
+    def deprecation_warning(self, message, *, stacklevel=0):
+        deprecation_warning(
+            message, stacklevel=stacklevel + 1, printer=self.report_error, is_error=False)
+
+    def deprecated_feature(self, message):
         if self.params.get('logger') is not None:
-            self.params['logger'].warning(f'DeprecationWarning: {message}')
-        else:
-            self.to_stderr(f'{self._format_err("DeprecationWarning:", self.Styles.ERROR)} {message}', True)
+            self.params['logger'].warning(f'Deprecated Feature: {message}')
+        self.to_stderr(f'{self._format_err("Deprecated Feature:", self.Styles.ERROR)} {message}', True)
 
     def report_error(self, message, *args, **kwargs):
         '''

yt_dlp/__init__.py

@@ -64,6 +64,8 @@
 from .longname import DEFAULT_DELIMITER
 from .YoutubeDL import YoutubeDL
 
+_IN_CLI = False
+
 
 def _exit(status=0, *args):
     for msg in args:
@@ -356,6 +358,7 @@ def parse_chapters(name, value):
 
     # Cookies from browser
     if opts.cookiesfrombrowser:
+        container = None
         mobj = re.match(r'(?P<name>[^+:]+)(\s*\+\s*(?P<keyring>[^:]+))?(\s*:(?P<profile>.+))?', opts.cookiesfrombrowser)
         if mobj is None:
             raise ValueError(f'invalid cookies from browser arguments: {opts.cookiesfrombrowser}')
@@ -364,12 +367,15 @@ def parse_chapters(name, value):
         if browser_name not in SUPPORTED_BROWSERS:
             raise ValueError(f'unsupported browser specified for cookies: "{browser_name}". '
                              f'Supported browsers are: {", ".join(sorted(SUPPORTED_BROWSERS))}')
+        elif profile and browser_name == 'firefox':
+            if ':' in profile and not os.path.exists(profile):
+                profile, container = profile.split(':', 1)
         if keyring is not None:
             keyring = keyring.upper()
             if keyring not in SUPPORTED_KEYRINGS:
                 raise ValueError(f'unsupported keyring specified for cookies: "{keyring}". '
                                  f'Supported keyrings are: {", ".join(sorted(SUPPORTED_KEYRINGS))}')
-        opts.cookiesfrombrowser = (browser_name, profile, keyring)
+        opts.cookiesfrombrowser = (browser_name, profile, keyring, container)
 
     # MetadataParser
     def metadataparser_actions(f):

yt_dlp/__main__.py

@@ -19,4 +19,5 @@
 import yt_dlp
 
 if __name__ == '__main__':
+    yt_dlp._IN_CLI = True
     yt_dlp.main()

yt_dlp/cookies.py

@@ -3,6 +3,7 @@
 import http.cookiejar
 import json
 import os
+import re
 import shutil
 import struct
 import subprocess
@@ -24,7 +25,7 @@
     sqlite3,
 )
 from .minicurses import MultilinePrinter, QuietMultilinePrinter
-from .utils import Popen, YoutubeDLCookieJar, error_to_str, expand_path
+from .utils import Popen, YoutubeDLCookieJar, error_to_str, expand_path, try_call
 
 CHROMIUM_BASED_BROWSERS = {'brave', 'chrome', 'chromium', 'edge', 'opera', 'vivaldi'}
 SUPPORTED_BROWSERS = CHROMIUM_BASED_BROWSERS | {'firefox', 'safari'}
@@ -85,8 +86,9 @@ def _create_progress_bar(logger):
 def load_cookies(cookie_file, browser_specification, ydl):
     cookie_jars = []
     if browser_specification is not None:
-        browser_name, profile, keyring = _parse_browser_specification(*browser_specification)
-        cookie_jars.append(extract_cookies_from_browser(browser_name, profile, YDLLogger(ydl), keyring=keyring))
+        browser_name, profile, keyring, container = _parse_browser_specification(*browser_specification)
+        cookie_jars.append(
+            extract_cookies_from_browser(browser_name, profile, YDLLogger(ydl), keyring=keyring, container=container))
 
     if cookie_file is not None:
         is_filename = YoutubeDLCookieJar.is_path(cookie_file)
@@ -101,9 +103,9 @@ def load_cookies(cookie_file, browser_specification, ydl):
     return _merge_cookie_jars(cookie_jars)
 
 
-def extract_cookies_from_browser(browser_name, profile=None, logger=YDLLogger(), *, keyring=None):
+def extract_cookies_from_browser(browser_name, profile=None, logger=YDLLogger(), *, keyring=None, container=None):
     if browser_name == 'firefox':
-        return _extract_firefox_cookies(profile, logger)
+        return _extract_firefox_cookies(profile, container, logger)
     elif browser_name == 'safari':
         return _extract_safari_cookies(profile, logger)
     elif browser_name in CHROMIUM_BASED_BROWSERS:
@@ -112,7 +114,7 @@ def extract_cookies_from_browser(browser_name, profile=None, logger=YDLLogger(),
         raise ValueError(f'unknown browser: {browser_name}')
 
 
-def _extract_firefox_cookies(profile, logger):
+def _extract_firefox_cookies(profile, container, logger):
     logger.info('Extracting cookies from firefox')
     if not sqlite3:
         logger.warning('Cannot extract cookies from firefox without sqlite3 support. '
@@ -126,6 +128,20 @@ def _extract_firefox_cookies(profile, logger):
     else:
         search_root = os.path.join(_firefox_browser_dir(), profile)
 
+    container_id = None
+    if container is not None:
+        containers_path = os.path.join(search_root, 'containers.json')
+        if not os.path.isfile(containers_path) or not os.access(containers_path, os.R_OK):
+            raise FileNotFoundError(f'could not read containers.json in {search_root}')
+        with open(containers_path, 'r') as containers:
+            identities = json.load(containers).get('identities', [])
+        container_id = next((context.get('userContextId') for context in identities if container in (
+            context.get('name'),
+            try_call(lambda: re.fullmatch(r'userContext([^\.]+)\.label', context['l10nID']).group())
+        )), None)
+        if not isinstance(container_id, int):
+            raise ValueError(f'could not find firefox container "{container}" in containers.json')
+
     cookie_database_path = _find_most_recently_used_file(search_root, 'cookies.sqlite', logger)
     if cookie_database_path is None:
         raise FileNotFoundError(f'could not find firefox cookies database in {search_root}')
@@ -135,7 +151,18 @@ def _extract_firefox_cookies(profile, logger):
         cursor = None
         try:
             cursor = _open_database_copy(cookie_database_path, tmpdir)
-            cursor.execute('SELECT host, name, value, path, expiry, isSecure FROM moz_cookies')
+            origin_attributes = ''
+            if isinstance(container_id, int):
+                origin_attributes = f'^userContextId={container_id}'
+                logger.debug(
+                    f'Only loading cookies from firefox container "{container}", ID {container_id}')
+            try:
+                cursor.execute(
+                    'SELECT host, name, value, path, expiry, isSecure FROM moz_cookies WHERE originAttributes=?',
+                    (origin_attributes, ))
+            except sqlite3.OperationalError:
+                logger.debug('Database exception, loading all cookies')
+                cursor.execute('SELECT host, name, value, path, expiry, isSecure FROM moz_cookies')
             jar = YoutubeDLCookieJar()
             with _create_progress_bar(logger) as progress_bar:
                 table = cursor.fetchall()
@@ -948,11 +975,11 @@ def _is_path(value):
     return os.path.sep in value
 
 
-def _parse_browser_specification(browser_name, profile=None, keyring=None):
+def _parse_browser_specification(browser_name, profile=None, keyring=None, container=None):
     if browser_name not in SUPPORTED_BROWSERS:
         raise ValueError(f'unsupported browser: "{browser_name}"')
     if keyring not in (None, *SUPPORTED_KEYRINGS):
         raise ValueError(f'unsupported keyring: "{keyring}"')
     if profile is not None and _is_path(profile):
         profile = os.path.expanduser(profile)
-    return browser_name, profile, keyring
+    return browser_name, profile, keyring, container

yt_dlp/downloader/common.py

@@ -88,6 +88,7 @@ def _set_ydl(self, ydl: 'YoutubeDL'):
 
         for func in (
             'deprecation_warning',
+            'deprecated_feature',
             'report_error',
             'report_file_already_downloaded',
             'report_warning',

yt_dlp/downloader/fragment.py

@@ -65,8 +65,8 @@ class FragmentFD(FileDownloader):
     """
 
     def report_retry_fragment(self, err, frag_index, count, retries):
-        self.deprecation_warning(
-            'yt_dlp.downloader.FragmentFD.report_retry_fragment is deprecated. Use yt_dlp.downloader.FileDownloader.report_retry instead')
+        self.deprecation_warning('yt_dlp.downloader.FragmentFD.report_retry_fragment is deprecated. '
+                                 'Use yt_dlp.downloader.FileDownloader.report_retry instead')
         return self.report_retry(err, count, retries, frag_index)
 
     def report_skip_fragment(self, frag_index, err=None):

yt_dlp/extractor/_extractors.py

@@ -475,6 +475,7 @@
     EpiconIE,
     EpiconSeriesIE,
 )
+from .epoch import EpochIE
 from .eporner import EpornerIE
 from .eroprofile import (
     EroProfileIE,
@@ -496,6 +497,7 @@
 from .esri import EsriVideoIE
 from .europa import EuropaIE
 from .europeantour import EuropeanTourIE
+from .eurosport import EurosportIE
 from .euscreen import EUScreenIE
 from .evoload import EvoLoadIE
 from .expotv import ExpoTVIE

yt_dlp/extractor/bilibili.py

@@ -218,6 +218,9 @@ def _real_extract(self, url):
 
         durl = traverse_obj(video_info, ('dash', 'video'))
         audios = traverse_obj(video_info, ('dash', 'audio')) or []
+        flac_audio = traverse_obj(video_info, ('dash', 'flac', 'audio'))
+        if flac_audio:
+            audios.append(flac_audio)
         entries = []
 
         RENDITIONS = ('qn=80&quality=80&type=', 'quality=2&type=mp4')

yt_dlp/extractor/common.py

@@ -1823,9 +1823,8 @@ def _get_field_setting(self, field, key):
             if field not in self.settings:
                 if key in ('forced', 'priority'):
                     return False
-                self.ydl.deprecation_warning(
-                    f'Using arbitrary fields ({field}) for format sorting is deprecated '
-                    'and may be removed in a future version')
+                self.ydl.deprecated_feature(f'Using arbitrary fields ({field}) for format sorting is '
+                                            'deprecated and may be removed in a future version')
                 self.settings[field] = {}
             propObj = self.settings[field]
             if key not in propObj:
@@ -1910,9 +1909,8 @@ def add_item(field, reverse, closest, limit_text):
                 if self._get_field_setting(field, 'type') == 'alias':
                     alias, field = field, self._get_field_setting(field, 'field')
                     if self._get_field_setting(alias, 'deprecated'):
-                        self.ydl.deprecation_warning(
-                            f'Format sorting alias {alias} is deprecated '
-                            f'and may be removed in a future version. Please use {field} instead')
+                        self.ydl.deprecated_feature(f'Format sorting alias {alias} is deprecated and may '
+                                                    'be removed in a future version. Please use {field} instead')
                 reverse = match.group('reverse') is not None
                 closest = match.group('separator') == '~'
                 limit_text = match.group('limit')

yt_dlp/extractor/crunchyroll.py

@@ -720,15 +720,20 @@ class CrunchyrollBetaBaseIE(CrunchyrollBaseIE):
 
     def _get_params(self, lang):
         if not CrunchyrollBetaBaseIE.params:
+            if self._get_cookies(f'https://beta.crunchyroll.com/{lang}').get('etp_rt'):
+                grant_type, key = 'etp_rt_cookie', 'accountAuthClientId'
+            else:
+                grant_type, key = 'client_id', 'anonClientId'
+
             initial_state, app_config = self._get_beta_embedded_json(self._download_webpage(
                 f'https://beta.crunchyroll.com/{lang}', None, note='Retrieving main page'), None)
             api_domain = app_config['cxApiParams']['apiDomain']
-            basic_token = str(base64.b64encode(('%s:' % app_config['cxApiParams']['accountAuthClientId']).encode('ascii')), 'ascii')
+
             auth_response = self._download_json(
-                f'{api_domain}/auth/v1/token', None, note='Authenticating with cookie',
+                f'{api_domain}/auth/v1/token', None, note=f'Authenticating with grant_type={grant_type}',
                 headers={
-                    'Authorization': 'Basic ' + basic_token
-                }, data='grant_type=etp_rt_cookie'.encode('ascii'))
+                    'Authorization': 'Basic ' + str(base64.b64encode(('%s:' % app_config['cxApiParams'][key]).encode('ascii')), 'ascii')
+                }, data=f'grant_type={grant_type}'.encode('ascii'))
             policy_response = self._download_json(
                 f'{api_domain}/index/v2', None, note='Retrieving signed policy',
                 headers={
@@ -747,21 +752,6 @@ def _get_params(self, lang):
             CrunchyrollBetaBaseIE.params = (api_domain, bucket, params)
         return CrunchyrollBetaBaseIE.params
 
-    def _redirect_from_beta(self, url, lang, internal_id, display_id, is_episode, iekey):
-        initial_state, app_config = self._get_beta_embedded_json(self._download_webpage(url, display_id), display_id)
-        content_data = initial_state['content']['byId'][internal_id]
-        if is_episode:
-            video_id = content_data['external_id'].split('.')[1]
-            series_id = content_data['episode_metadata']['series_slug_title']
-        else:
-            series_id = content_data['slug_title']
-        series_id = re.sub(r'-{2,}', '-', series_id)
-        url = f'https://www.crunchyroll.com/{lang}{series_id}'
-        if is_episode:
-            url = url + f'/{display_id}-{video_id}'
-        self.to_screen(f'{display_id}: Not logged in. Redirecting to non-beta site - {url}')
-        return self.url_result(url, iekey, display_id)
-
 
 class CrunchyrollBetaIE(CrunchyrollBetaBaseIE):
     IE_NAME = 'crunchyroll:beta'
@@ -800,10 +790,6 @@ class CrunchyrollBetaIE(CrunchyrollBetaBaseIE):
 
     def _real_extract(self, url):
         lang, internal_id, display_id = self._match_valid_url(url).group('lang', 'id', 'display_id')
-
-        if not self._get_cookies(url).get('etp_rt'):
-            return self._redirect_from_beta(url, lang, internal_id, display_id, True, CrunchyrollIE.ie_key())
-
         api_domain, bucket, params = self._get_params(lang)
 
         episode_response = self._download_json(
@@ -897,10 +883,6 @@ class CrunchyrollBetaShowIE(CrunchyrollBetaBaseIE):
 
     def _real_extract(self, url):
         lang, internal_id, display_id = self._match_valid_url(url).group('lang', 'id', 'display_id')
-
-        if not self._get_cookies(url).get('etp_rt'):
-            return self._redirect_from_beta(url, lang, internal_id, display_id, False, CrunchyrollShowPlaylistIE.ie_key())
-
         api_domain, bucket, params = self._get_params(lang)
 
         series_response = self._download_json(