From 15f9c35ab8c7182359d2dc5ac4fdc8e4adfb2e88 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Thu, 28 Oct 2021 15:55:38 +0100
Subject: [PATCH 01/20] Fix test_youtube_playlist_noplaylist

---
 test/test_youtube_lists.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py
index cf2fdf14f62..72820972e2d 100644
--- a/test/test_youtube_lists.py
+++ b/test/test_youtube_lists.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 # Allow direct execution
@@ -9,11 +10,10 @@
 
 from test.helper import FakeYDL
 
-
 from youtube_dl.extractor import (
+    YoutubeIE,
     YoutubePlaylistIE,
     YoutubeTabIE,
-    YoutubeIE,
 )
 
 
@@ -25,9 +25,11 @@ def assertIsPlaylist(self, info):
     def test_youtube_playlist_noplaylist(self):
         dl = FakeYDL()
         dl.params['noplaylist'] = True
+        dl.params['format'] = 'best'
         ie = YoutubePlaylistIE(dl)
         result = ie.extract('https://www.youtube.com/watch?v=FXxLjLQi3Fg&list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re')
         self.assertEqual(result['_type'], 'url')
+        result = dl.extract_info(result['url'], download=False, ie_key=result.get('ie_key'), process=False)
         self.assertEqual(YoutubeIE().extract_id(result['url']), 'FXxLjLQi3Fg')
 
     def test_youtube_course(self):

From ea31a0bb168c1006c454cc19e222a87871bdd8c8 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Thu, 28 Oct 2021 15:57:10 +0100
Subject: [PATCH 02/20] Remove obsolete test_youtube_course

---
 test/test_youtube_lists.py | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py
index 72820972e2d..e1636a1a6d6 100644
--- a/test/test_youtube_lists.py
+++ b/test/test_youtube_lists.py
@@ -32,16 +32,6 @@ def test_youtube_playlist_noplaylist(self):
         result = dl.extract_info(result['url'], download=False, ie_key=result.get('ie_key'), process=False)
         self.assertEqual(YoutubeIE().extract_id(result['url']), 'FXxLjLQi3Fg')
 
-    def test_youtube_course(self):
-        dl = FakeYDL()
-        ie = YoutubePlaylistIE(dl)
-        # TODO find a > 100 (paginating?) videos course
-        result = ie.extract('https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')
-        entries = list(result['entries'])
-        self.assertEqual(YoutubeIE().extract_id(entries[0]['url']), 'j9WZyLZCBzs')
-        self.assertEqual(len(entries), 25)
-        self.assertEqual(YoutubeIE().extract_id(entries[-1]['url']), 'rYefUsYuEp0')
-
     def test_youtube_mix(self):
         dl = FakeYDL()
         ie = YoutubePlaylistIE(dl)

From 050f74ff0e9cccb30e1eadba0ffef06c11cae5ba Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Fri, 29 Oct 2021 03:03:00 +0100
Subject: [PATCH 03/20] Fix test_youtube_mix

---
 test/test_youtube_lists.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py
index e1636a1a6d6..fae8a950a4b 100644
--- a/test/test_youtube_lists.py
+++ b/test/test_youtube_lists.py
@@ -34,12 +34,14 @@ def test_youtube_playlist_noplaylist(self):
 
     def test_youtube_mix(self):
         dl = FakeYDL()
-        ie = YoutubePlaylistIE(dl)
-        result = ie.extract('https://www.youtube.com/watch?v=W01L70IGBgE&index=2&list=RDOQpdSVF_k_w')
-        entries = result['entries']
+        dl.params['format'] = 'best'
+        ie = YoutubeTabIE(dl)
+        result = dl.extract_info('https://www.youtube.com/watch?v=uVJ0Il5WvbE&list=PLhQjrBD2T381k8ul4WQ8SQ165XqY149WW',
+                                 download=False, ie_key=ie.ie_key(), process=True)
+        entries = (result or {}).get('entries', [{'id': 'not_found', }])
         self.assertTrue(len(entries) >= 50)
         original_video = entries[0]
-        self.assertEqual(original_video['id'], 'OQpdSVF_k_w')
+        self.assertEqual(original_video['id'], 'uVJ0Il5WvbE')
 
     def test_youtube_toptracks(self):
         print('Skipping: The playlist page gives error 500')

From 040353942cc580337bbed09f536cbeb9c615a08d Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Fri, 29 Oct 2021 03:10:35 +0100
Subject: [PATCH 04/20] Remove obsolete non-working test_youtube_toptracks

---
 test/test_youtube_lists.py | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py
index fae8a950a4b..69c5d52ebb5 100644
--- a/test/test_youtube_lists.py
+++ b/test/test_youtube_lists.py
@@ -43,15 +43,6 @@ def test_youtube_mix(self):
         original_video = entries[0]
         self.assertEqual(original_video['id'], 'uVJ0Il5WvbE')
 
-    def test_youtube_toptracks(self):
-        print('Skipping: The playlist page gives error 500')
-        return
-        dl = FakeYDL()
-        ie = YoutubePlaylistIE(dl)
-        result = ie.extract('https://www.youtube.com/playlist?list=MCUS')
-        entries = result['entries']
-        self.assertEqual(len(entries), 100)
-
     def test_youtube_flat_playlist_extraction(self):
         dl = FakeYDL()
         dl.params['extract_flat'] = True

From ad1356f29fbdb9add8b91d77ee1b8f9861dd56c4 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Mon, 1 Nov 2021 04:44:57 +0000
Subject: [PATCH 05/20] Fix test_youtube_flat_playlist_extraction

---
 test/test_youtube_lists.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py
index 69c5d52ebb5..07a6b6d06c5 100644
--- a/test/test_youtube_lists.py
+++ b/test/test_youtube_lists.py
@@ -52,7 +52,7 @@ def test_youtube_flat_playlist_extraction(self):
         entries = list(result['entries'])
         self.assertTrue(len(entries) == 1)
         video = entries[0]
-        self.assertEqual(video['_type'], 'url_transparent')
+        self.assertEqual(video['_type'], 'url')
         self.assertEqual(video['ie_key'], 'Youtube')
         self.assertEqual(video['id'], 'BaW_jenozKc')
         self.assertEqual(video['url'], 'BaW_jenozKc')

From 20fc43475289bd3335e5f96a8e0d8f5432f1d806 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Mon, 1 Nov 2021 04:45:42 +0000
Subject: [PATCH 06/20] Unthrottle download speed by responding to "n"
 parameter challenge

---
 youtube_dl/extractor/youtube.py | 313 ++++++++++++++++++++++++++++++--
 1 file changed, 297 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index dc4bd4a77f9..b3085ccd4e7 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2,6 +2,12 @@
 
 from __future__ import unicode_literals
 
+# should probably have been in compat.py
+try:
+    from future_builtins import map
+except ImportError:
+    pass
+
 import itertools
 import json
 import os.path
@@ -1253,6 +1259,17 @@ def _extract_player_info(cls, player_url):
             raise ExtractorError('Cannot identify player %r' % player_url)
         return id_m.group('id')
 
+    def _get_player_code(self, video_id, player_url, player_id=None):
+        if not player_id:
+            player_id = self._extract_player_info(player_url)
+
+        if player_id not in self._code_cache:
+            self._code_cache[player_id] = self._download_webpage(
+                player_url, video_id,
+                note='Downloading player ' + player_id,
+                errnote='Download of %s failed' % player_url)
+        return self._code_cache[player_id]
+
     def _extract_signature_function(self, video_id, player_url, example_sig):
         player_id = self._extract_player_info(player_url)
 
@@ -1265,12 +1282,7 @@ def _extract_signature_function(self, video_id, player_url, example_sig):
         if cache_spec is not None:
             return lambda s: ''.join(s[i] for i in cache_spec)
 
-        if player_id not in self._code_cache:
-            self._code_cache[player_id] = self._download_webpage(
-                player_url, video_id,
-                note='Downloading player ' + player_id,
-                errnote='Download of %s failed' % player_url)
-        code = self._code_cache[player_id]
+        code = self._get_player_code(video_id, player_url, player_id)
         res = self._parse_sig_js(code)
 
         test_string = ''.join(map(compat_chr, range(len(example_sig))))
@@ -1349,11 +1361,6 @@ def _decrypt_signature(self, s, video_id, player_url):
         if player_url is None:
             raise ExtractorError('Cannot decrypt signature without player_url')
 
-        if player_url.startswith('//'):
-            player_url = 'https:' + player_url
-        elif not re.match(r'https?://', player_url):
-            player_url = compat_urlparse.urljoin(
-                'https://www.youtube.com', player_url)
         try:
             player_id = (player_url, self._signature_cache_id(s))
             if player_id not in self._player_cache:
@@ -1370,6 +1377,280 @@ def _decrypt_signature(self, s, video_id, player_url):
             raise ExtractorError(
                 'Signature extraction failed: ' + tb, cause=e)
 
+    def _extract_player_url(self, webpage):
+        player_url = self._search_regex(
+            r'"(?:PLAYER_JS_URL|jsUrl)"\s*:\s*"([^"]+)"',
+            webpage or '', 'player URL', fatal=False)
+        if not player_url:
+            return
+        if player_url.startswith('//'):
+            player_url = 'https:' + player_url
+        elif not re.match(r'https?://', player_url):
+            player_url = compat_urlparse.urljoin(
+                'https://www.youtube.com', player_url)
+        return player_url
+
+    # Based on an equivalent function [1] in the youtube.lua script from VLC
+    # Many thanks to @linkfanel [2]
+    # NB This code could fail if YT should revise the player code and would then have
+    # to be reworked (thankless task previously undertaken at [1])
+    # 1. https://code.videolan.org/videolan/vlc/-/blob/4fb284e5af69aa9ac2100ccbdd3b88debec9987f/share/lua/playlist/youtube.lua#L116
+    # 2. https://github.com/ytdl-org/youtube-dl/issues/30097#issuecomment-950157377
+    def _n_descramble(self, n_param, js):
+        """Compute the response to YT's "n" parameter challenge
+
+        Args:
+        n_param -- challenge string that is the value of the
+                   URL's "n" query parameter
+        js      -- text of the JS player code that includes the
+                   challenge response algorithm
+        """
+        if not js:
+            return
+
+        # helper functions (part 1)
+        def isiterable(x):
+            try:
+                return x.__getitem__ and True
+            except AttributeError:
+                return False
+
+        def find_first(pattern, string, flags=0, groups=1):
+            pattern = re.compile(pattern, flags)
+            return next(map(lambda m: m.groups() if groups is True else m.group(groups),
+                            pattern.finditer(string)),
+                        (None, ) * pattern.groups if groups is True else None)
+
+        # Look for the descrambler function's name
+        # a.D&&(b=a.get("n"))&&(b=lha(b),a.set("n",b))}};
+        descrambler = find_first(r'[=(,&|](\w+)\(\w+\),\w+\.set\("n",', js)
+        if not descrambler:
+            self.report_warning("Couldn't extract YouTube video throttling parameter descrambling function name")
+            return
+        # Fetch the code of the descrambler function
+        # lha=function(a){var b=a.split(""),c=[310282131,"KLf3",b,null,function(d,e){d.push(e)},-45817231, [data and transformations...] ,1248130556];c[3]=c;c[15]=c;c[18]=c;try{c[40](c[14],c[2]),c[25](c[48]),c[21](c[32],c[23]), [scripted calls...] ,c[25](c[33],c[3])}catch(d){return"enhanced_except_4ZMBnuz-_w8_"+a}return b.join("")};
+        code = find_first(r'(?s)%s=function\([^)]+\)\{(.+?)\};' % (descrambler, ), js)
+        if not code:
+            self.report_warning("Couldn't extract YouTube video throttling parameter descrambling code")
+            return
+        # Split code into two main sections: 1/ data and transformations,
+        # and 2/ a script of calls
+        datac, script = find_first(r'(?s)c=\[(.+)\];.+?;try\{(.+)\}catch\(', code, groups=True)
+        if not datac or not script:
+            self.report_warning("Couldn't extract YouTube video throttling parameter descrambling rules")
+            return
+        # Split "n" parameter into a table as descrambling operates on it
+        # as one of several arrays - in Python just copy it as a list
+        n = list(n_param)
+        # Helper: table_len = function() ... end - in Python just use len
+
+        # Common routine shared by the compound transformations,
+        # compounding the "n" parameter with an input string,
+        # character by character using a Base64 alphabet.
+        # d.forEach(function(l,m,n){this.push(n[m]=h[(h.indexOf(l)-h.indexOf(this[m])+m-32+f--)%h.length])},e.split(""))
+        def compound(ntab, strg, alphabet, charcode):
+            if ntab != n or type(strg) != compat_str:
+                return True
+            inp = list(strg)
+            llen = len(alphabet)
+            ntab_copy = ntab[:]
+            for i, c in enumerate(ntab_copy):
+                if type(c) != compat_str:
+                    return True
+                pos1 = alphabet.find(c)
+                pos2 = alphabet.find(inp[i])
+                if pos1 < 0 or pos2 < 0:
+                    return True
+                pos = (pos1 - pos2 + charcode - 32) % llen
+                newc = alphabet[pos]
+                ntab[i] = newc
+                inp.append(newc)
+
+        # The data section contains among others function code for a number
+        # of transformations, most of which are basic array operations.
+        # We can match these functions' code to identify them, and emulate
+        # the corresponding transformations.
+
+        # helper fns (in-place)
+        def swap(s, i, j):
+            x = s[i]
+            s[i] = s[j]
+            s[j] = x
+
+        def rotate(s, i):
+            tmp = s[:]
+            tmp[i:] = s
+            tmp[:i] = tmp[len(s):]
+            s[:] = tmp[:len(s)]
+
+        def remove(s, i):
+            del s[i]
+
+        # Compounding functions use a subfunction, so we need to be
+        # more specific in how much parsed data we consume.
+        cp_skip = r'(?s)^.*?\},e\.split\(""\)\)},\s*(.*)$'
+        def_skip = r"(?s)^.*?\},\s*(.*)$"
+
+        trans = (
+            # fn_name, fn, fn_detect_pattern, skip_re
+            ('reverse', lambda tab, _: tab.reverse(),
+                        # noqa: E127
+                        # function(d){d.reverse()}
+                        # function(d){for(var e=d.length;e;)d.push(d.splice(--e,1)[0])}
+                        r"^function\(d\)",
+                        def_skip),
+            ('append', lambda tab, val: tab.append(val),
+                       # noqa: E127
+                       # function(d,e){d.push(e)}
+                       r"^function\(d,e\){d\.push\(e\)\},",
+                       def_skip),
+            ('remove', lambda tab, i: remove(tab, i % len(tab)) if type(i) == int else True,
+                       # noqa: E127
+                       # function(d,e){e=(e%d.length+d.length)%d.length;d.splice(e,1)}
+                       r"^[^}]+?;d\.splice\(e,1\)\},",
+                       def_skip),
+            ('swap', lambda tab, i: swap(tab, 0, i % len(tab)) if type(i) == int else True,
+                     # noqa: E127
+                     # function(d,e){e=(e%d.length+d.length)%d.length;var f=d[0];d[0]=d[e];d[e]=f}
+                     # function(d,e){e=(e%d.length+d.length)%d.length;d.splice(0,1,d.splice(e,1,d[0])[0])}
+                     r"^[^}]+?;(?:var\sf=d\[0\];d\[0\]=d\[e\];d\[e\]=f|d\.splice\(0,1,d\.splice\(e,1,d\[0\]\)\[0\]\))\},",
+                     def_skip),
+            ('rotate', lambda tab, shift: rotate(tab, shift % len(tab)) if type(shift) == int else True,
+                       # noqa: E127
+                       # function(d,e){for(e=(e%d.length+d.length)%d.length;e--;)d.unshift(d.pop())}
+                       # function(d,e){e=(e%d.length+d.length)%d.length;d.splice(-e).reverse().forEach(function(f){d.unshift(f)})}
+                       r"^[^}]+?d\.unshift\((?:d\.pop\(\)|f\)\})\)},",
+                       def_skip),
+            # Compound transformations first build a variation of a
+            # Base64 alphabet, then in a common section, compound the
+            # "n" parameter with an input string, character by character.
+            ('compound1', lambda tab, s: compound(tab, s, "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-_", 96),
+                          # noqa: E127
+                          # function(d,e){for(var f=64,h=[];++f-h.length-32;)switch(f){case 58:f=96;continue;case 91:f=44;break;case 65:f=47;continue;case 46:f=153;case 123:f-=58;default:h.push(String.fromCharCode(f))} [ compound... ] }
+                          r"^[^}]+?case\s58:f=96;",
+                          cp_skip),
+            ('compound2', lambda tab, s: compound(tab, s, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_", 96),
+                          # noqa: E127
+                          # function(d,e){for(var f=64,h=[];++f-h.length-32;){switch(f){case 58:f-=14;case 91:case 92:case 93:continue;case 123:f=47;case 94:case 95:case 96:continue;case 46:f=95}h.push(String.fromCharCode(f))} [ compound... ] }
+                          # function(d,e){for(var f=64,h=[];++f-h.length-32;)switch(f){case 46:f=95;default:h.push(String.fromCharCode(f));case 94:case 95:case 96:break;case 123:f-=76;case 92:case 93:continue;case 58:f=44;case 91:} [ compound... ] }
+
+                          r"^[^}]+?case\s58:f(?:-=14|=44);",
+                          cp_skip),
+            # Fallback
+            ('unid', lambda _, __: self.report_warning("Couldn't apply unidentified YouTube video throttling parameter transformation, aborting descrambling") or True,
+                     # noqa: E127
+                     None,
+                     def_skip),
+        )
+        # The data section actually mixes input data, reference to the
+        # "n" parameter array, and self-reference to its own array, with
+        # transformation functions used to modify itself. We parse it
+        # as such into a table.
+        data = []
+        datac += ","
+        while datac:
+            # Transformation functions
+            if re.match(r"^function\(", datac):
+                name, el, _, skip = next(
+                    itertools.dropwhile(
+                        lambda x: x[2] is not None and not re.match(x[2], datac), trans))
+                datac = find_first(skip, datac)
+            # String input data
+            elif re.match(r'^"[^"]*",', datac):
+                el, datac = find_first(r'(?s)^"([^"]*)",\s*(.*)$', datac, groups=True)
+            # Integer input data
+            elif re.match(r'^-?\d+,', datac):
+                el, datac = find_first(r"(?s)^(.*?),\s*(.*)$", datac, groups=True)
+                el = int(el)
+            # Reference to "n" parameter array
+            elif re.match('^b,', datac):
+                el = n
+                datac = find_first(r"(?s)^b,\s*(.*)$", datac)
+            # Replaced by self-reference to data array after its declaration
+            elif re.match('^null,', datac):
+                el = data
+                datac = find_first(r"(?s)^null,\s*(.*)$", datac)
+            else:
+                self.report_warning("Couldn't parse unidentified YouTube video throttling parameter descrambling data"
+                                    '\nNear: "%s"' % datac[:64])
+                el = False
+                # Lua tables can't contain nil values: Python can, but still use False
+                datac = find_first(r"(?s)^[^,]*?,\s*(.*)$", datac)
+            data.append(el)
+
+        # Debugging helper to print data array elements
+        def prd(el, tab=None):
+            if not el:
+                return "???"
+            elif el == n:
+                return "n"
+            elif el == data:
+                return "data"
+            elif type(el) == compat_str:
+                return '"%s"' % (el, )
+            elif type(el) == int:
+                if isiterable(tab):
+                    return "%d -> %d" % (el, el % len(tab), )
+                return "%d" % (el, )
+            else:
+                for tr in trans:
+                    if el == tr[1]:
+                        return tr[0]
+            return repr(el)
+
+        # The script section contains a series of calls to elements of
+        # the data section array onto other elements of it: calls to
+        # transformations, with a reference to the data array itself or
+        # the "n" parameter array as first argument, and often input data
+        # as a second argument. We parse and emulate those calls to follow
+        # the descrambling script.
+        # c[40](c[14],c[2]),c[25](c[48]),c[21](c[32],c[23]), [...]
+        for ifunc, itab, iarg in map(lambda m: m.groups(),
+                                     re.finditer(r"c\[(\d+)\]\(c\[(\d+)\](?:,\s*c\[(\d+)\]|[^)]*?)\)", script)):
+            func = data[int(ifunc)]
+            tab = data[int(itab)]
+            arg = iarg and data[int(iarg)]
+            # Uncomment to debug transformation chain
+            # nprev = ''.join(n)
+            # dprev = ' '.join(map(prd, data))
+            # print(''.join(('"n" parameter transformation: ', prd(func), "(", prd(tab), (", " + prd(arg, tab)) if arg else '', ") ", ifunc, "(", itab, (", " + iarg) if iarg else "", ")")))
+            if not callable(func) or not isiterable(tab) or func(tab, arg):
+                self.report_warning("Invalid data type encountered during YouTube video throttling parameter descrambling transformation chain, aborting"
+                                    "\nCouldn't descramble YouTube throttling URL parameter: data transfer will be throttled")
+                self.report_warning("Couldn't process youtube video URL, please check for updates to this script")
+                break
+            # Uncomment to debug transformation chain
+            # nnew = ''.join(n)
+            # if nprev != nnew:
+            #     print('from: ' + nprev + "\nto:   " + nnew)
+            # dnew = ' '.join(map(prd, data))
+            # if dprev != dnew:
+            #     print('from: ' + dprev + "\nto:   " + dnew)
+        return ''.join(n)
+
+    def _unthrottle_format_urls(self, video_id, player_url, formats):
+        if not player_url:
+            return
+        player_id = self._extract_player_info(player_url)
+        code = self._get_player_code(video_id, player_url, player_id)
+        n_cache = {}
+        for fmt in formats:
+            parsed_fmt_url = compat_urlparse.urlparse(fmt['url'])
+            qs = compat_urlparse.parse_qs(parsed_fmt_url.query)
+            n_param = qs.get('n')
+            if not n_param:
+                continue
+            n_param = n_param[-1]
+            n_response = n_cache.get(n_param)
+            if not n_response:
+                n_response = self._n_descramble(n_param, code)
+                if n_response:
+                    n_cache[n_param] = n_response
+            if n_response:
+                qs['n'] = [n_response]
+                fmt['url'] = compat_urlparse.urlunparse(
+                    parsed_fmt_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
+
     def _mark_watched(self, video_id, player_response):
         playback_url = url_or_none(try_get(
             player_response,
@@ -1631,11 +1912,7 @@ def feed_entry(name):
                 if not (sc and fmt_url and encrypted_sig):
                     continue
                 if not player_url:
-                    if not webpage:
-                        continue
-                    player_url = self._search_regex(
-                        r'"(?:PLAYER_JS_URL|jsUrl)"\s*:\s*"([^"]+)"',
-                        webpage, 'player URL', fatal=False)
+                    player_url = self._extract_player_url(webpage)
                 if not player_url:
                     continue
                 signature = self._decrypt_signature(sc['s'][0], video_id, player_url)
@@ -1781,6 +2058,10 @@ def feed_entry(name):
         is_live = video_details.get('isLive')
         owner_profile_url = microformat.get('ownerProfileUrl')
 
+        if not player_url:
+            player_url = self._extract_player_url(webpage)
+        self._unthrottle_format_urls(video_id, player_url, formats)
+
         info = {
             'id': video_id,
             'title': self._live_title(video_title) if is_live else video_title,

From 9db547ddd0dd8d9afbc3df1c0644acd0eecf2310 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Sat, 13 Nov 2021 03:05:38 +0000
Subject: [PATCH 07/20] Update for player 8d287e4d

3-arg function, compound as a transformation
---
 youtube_dl/extractor/youtube.py | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index b3085ccd4e7..11845114ac0 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1390,12 +1390,13 @@ def _extract_player_url(self, webpage):
                 'https://www.youtube.com', player_url)
         return player_url
 
-    # Based on an equivalent function [1] in the youtube.lua script from VLC
-    # Many thanks to @linkfanel [2]
+    # Based on an equivalent function [2] in the youtube.lua script from VLC
+    # Many thanks to @linkfanel [3]
     # NB This code could fail if YT should revise the player code and would then have
-    # to be reworked (thankless task previously undertaken at [1])
-    # 1. https://code.videolan.org/videolan/vlc/-/blob/4fb284e5af69aa9ac2100ccbdd3b88debec9987f/share/lua/playlist/youtube.lua#L116
-    # 2. https://github.com/ytdl-org/youtube-dl/issues/30097#issuecomment-950157377
+    # to be reworked (thankless task previously undertaken at [1] and [2])
+    # 1. https://github.com/ytdl-org/youtube-dl/issues/29326#issuecomment-894619419
+    # 2. https://code.videolan.org/videolan/vlc/-/blob/4fb284e5af69aa9ac2100ccbdd3b88debec9987f/share/lua/playlist/youtube.lua#L116
+    # 3. https://github.com/ytdl-org/youtube-dl/issues/30097#issuecomment-950157377
     def _n_descramble(self, n_param, js):
         """Compute the response to YT's "n" parameter challenge
 
@@ -1521,6 +1522,11 @@ def remove(s, i):
                        # function(d,e){e=(e%d.length+d.length)%d.length;d.splice(-e).reverse().forEach(function(f){d.unshift(f)})}
                        r"^[^}]+?d\.unshift\((?:d\.pop\(\)|f\)\})\)},",
                        def_skip),
+            ('compound', lambda tab, s: compound(tab, s, "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-_", 96),
+                         # noqa: E127
+                         # function(d,e,f){var h=f.length;d.forEach(function(l,m,n){this.push(n[m]=f[(f.indexOf(l)-f.indexOf(this[m])+m+h--)%f.length])},e.split(""))}
+                         r"^function\(\w,\w,\w\)",
+                         cp_skip),
             # Compound transformations first build a variation of a
             # Base64 alphabet, then in a common section, compound the
             # "n" parameter with an input string, character by character.
@@ -1533,7 +1539,6 @@ def remove(s, i):
                           # noqa: E127
                           # function(d,e){for(var f=64,h=[];++f-h.length-32;){switch(f){case 58:f-=14;case 91:case 92:case 93:continue;case 123:f=47;case 94:case 95:case 96:continue;case 46:f=95}h.push(String.fromCharCode(f))} [ compound... ] }
                           # function(d,e){for(var f=64,h=[];++f-h.length-32;)switch(f){case 46:f=95;default:h.push(String.fromCharCode(f));case 94:case 95:case 96:break;case 123:f-=76;case 92:case 93:continue;case 58:f=44;case 91:} [ compound... ] }
-
                           r"^[^}]+?case\s58:f(?:-=14|=44);",
                           cp_skip),
             # Fallback
@@ -1606,7 +1611,8 @@ def prd(el, tab=None):
         # the descrambling script.
         # c[40](c[14],c[2]),c[25](c[48]),c[21](c[32],c[23]), [...]
         for ifunc, itab, iarg in map(lambda m: m.groups(),
-                                     re.finditer(r"c\[(\d+)\]\(c\[(\d+)\](?:,\s*c\[(\d+)\]|[^)]*?)\)", script)):
+                                     # c[m](c[mm]{,c[mmm]{,c[mmmm]}})
+                                     re.finditer(r"c\[(\d+)\]\(c\[(\d+)\](?:,\s*c\[(\d+)\])?(?:,\s*c\[\d+\]\(\))?[^)]*?\)", script)):
             func = data[int(ifunc)]
             tab = data[int(itab)]
             arg = iarg and data[int(iarg)]

From 58bfb65a62593ac7b3fae38ffd2040d5c5e2da04 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 16 Nov 2021 05:49:44 +0000
Subject: [PATCH 08/20] Update for player 2dfe380c

Ad hoc update of c[] array after parsing core descrambler
Use specified b64 alphabet with compound(,,alphabet,), not guess
---
 youtube_dl/extractor/youtube.py | 51 +++++++++++++++++++++++++++------
 1 file changed, 43 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 11845114ac0..c5e3a300fe9 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1436,10 +1436,18 @@ def find_first(pattern, string, flags=0, groups=1):
             return
         # Split code into two main sections: 1/ data and transformations,
         # and 2/ a script of calls
-        datac, script = find_first(r'(?s)c=\[(.+)\];.+?;try\{(.+)\}catch\(', code, groups=True)
+        datac, xc, script = find_first(r'(?s)c=\[(.+)\];(.+?);try\{(.+)\}catch\(', code, groups=True)
         if not datac or not script:
             self.report_warning("Couldn't extract YouTube video throttling parameter descrambling rules")
             return
+        if xc:
+            # ad hoc update of c[] array after declaration (player 2dfe380c, ...)
+            xc = [int(x) for x in re.findall(r'c\[(\d+)\]=c\b', xc)]
+            if not xc:
+                self.report_warning("Couldn't extract YouTube video throttling parameter additional rules")
+        else:
+            xc = []
+
         # Split "n" parameter into a table as descrambling operates on it
         # as one of several arrays - in Python just copy it as a list
         n = list(n_param)
@@ -1487,6 +1495,11 @@ def rotate(s, i):
         def remove(s, i):
             del s[i]
 
+        b64_alphabets = [
+            "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-_",
+            "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_",
+        ]
+
         # Compounding functions use a subfunction, so we need to be
         # more specific in how much parsed data we consume.
         cp_skip = r'(?s)^.*?\},e\.split\(""\)\)},\s*(.*)$'
@@ -1522,20 +1535,31 @@ def remove(s, i):
                        # function(d,e){e=(e%d.length+d.length)%d.length;d.splice(-e).reverse().forEach(function(f){d.unshift(f)})}
                        r"^[^}]+?d\.unshift\((?:d\.pop\(\)|f\)\})\)},",
                        def_skip),
-            ('compound', lambda tab, s: compound(tab, s, "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-_", 96),
+            ('alphabet1', lambda: b64_alphabets[0],
+                          # noqa: E127
+                          # function(){for(var d=64,e=[];++d-e.length-32;){switch(d){case 91:d=44;continue;case 123:d=65;break;case 65:d-=18;continue;case 58:d=96;continue;case 46:d=95}e.push(String.fromCharCode(d))}return e}
+                          r"^[^}]+?case\s58:d=96;.+?\}return",
+                          def_skip),
+            ('alphabet2', lambda: b64_alphabets[1],
+                          # noqa: E127
+                          # function(){for(var d=64,e=[];++d-e.length-32;)switch(d){case 46:d=95;default:e.push(String.fromCharCode(d));case 94:case 95:case 96:break;case 123:d-=76;case 92:case 93:continue;case 58:d=44;case 91:}return e}
+                          r"^[^}]+?case\s58:d=44;[^}]+\}return",
+                          def_skip),
+            ('compound', lambda tab, s, alphabet: compound(tab, s, alphabet, 96),
                          # noqa: E127
                          # function(d,e,f){var h=f.length;d.forEach(function(l,m,n){this.push(n[m]=f[(f.indexOf(l)-f.indexOf(this[m])+m+h--)%f.length])},e.split(""))}
                          r"^function\(\w,\w,\w\)",
                          cp_skip),
+
             # Compound transformations first build a variation of a
             # Base64 alphabet, then in a common section, compound the
             # "n" parameter with an input string, character by character.
-            ('compound1', lambda tab, s: compound(tab, s, "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-_", 96),
+            ('compound1', lambda tab, s: compound(tab, s, b64_alphabets[0], 96),
                           # noqa: E127
                           # function(d,e){for(var f=64,h=[];++f-h.length-32;)switch(f){case 58:f=96;continue;case 91:f=44;break;case 65:f=47;continue;case 46:f=153;case 123:f-=58;default:h.push(String.fromCharCode(f))} [ compound... ] }
                           r"^[^}]+?case\s58:f=96;",
                           cp_skip),
-            ('compound2', lambda tab, s: compound(tab, s, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_", 96),
+            ('compound2', lambda tab, s: compound(tab, s, b64_alphabets[1], 96),
                           # noqa: E127
                           # function(d,e){for(var f=64,h=[];++f-h.length-32;){switch(f){case 58:f-=14;case 91:case 92:case 93:continue;case 123:f=47;case 94:case 95:case 96:continue;case 46:f=95}h.push(String.fromCharCode(f))} [ compound... ] }
                           # function(d,e){for(var f=64,h=[];++f-h.length-32;)switch(f){case 46:f=95;default:h.push(String.fromCharCode(f));case 94:case 95:case 96:break;case 123:f-=76;case 92:case 93:continue;case 58:f=44;case 91:} [ compound... ] }
@@ -1583,6 +1607,10 @@ def remove(s, i):
                 datac = find_first(r"(?s)^[^,]*?,\s*(.*)$", datac)
             data.append(el)
 
+        # Additional rules
+        for idx in xc:
+            data[idx] = data
+
         # Debugging helper to print data array elements
         def prd(el, tab=None):
             if not el:
@@ -1610,12 +1638,19 @@ def prd(el, tab=None):
         # as a second argument. We parse and emulate those calls to follow
         # the descrambling script.
         # c[40](c[14],c[2]),c[25](c[48]),c[21](c[32],c[23]), [...]
-        for ifunc, itab, iarg in map(lambda m: m.groups(),
-                                     # c[m](c[mm]{,c[mmm]{,c[mmmm]}})
-                                     re.finditer(r"c\[(\d+)\]\(c\[(\d+)\](?:,\s*c\[(\d+)\])?(?:,\s*c\[\d+\]\(\))?[^)]*?\)", script)):
-            func = data[int(ifunc)]
+        for ifunc, itab, iarg, iextra in map(lambda m: m.groups(),
+                                             # c[m](c[mm]{,c[mmm]{,c[mmmm]}})
+                                             re.finditer(
+                                                 r"c\[(\d+)\]\(c\[(\d+)\](?:,\s*c\[(\d+)\])?(?:,\s*c\[(\d+)\]\(\))?[^)]*?\)",
+                                                 script)):
             tab = data[int(itab)]
             arg = iarg and data[int(iarg)]
+            if iextra:
+                alphabet = data[int(iextra)]()
+                func = lambda t, s: data[int(ifunc)](t, s, alphabet)
+            else:
+                func = data[int(ifunc)]
+
             # Uncomment to debug transformation chain
             # nprev = ''.join(n)
             # dprev = ' '.join(map(prd, data))

From 622b87d0fb10a50283b12ecd5304e66dd396809b Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 18 Nov 2021 01:42:07 +0000
Subject: [PATCH 09/20] Update for player 68e11abe

---
 youtube_dl/extractor/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index c5e3a300fe9..da73e130c2f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1543,7 +1543,8 @@ def remove(s, i):
             ('alphabet2', lambda: b64_alphabets[1],
                           # noqa: E127
                           # function(){for(var d=64,e=[];++d-e.length-32;)switch(d){case 46:d=95;default:e.push(String.fromCharCode(d));case 94:case 95:case 96:break;case 123:d-=76;case 92:case 93:continue;case 58:d=44;case 91:}return e}
-                          r"^[^}]+?case\s58:d=44;[^}]+\}return",
+                          # function(){for(var d=64,e=[];++d-e.length-32;){switch(d){case 58:d-=14;case 91:case 92:case 93:continue;case 123:d=47;case 94:case 95:case 96:continue;case 46:d=95}e.push(String.fromCharCode(d))}return e}
+                          r"^[^}]+?case\s58:d(?:=44;|-=14;[^}]+\})[^}]+\}return",
                           def_skip),
             ('compound', lambda tab, s, alphabet: compound(tab, s, alphabet, 96),
                          # noqa: E127

From d69983a8756fa75e16606bcd1b2ba50ec115ed16 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 23 Nov 2021 18:53:35 +0000
Subject: [PATCH 10/20] Update for player f1ca6900

Numbers can be float literals, eg '1E3'
---
 youtube_dl/extractor/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index da73e130c2f..e61157bfafc 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1589,9 +1589,9 @@ def remove(s, i):
             elif re.match(r'^"[^"]*",', datac):
                 el, datac = find_first(r'(?s)^"([^"]*)",\s*(.*)$', datac, groups=True)
             # Integer input data
-            elif re.match(r'^-?\d+,', datac):
+            elif re.match(r'^-?\d+(?:E\d+)?,', datac):
                 el, datac = find_first(r"(?s)^(.*?),\s*(.*)$", datac, groups=True)
-                el = int(el)
+                el = int(float(el) if 'E' in el else el)
             # Reference to "n" parameter array
             elif re.match('^b,', datac):
                 el = n

From 3b9b1cd4314f9d44630a2a65abfff8abab454887 Mon Sep 17 00:00:00 2001
From: lanegramling <lanegramling@gmail.com>
Date: Tue, 14 Dec 2021 21:04:03 -0700
Subject: [PATCH 11/20] [youtube] Fix function signature parser (refs
 ytdl-org/#30363)

---
 youtube_dl/extractor/youtube.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e61157bfafc..de73fecc857 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1335,10 +1335,10 @@ def _parse_sig_js(self, jscode):
         funcname = self._search_regex(
             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
-             r'\bm=(?P<sig>[a-zA-Z0-9$]{2})\(decodeURIComponent\(h\.s\)\)',
-             r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2})\(decodeURIComponent\(c\)\)',
-             r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
-             r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
+             r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
+             r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
+             r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
+             r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
              # Obsolete patterns
              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',

From 32f049b37a08cb092190f63a0276155ca54a5a44 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Mon, 1 Nov 2021 13:34:29 +0000
Subject: [PATCH 12/20] Add compat_map/filter and use the former

---
 youtube_dl/compat.py            | 21 +++++++++++++++++++++
 youtube_dl/extractor/youtube.py |  7 +------
 2 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 9e45c454b26..29e0d3a0292 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2962,6 +2962,25 @@ def unpack(self, string):
         compat_Struct = struct.Struct
 
 
+# compat_map/filter() returning an iterator, supposedly the
+# same versioning as for zip below
+try:
+    from future_builtins import map as compat_map
+except ImportError:
+    try:
+        from itertools import imap as compat_map
+    except ImportError:
+        compat_map = map
+
+try:
+    from future_builtins import filter as compat_filter
+except ImportError:
+    try:
+        from itertools import ifilter as compat_filter
+    except ImportError:
+        compat_filter = filter
+
+
 try:
     from future_builtins import zip as compat_zip
 except ImportError:  # not 2.6+ or is 3.x
@@ -3015,6 +3034,7 @@ def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
     'compat_etree_fromstring',
     'compat_etree_register_namespace',
     'compat_expanduser',
+    'compat_filter',
     'compat_get_terminal_size',
     'compat_getenv',
     'compat_getpass',
@@ -3026,6 +3046,7 @@ def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
     'compat_integer_types',
     'compat_itertools_count',
     'compat_kwargs',
+    'compat_map',
     'compat_numeric_types',
     'compat_ord',
     'compat_os_name',
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index de73fecc857..6f3162decd6 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2,12 +2,6 @@
 
 from __future__ import unicode_literals
 
-# should probably have been in compat.py
-try:
-    from future_builtins import map
-except ImportError:
-    pass
-
 import itertools
 import json
 import os.path
@@ -19,6 +13,7 @@
 from ..compat import (
     compat_chr,
     compat_HTTPError,
+    compat_map as map,
     compat_parse_qs,
     compat_str,
     compat_urllib_parse_unquote_plus,

From 3b24b4a599408d62fc20240cdc6fac5b9d612071 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Dec 2021 01:43:16 +0700
Subject: [PATCH 13/20] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 680fffdf8bc..e530e6aead2 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,28 @@
+version <unreleased>
+
+Core
+* [postprocessor/ffmpeg] Show ffmpeg output on error (#22680, #29336)
+
+Extractors
+* [youtube] Update signature function patterns (#30363, #30366)
+* [peertube] Only call description endpoint if necessary (#29383)
+* [periscope] Pass referer to HLS requests (#29419)
+- [liveleak] Remove extractor (#17625, #24222, #29331)
++ [pornhub] Add support for pornhubthbh7ap3u.onion
+* [pornhub] Detect geo restriction
+* [pornhub] Dismiss tbr extracted from download URLs (#28927)
+* [curiositystream:collection] Extend _VALID_URL (#26326, #29117)
+* [youtube] Make get_video_info processing more robust (#29333)
+* [youtube] Workaround for get_video_info request (#29333)
+* [bilibili] Strip uploader name (#29202)
+* [youtube] Update invidious instance list (#29281)
+* [umg:de] Update GraphQL API URL (#29304)
+* [nrk] Switch psapi URL to https (#29344)
++ [egghead] Add support for app.egghead.io (#28404, #29303)
+* [appleconnect] Fix extraction (#29208)
++ [orf:tvthek] Add support for MPD formats (#28672, #29236)
+
+
 version 2021.06.06
 
 Extractors

From 830fb46973fe55c2f7b5ed315d56609f206c10cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Dec 2021 01:49:07 +0700
Subject: [PATCH 14/20] release 2021.12.17

---
 .github/ISSUE_TEMPLATE/1_broken_site.md          | 6 +++---
 .github/ISSUE_TEMPLATE/2_site_support_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/3_site_feature_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/4_bug_report.md           | 6 +++---
 .github/ISSUE_TEMPLATE/5_feature_request.md      | 4 ++--
 ChangeLog                                        | 2 +-
 docs/supportedsites.md                           | 2 --
 youtube_dl/version.py                            | 2 +-
 8 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md
index 4eb50523189..e5405c23590 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.md
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.06.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.12.17. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -26,7 +26,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support
-- [ ] I've verified that I'm running youtube-dl version **2021.06.06**
+- [ ] I've verified that I'm running youtube-dl version **2021.12.17**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar issues including closed ones
@@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.06.06
+ [debug] youtube-dl version 2021.12.17
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md
index 9fed0b489e8..33b01ce7fda 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.md
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md
@@ -19,7 +19,7 @@ labels: 'site-support-request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.06.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.12.17. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
 - Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a new site support request
-- [ ] I've verified that I'm running youtube-dl version **2021.06.06**
+- [ ] I've verified that I'm running youtube-dl version **2021.12.17**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that none of provided URLs violate any copyrights
 - [ ] I've searched the bugtracker for similar site support requests including closed ones
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
index 573e8ded0ab..285610cc70e 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
@@ -18,13 +18,13 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.06.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.12.17. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a site feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.06.06**
+- [ ] I've verified that I'm running youtube-dl version **2021.12.17**
 - [ ] I've searched the bugtracker for similar site feature requests including closed ones
 
 
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md
index c0031bf7a32..af73525fbc9 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.md
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.06.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.12.17. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support issue
-- [ ] I've verified that I'm running youtube-dl version **2021.06.06**
+- [ ] I've verified that I'm running youtube-dl version **2021.12.17**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar bug reports including closed ones
@@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.06.06
+ [debug] youtube-dl version 2021.12.17
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md
index 1138ab2ca7e..42c878b83af 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.md
@@ -19,13 +19,13 @@ labels: 'request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.06.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.12.17. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.06.06**
+- [ ] I've verified that I'm running youtube-dl version **2021.12.17**
 - [ ] I've searched the bugtracker for similar feature requests including closed ones
 
 
diff --git a/ChangeLog b/ChangeLog
index e530e6aead2..6588642825b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2021.12.17
 
 Core
 * [postprocessor/ffmpeg] Show ffmpeg output on error (#22680, #29336)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index ed0d5e9d990..ae2a6b8b02b 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -472,8 +472,6 @@
  - **LinuxAcademy**
  - **LiTV**
  - **LiveJournal**
- - **LiveLeak**
- - **LiveLeakEmbed**
  - **livestream**
  - **livestream:original**
  - **LnkGo**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 461dd87cafe..b82fbc702ff 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.06.06'
+__version__ = '2021.12.17'

From d577b929aa83b9a0717f5c19c45dfaf1e12454d8 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Tue, 2 Nov 2021 11:18:39 +0000
Subject: [PATCH 15/20] Back-port JS interpreter upgrade from yt-dlp PR #1437

---
 test/test_jsinterp.py  |  51 +++++
 youtube_dl/compat.py   |   5 +
 youtube_dl/jsinterp.py | 504 ++++++++++++++++++++++++++++++++---------
 3 files changed, 453 insertions(+), 107 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index c24b8ca742a..4d05ea61013 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -112,6 +112,57 @@ def test_call(self):
         ''')
         self.assertEqual(jsi.call_function('z'), 5)
 
+    def test_for_loop(self):
+        # function x() { a=0; for (i=0; i-10; i++) {a++} a }
+        jsi = JSInterpreter('''
+        function x() { a=0; for (i=0; i-10; i = i + 1) {a++} a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 10)
+
+    def test_switch(self):
+        jsi = JSInterpreter('''
+        function x(f) { switch(f){
+            case 1:f+=1;
+            case 2:f+=2;
+            case 3:f+=3;break;
+            case 4:f+=4;
+            default:f=0;
+        } return f }
+        ''')
+        self.assertEqual(jsi.call_function('x', 1), 7)
+        self.assertEqual(jsi.call_function('x', 3), 6)
+        self.assertEqual(jsi.call_function('x', 5), 0)
+
+    def test_try(self):
+        jsi = JSInterpreter('''
+        function x() { try{return 10} catch(e){return 5} }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 10)
+
+    def test_for_loop_continue(self):
+        jsi = JSInterpreter('''
+        function x() { a=0; for (i=0; i-10; i++) { continue; a++ } a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 0)
+
+    def test_for_loop_break(self):
+        jsi = JSInterpreter('''
+        function x() { a=0; for (i=0; i-10; i++) { break; a++ } a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 0)
+
+    def test_literal_list(self):
+        jsi = JSInterpreter('''
+        function x() { [1, 2, "asdf", [5, 6, 7]][3] }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [5, 6, 7])
+
+    def test_comma(self):
+        jsi = JSInterpreter('''
+        function x() { a=5; a -= 1, a+=3; return a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 7)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 29e0d3a0292..2004a405a8e 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -21,6 +21,10 @@
 import sys
 import xml.etree.ElementTree
 
+try:
+    import collections.abc as compat_collections_abc
+except ImportError:
+    import collections as compat_collections_abc
 
 try:
     import urllib.request as compat_urllib_request
@@ -3025,6 +3029,7 @@ def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
     'compat_b64decode',
     'compat_basestring',
     'compat_chr',
+    'compat_collections_abc',
     'compat_cookiejar',
     'compat_cookiejar_Cookie',
     'compat_cookies',
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 7bda596102a..061e92c2a4a 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -8,6 +8,15 @@
     ExtractorError,
     remove_quotes,
 )
+from .compat import (
+    compat_collections_abc
+)
+MutableMapping = compat_collections_abc.MutableMapping
+
+
+class Nonlocal:
+    pass
+
 
 _OPERATORS = [
     ('|', operator.or_),
@@ -22,11 +31,55 @@
     ('*', operator.mul),
 ]
 _ASSIGN_OPERATORS = [(op + '=', opfunc) for op, opfunc in _OPERATORS]
-_ASSIGN_OPERATORS.append(('=', lambda cur, right: right))
+_ASSIGN_OPERATORS.append(('=', (lambda cur, right: right)))
 
 _NAME_RE = r'[a-zA-Z_$][a-zA-Z_$0-9]*'
 
 
+class JS_Break(ExtractorError):
+    def __init__(self):
+        ExtractorError.__init__(self, 'Invalid break')
+
+
+class JS_Continue(ExtractorError):
+    def __init__(self):
+        ExtractorError.__init__(self, 'Invalid continue')
+
+
+class LocalNameSpace(MutableMapping):
+    def __init__(self, *stack):
+        self.stack = tuple(stack)
+
+    def __getitem__(self, key):
+        for scope in self.stack:
+            if key in scope:
+                return scope[key]
+        raise KeyError(key)
+
+    def __setitem__(self, key, value):
+        for scope in self.stack:
+            if key in scope:
+                scope[key] = value
+                break
+        else:
+            self.stack[0][key] = value
+        return value
+
+    def __delitem__(self, key):
+        raise NotImplementedError('Deleting is not supported')
+
+    def __iter__(self):
+        for scope in self.stack:
+            for scope_item in iter(scope):
+                yield scope_item
+
+    def __len__(self, key):
+        return len(iter(self))
+
+    def __repr__(self):
+        return 'LocalNameSpace%s' % (self.stack, )
+
+
 class JSInterpreter(object):
     def __init__(self, code, objects=None):
         if objects is None:
@@ -34,11 +87,58 @@ def __init__(self, code, objects=None):
         self.code = code
         self._functions = {}
         self._objects = objects
+        self.__named_object_counter = 0
+
+    def _named_object(self, namespace, obj):
+        self.__named_object_counter += 1
+        name = '__youtube_dl_jsinterp_obj%s' % (self.__named_object_counter, )
+        namespace[name] = obj
+        return name
+
+    @staticmethod
+    def _separate(expr, delim=',', max_split=None):
+        if not expr:
+            return
+        parens = {'(': 0, '{': 0, '[': 0, ']': 0, '}': 0, ')': 0}
+        start, splits, pos, max_pos = 0, 0, 0, len(delim) - 1
+        for idx, char in enumerate(expr):
+            if char in parens:
+                parens[char] += 1
+            is_in_parens = (parens['['] - parens[']']
+                            or parens['('] - parens[')']
+                            or parens['{'] - parens['}'])
+            if char == delim[pos] and not is_in_parens:
+                if pos == max_pos:
+                    pos = 0
+                    yield expr[start: idx - max_pos]
+                    start = idx + 1
+                    splits += 1
+                    if max_split and splits >= max_split:
+                        break
+                else:
+                    pos += 1
+            else:
+                pos = 0
+        yield expr[start:]
+
+    @staticmethod
+    def _separate_at_paren(expr, delim):
+        separated = list(JSInterpreter._separate(expr, delim, 1))
+        if len(separated) < 2:
+            raise ExtractorError('No terminating paren {0} in {1}'.format(delim, expr))
+        return separated[0][1:].strip(), separated[1].strip()
 
     def interpret_statement(self, stmt, local_vars, allow_recursion=100):
         if allow_recursion < 0:
             raise ExtractorError('Recursion limit reached')
 
+        sub_statements = list(self._separate(stmt, ';'))
+        stmt = (sub_statements or ['']).pop()
+        for sub_stmt in sub_statements:
+            ret, should_abort = self.interpret_statement(sub_stmt, local_vars, allow_recursion - 1)
+            if should_abort:
+                return ret
+
         should_abort = False
         stmt = stmt.lstrip()
         stmt_m = re.match(r'var\s', stmt)
@@ -61,25 +161,119 @@ def interpret_expression(self, expr, local_vars, allow_recursion):
         if expr == '':  # Empty expression
             return None
 
+        if expr.startswith('{'):
+            inner, outer = self._separate_at_paren(expr, '}')
+            inner, should_abort = self.interpret_statement(inner, local_vars, allow_recursion - 1)
+            if not outer or should_abort:
+                return inner
+            else:
+                expr = json.dumps(inner) + outer
+
         if expr.startswith('('):
-            parens_count = 0
-            for m in re.finditer(r'[()]', expr):
-                if m.group(0) == '(':
-                    parens_count += 1
+            inner, outer = self._separate_at_paren(expr, ')')
+            inner = self.interpret_expression(inner, local_vars, allow_recursion)
+            if not outer:
+                return inner
+            else:
+                expr = json.dumps(inner) + outer
+
+        if expr.startswith('['):
+            inner, outer = self._separate_at_paren(expr, ']')
+            name = self._named_object(local_vars, [
+                self.interpret_expression(item, local_vars, allow_recursion)
+                for item in self._separate(inner)])
+            expr = name + outer
+
+        m = re.match(r'try\s*', expr)
+        if m:
+            if expr[m.end()] == '{':
+                try_expr, expr = self._separate_at_paren(expr[m.end():], '}')
+            else:
+                try_expr, expr = expr[m.end() - 1:], ''
+            ret, should_abort = self.interpret_statement(try_expr, local_vars, allow_recursion - 1)
+            if should_abort:
+                return ret
+            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
+
+        m = re.match(r'(?:(?P<catch>catch)|(?P<for>for)|(?P<switch>switch))\s*\(', expr)
+        md = m.groupdict() if m else {}
+        if md.get('catch'):
+            # We ignore the catch block
+            _, expr = self._separate_at_paren(expr, '}')
+            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
+
+        elif md.get('for'):
+            def raise_constructor_error(c):
+                raise ExtractorError(
+                    'Premature return in the initialization of a for loop in {0!r}'.format(c))
+
+            constructor, remaining = self._separate_at_paren(expr[m.end() - 1:], ')')
+            if remaining.startswith('{'):
+                body, expr = self._separate_at_paren(remaining, '}')
+            else:
+                m = re.match(r'switch\s*\(', remaining)  # FIXME
+                if m:
+                    switch_val, remaining = self._separate_at_paren(remaining[m.end() - 1:], ')')
+                    body, expr = self._separate_at_paren(remaining, '}')
+                    body = 'switch(%s){%s}' % (switch_val, body)
                 else:
-                    parens_count -= 1
-                    if parens_count == 0:
-                        sub_expr = expr[1:m.start()]
-                        sub_result = self.interpret_expression(
-                            sub_expr, local_vars, allow_recursion)
-                        remaining_expr = expr[m.end():].strip()
-                        if not remaining_expr:
-                            return sub_result
-                        else:
-                            expr = json.dumps(sub_result) + remaining_expr
+                    body, expr = remaining, ''
+            start, cndn, increment = self._separate(constructor, ';')
+            if self.interpret_statement(start, local_vars, allow_recursion - 1)[1]:
+                raise_constructor_error(constructor)
+            while True:
+                if not self.interpret_expression(cndn, local_vars, allow_recursion):
+                    break
+                try:
+                    ret, should_abort = self.interpret_statement(body, local_vars, allow_recursion - 1)
+                    if should_abort:
+                        return ret
+                except JS_Break:
+                    break
+                except JS_Continue:
+                    pass
+                if self.interpret_statement(increment, local_vars, allow_recursion - 1)[1]:
+                    raise_constructor_error(constructor)
+            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
+
+        elif md.get('switch'):
+            switch_val, remaining = self._separate_at_paren(expr[m.end() - 1:], ')')
+            switch_val = self.interpret_expression(switch_val, local_vars, allow_recursion)
+            body, expr = self._separate_at_paren(remaining, '}')
+            body, default = body.split('default:') if 'default:' in body else (body, None)
+            items = body.split('case ')[1:]
+            if default:
+                items.append('default:%s' % (default, ))
+            matched = False
+            for item in items:
+                case, stmt = [i.strip() for i in self._separate(item, ':', 1)]
+                matched = matched or case == 'default' or switch_val == self.interpret_expression(case, local_vars, allow_recursion)
+                if matched:
+                    try:
+                        ret, should_abort = self.interpret_statement(stmt, local_vars, allow_recursion - 1)
+                        if should_abort:
+                            return ret
+                    except JS_Break:
                         break
-            else:
-                raise ExtractorError('Premature end of parens in %r' % expr)
+            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
+
+        # Comma separated statements
+        sub_expressions = list(self._separate(expr))
+        expr = sub_expressions.pop().strip() if sub_expressions else ''
+        for sub_expr in sub_expressions:
+            self.interpret_expression(sub_expr, local_vars, allow_recursion)
+
+        for m in re.finditer(r'''(?x)
+                (?P<pre_sign>\+\+|--)(?P<var1>%(_NAME_RE)s)|
+                (?P<var2>%(_NAME_RE)s)(?P<post_sign>\+\+|--)''' % globals(), expr):
+            var = m.group('var1') or m.group('var2')
+            start, end = m.span()
+            sign = m.group('pre_sign') or m.group('post_sign')
+            ret = local_vars[var]
+            local_vars[var] += 1 if sign[0] == '+' else -1
+            if m.group('pre_sign'):
+                ret = local_vars[var]
+            expr = expr[:start] + json.dumps(ret) + expr[end:]
 
         for op, opfunc in _ASSIGN_OPERATORS:
             m = re.match(r'''(?x)
@@ -88,14 +282,13 @@ def interpret_expression(self, expr, local_vars, allow_recursion):
                 (?P<expr>.*)$''' % (_NAME_RE, re.escape(op)), expr)
             if not m:
                 continue
-            right_val = self.interpret_expression(
-                m.group('expr'), local_vars, allow_recursion - 1)
+            right_val = self.interpret_expression(m.group('expr'), local_vars, allow_recursion)
 
             if m.groupdict().get('index'):
                 lvar = local_vars[m.group('out')]
-                idx = self.interpret_expression(
-                    m.group('index'), local_vars, allow_recursion)
-                assert isinstance(idx, int)
+                idx = self.interpret_expression(m.group('index'), local_vars, allow_recursion)
+                if not isinstance(idx, int):
+                    raise ExtractorError('List indices must be integers: %s' % (idx, ))
                 cur = lvar[idx]
                 val = opfunc(cur, right_val)
                 lvar[idx] = val
@@ -109,8 +302,13 @@ def interpret_expression(self, expr, local_vars, allow_recursion):
         if expr.isdigit():
             return int(expr)
 
+        if expr == 'break':
+            raise JS_Break()
+        elif expr == 'continue':
+            raise JS_Continue()
+
         var_m = re.match(
-            r'(?!if|return|true|false)(?P<name>%s)$' % _NAME_RE,
+            r'(?!if|return|true|false|null)(?P<name>%s)$' % _NAME_RE,
             expr)
         if var_m:
             return local_vars[var_m.group('name')]
@@ -124,91 +322,161 @@ def interpret_expression(self, expr, local_vars, allow_recursion):
             r'(?P<in>%s)\[(?P<idx>.+)\]$' % _NAME_RE, expr)
         if m:
             val = local_vars[m.group('in')]
-            idx = self.interpret_expression(
-                m.group('idx'), local_vars, allow_recursion - 1)
+            idx = self.interpret_expression(m.group('idx'), local_vars, allow_recursion)
             return val[idx]
 
+        def raise_expr_error(where, op, exp):
+            raise ExtractorError('Premature {0} return of {1} in {2!r}'.format(where, op, exp))
+
+        for op, opfunc in _OPERATORS:
+            separated = list(self._separate(expr, op))
+            if len(separated) < 2:
+                continue
+            right_val = separated.pop()
+            left_val = op.join(separated)
+            left_val, should_abort = self.interpret_statement(
+                left_val, local_vars, allow_recursion - 1)
+            if should_abort:
+                raise_expr_error('left-side', op, expr)
+            right_val, should_abort = self.interpret_statement(
+                right_val, local_vars, allow_recursion - 1)
+            if should_abort:
+                raise_expr_error('right-side', op, expr)
+            return opfunc(left_val or 0, right_val)
+
         m = re.match(
-            r'(?P<var>%s)(?:\.(?P<member>[^(]+)|\[(?P<member2>[^]]+)\])\s*(?:\(+(?P<args>[^()]*)\))?$' % _NAME_RE,
+            r'(?P<var>%s)(?:\.(?P<member>[^(]+)|\[(?P<member2>[^]]+)\])\s*' % _NAME_RE,
             expr)
         if m:
             variable = m.group('var')
-            member = remove_quotes(m.group('member') or m.group('member2'))
-            arg_str = m.group('args')
+            nl = Nonlocal()
 
-            if variable in local_vars:
-                obj = local_vars[variable]
-            else:
-                if variable not in self._objects:
-                    self._objects[variable] = self.extract_object(variable)
-                obj = self._objects[variable]
-
-            if arg_str is None:
-                # Member access
-                if member == 'length':
-                    return len(obj)
-                return obj[member]
-
-            assert expr.endswith(')')
-            # Function call
-            if arg_str == '':
-                argvals = tuple()
+            nl.member = remove_quotes(m.group('member') or m.group('member2'))
+            arg_str = expr[m.end():]
+            if arg_str.startswith('('):
+                arg_str, remaining = self._separate_at_paren(arg_str, ')')
             else:
-                argvals = tuple([
+                arg_str, remaining = None, arg_str
+
+            def assertion(cndn, msg):
+                """ assert, but without risk of getting optimized out """
+                if not cndn:
+                    raise ExtractorError('{0} {1}: {2}'.format(nl.member, msg, expr))
+
+            def eval_method():
+                # nonlocal member
+                member = nl.member
+                if variable == 'String':
+                    obj = str
+                elif variable in local_vars:
+                    obj = local_vars[variable]
+                else:
+                    if variable not in self._objects:
+                        self._objects[variable] = self.extract_object(variable)
+                    obj = self._objects[variable]
+
+                if arg_str is None:
+                    # Member access
+                    if member == 'length':
+                        return len(obj)
+                    return obj[member]
+
+                # Function call
+                argvals = [
                     self.interpret_expression(v, local_vars, allow_recursion)
-                    for v in arg_str.split(',')])
-
-            if member == 'split':
-                assert argvals == ('',)
-                return list(obj)
-            if member == 'join':
-                assert len(argvals) == 1
-                return argvals[0].join(obj)
-            if member == 'reverse':
-                assert len(argvals) == 0
-                obj.reverse()
-                return obj
-            if member == 'slice':
-                assert len(argvals) == 1
-                return obj[argvals[0]:]
-            if member == 'splice':
-                assert isinstance(obj, list)
-                index, howMany = argvals
-                res = []
-                for i in range(index, min(index + howMany, len(obj))):
-                    res.append(obj.pop(index))
-                return res
-
-            return obj[member](argvals)
-
-        for op, opfunc in _OPERATORS:
-            m = re.match(r'(?P<x>.+?)%s(?P<y>.+)' % re.escape(op), expr)
-            if not m:
-                continue
-            x, abort = self.interpret_statement(
-                m.group('x'), local_vars, allow_recursion - 1)
-            if abort:
-                raise ExtractorError(
-                    'Premature left-side return of %s in %r' % (op, expr))
-            y, abort = self.interpret_statement(
-                m.group('y'), local_vars, allow_recursion - 1)
-            if abort:
-                raise ExtractorError(
-                    'Premature right-side return of %s in %r' % (op, expr))
-            return opfunc(x, y)
+                    for v in self._separate(arg_str)]
+
+                if obj == str:
+                    if member == 'fromCharCode':
+                        assertion(argvals, 'takes one or more arguments')
+                        return ''.join(map(chr, argvals))
+                    raise ExtractorError('Unsupported string method %s' % (member, ))
+
+                if member == 'split':
+                    assertion(argvals, 'takes one or more arguments')
+                    assertion(argvals == [''], 'with arguments is not implemented')
+                    return list(obj)
+                elif member == 'join':
+                    assertion(isinstance(obj, list), 'must be applied on a list')
+                    assertion(len(argvals) == 1, 'takes exactly one argument')
+                    return argvals[0].join(obj)
+                elif member == 'reverse':
+                    assertion(not argvals, 'does not take any arguments')
+                    obj.reverse()
+                    return obj
+                elif member == 'slice':
+                    assertion(isinstance(obj, list), 'must be applied on a list')
+                    assertion(len(argvals) == 1, 'takes exactly one argument')
+                    return obj[argvals[0]:]
+                elif member == 'splice':
+                    assertion(isinstance(obj, list), 'must be applied on a list')
+                    assertion(argvals, 'takes one or more arguments')
+                    index, howMany = (argvals + [len(obj)])[:2]
+                    if index < 0:
+                        index += len(obj)
+                    add_items = argvals[2:]
+                    res = []
+                    for i in range(index, min(index + howMany, len(obj))):
+                        res.append(obj.pop(index))
+                    for i, item in enumerate(add_items):
+                        obj.insert(index + i, item)
+                    return res
+                elif member == 'unshift':
+                    assertion(isinstance(obj, list), 'must be applied on a list')
+                    assertion(argvals, 'takes one or more arguments')
+                    for item in reversed(argvals):
+                        obj.insert(0, item)
+                    return obj
+                elif member == 'pop':
+                    assertion(isinstance(obj, list), 'must be applied on a list')
+                    assertion(not argvals, 'does not take any arguments')
+                    if not obj:
+                        return
+                    return obj.pop()
+                elif member == 'push':
+                    assertion(argvals, 'takes one or more arguments')
+                    obj.extend(argvals)
+                    return obj
+                elif member == 'forEach':
+                    assertion(argvals, 'takes one or more arguments')
+                    assertion(len(argvals) <= 2, 'takes at-most 2 arguments')
+                    f, this = (argvals + [''])[:2]
+                    return [f((item, idx, obj), this=this) for idx, item in enumerate(obj)]
+                elif member == 'indexOf':
+                    assertion(argvals, 'takes one or more arguments')
+                    assertion(len(argvals) <= 2, 'takes at-most 2 arguments')
+                    idx, start = (argvals + [0])[:2]
+                    try:
+                        return obj.index(idx, start)
+                    except ValueError:
+                        return -1
+
+                if isinstance(obj, list):
+                    member = int(member)
+                    nl.member = member
+                return obj[member](argvals)
+
+            if remaining:
+                return self.interpret_expression(
+                    self._named_object(local_vars, eval_method()) + remaining,
+                    local_vars, allow_recursion)
+            else:
+                return eval_method()
 
-        m = re.match(
-            r'^(?P<func>%s)\((?P<args>[a-zA-Z0-9_$,]*)\)$' % _NAME_RE, expr)
+        m = re.match(r'^(?P<func>%s)\((?P<args>[a-zA-Z0-9_$,]*)\)$' % _NAME_RE, expr)
         if m:
             fname = m.group('func')
             argvals = tuple([
                 int(v) if v.isdigit() else local_vars[v]
-                for v in m.group('args').split(',')]) if len(m.group('args')) > 0 else tuple()
-            if fname not in self._functions:
+                for v in self._separate(m.group('args'))])
+            if fname in local_vars:
+                return local_vars[fname](argvals)
+            elif fname not in self._functions:
                 self._functions[fname] = self.extract_function(fname)
             return self._functions[fname](argvals)
 
-        raise ExtractorError('Unsupported JS expression %r' % expr)
+        if expr:
+            raise ExtractorError('Unsupported JS expression %r' % expr)
 
     def extract_object(self, objname):
         _FUNC_NAME_RE = r'''(?:[a-zA-Z$0-9]+|"[a-zA-Z$0-9]+"|'[a-zA-Z$0-9]+')'''
@@ -233,30 +501,52 @@ def extract_object(self, objname):
 
         return obj
 
-    def extract_function(self, funcname):
+    def extract_function_code(self, funcname):
+        """ @returns argnames, code """
         func_m = re.search(
             r'''(?x)
-                (?:function\s+%s|[{;,]\s*%s\s*=\s*function|var\s+%s\s*=\s*function)\s*
+                (?:function\s+%(f_n)s|[{;,]\s*%(f_n)s\s*=\s*function|var\s+%(f_n)s\s*=\s*function)\s*
                 \((?P<args>[^)]*)\)\s*
-                \{(?P<code>[^}]+)\}''' % (
-                re.escape(funcname), re.escape(funcname), re.escape(funcname)),
+                (?P<code>\{(?:(?!};)[^"]|"([^"]|\\")*")+\})''' % {'f_n': re.escape(funcname), },
             self.code)
+        code, _ = self._separate_at_paren(func_m.group('code'), '}')  # refine the match
         if func_m is None:
             raise ExtractorError('Could not find JS function %r' % funcname)
-        argnames = func_m.group('args').split(',')
+        return func_m.group('args').split(','), code
 
-        return self.build_function(argnames, func_m.group('code'))
+    def extract_function(self, funcname):
+        return self.extract_function_from_code(*self.extract_function_code(funcname))
+
+    def extract_function_from_code(self, argnames, code, *global_stack):
+        local_vars = {}
+        while True:
+            mobj = re.search(r'function\((?P<args>[^)]*)\)\s*{', code)
+            if mobj is None:
+                break
+            start, body_start = mobj.span()
+            body, remaining = self._separate_at_paren(code[body_start - 1:], '}')
+            name = self._named_object(
+                local_vars,
+                self.extract_function_from_code(
+                    [str.strip(x) for x in mobj.group('args').split(',')],
+                    body, local_vars, *global_stack))
+            code = code[:start] + name + remaining
+        return self.build_function(argnames, code, local_vars, *global_stack)
 
     def call_function(self, funcname, *args):
-        f = self.extract_function(funcname)
-        return f(args)
-
-    def build_function(self, argnames, code):
-        def resf(args):
-            local_vars = dict(zip(argnames, args))
-            for stmt in code.split(';'):
-                res, abort = self.interpret_statement(stmt, local_vars)
-                if abort:
+        return self.extract_function(funcname)(args)
+
+    def build_function(self, argnames, code, *global_stack):
+        global_stack = list(global_stack) or [{}]
+        local_vars = global_stack.pop(0)
+
+        def resf(args, **kwargs):
+            local_vars.update(dict(zip(argnames, args)))
+            local_vars.update(kwargs)
+            var_stack = LocalNameSpace(local_vars, *global_stack)
+            for stmt in self._separate(code.replace('\n', ''), ';'):
+                ret, should_abort = self.interpret_statement(stmt, var_stack)
+                if should_abort:
                     break
-            return res
+            return ret
         return resf

From 116500e2ba6c504008985163bed20d8f4c7c88e7 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Thu, 4 Nov 2021 12:48:06 +0000
Subject: [PATCH 16/20] Handle default in switch better

Add https://github.com/yt-dlp/yt-dlp/commit/a1fc7ca0743c8df06416e68ee74b64e07dfe7135
Thanks coletdjnz
---
 test/test_jsinterp.py  | 15 +++++++++++++++
 youtube_dl/jsinterp.py | 23 ++++++++++++++---------
 2 files changed, 29 insertions(+), 9 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 4d05ea61013..acdabffb1f8 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -133,6 +133,21 @@ def test_switch(self):
         self.assertEqual(jsi.call_function('x', 3), 6)
         self.assertEqual(jsi.call_function('x', 5), 0)
 
+    def test_switch_default(self):
+        jsi = JSInterpreter('''
+        function x(f) { switch(f){
+            case 2: f+=2;
+            default: f-=1;
+            case 5:
+            case 6: f+=6;
+            case 0: break;
+            case 1: f+=1;
+        } return f }
+        ''')
+        self.assertEqual(jsi.call_function('x', 1), 2)
+        self.assertEqual(jsi.call_function('x', 5), 11)
+        self.assertEqual(jsi.call_function('x', 9), 14)
+
     def test_try(self):
         jsi = JSInterpreter('''
         function x() { try{return 10} catch(e){return 5} }
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 061e92c2a4a..c35765702a6 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -240,21 +240,26 @@ def raise_constructor_error(c):
             switch_val, remaining = self._separate_at_paren(expr[m.end() - 1:], ')')
             switch_val = self.interpret_expression(switch_val, local_vars, allow_recursion)
             body, expr = self._separate_at_paren(remaining, '}')
-            body, default = body.split('default:') if 'default:' in body else (body, None)
-            items = body.split('case ')[1:]
-            if default:
-                items.append('default:%s' % (default, ))
-            matched = False
-            for item in items:
-                case, stmt = [i.strip() for i in self._separate(item, ':', 1)]
-                matched = matched or case == 'default' or switch_val == self.interpret_expression(case, local_vars, allow_recursion)
-                if matched:
+            items = body.replace('default:', 'case default:').split('case ')[1:]
+            for default in (False, True):
+                matched = False
+                for item in items:
+                    case, stmt = [i.strip() for i in self._separate(item, ':', 1)]
+                    if default:
+                        matched = matched or case == 'default'
+                    elif not matched:
+                        matched = (case != 'default'
+                                   and switch_val == self.interpret_expression(case, local_vars, allow_recursion))
+                    if not matched:
+                        continue
                     try:
                         ret, should_abort = self.interpret_statement(stmt, local_vars, allow_recursion - 1)
                         if should_abort:
                             return ret
                     except JS_Break:
                         break
+                if matched:
+                    break
             return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
 
         # Comma separated statements

From e74c63be530a27d692b65ec233c9c22d97fd7a74 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 27 Nov 2021 02:06:13 +0000
Subject: [PATCH 17/20] Fix splice to handle float

Needed for new youtube js player f1ca6900
Add https://github.com/yt-dlp/yt-dlp/commit/57dbe8077f8d00e0fffac53669f40cd7d584474f#diff-729b57caa8d006426f6a8960c061f519a8b6658682284015e069745af52ffb07
---
 youtube_dl/jsinterp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index c35765702a6..c75cf45b954 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -416,7 +416,7 @@ def eval_method():
                 elif member == 'splice':
                     assertion(isinstance(obj, list), 'must be applied on a list')
                     assertion(argvals, 'takes one or more arguments')
-                    index, howMany = (argvals + [len(obj)])[:2]
+                    index, howMany = map(int, (argvals + [len(obj)])[:2])
                     if index < 0:
                         index += len(obj)
                     add_items = argvals[2:]

From ff2dcd5b428016058706d2695c9b0afc8edfd14d Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 27 Nov 2021 03:18:29 +0000
Subject: [PATCH 18/20] Back-port test_youtube_signature.py from yt-dlp and fix
 JSInterp accordingly

---
 test/test_youtube_signature.py | 87 ++++++++++++++++++++++++----------
 youtube_dl/jsinterp.py         |  9 ++--
 2 files changed, 68 insertions(+), 28 deletions(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 627d4cb9255..c8e85b5005a 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -14,9 +14,10 @@
 
 from test.helper import FakeYDL
 from youtube_dl.extractor import YoutubeIE
+from youtube_dl.jsinterp import JSInterpreter
 from youtube_dl.compat import compat_str, compat_urlretrieve
 
-_TESTS = [
+_SIG_TESTS = [
     (
         'https://s.ytimg.com/yts/jsbin/html5player-vflHOr_nV.js',
         86,
@@ -64,6 +65,25 @@
     )
 ]
 
+_NSIG_TESTS = [
+    (
+        'https://www.youtube.com/s/player/9216d1f7/player_ias.vflset/en_US/base.js',
+        'SLp9F5bwjAdhE9F-', 'gWnb9IK2DJ8Q1w',
+    ),
+    (
+        'https://www.youtube.com/s/player/f8cb7a3b/player_ias.vflset/en_US/base.js',
+        'oBo2h5euWy6osrUt', 'ivXHpm7qJjJN',
+    ),
+    (
+        'https://www.youtube.com/s/player/2dfe380c/player_ias.vflset/en_US/base.js',
+        'oBo2h5euWy6osrUt', '3DIBbn3qdQ',
+    ),
+    (
+        'https://www.youtube.com/s/player/f1ca6900/player_ias.vflset/en_US/base.js',
+        'cu3wyu6LQn2hse', 'jvxetvmlI9AN9Q',
+    ),
+]
+
 
 class TestPlayerInfo(unittest.TestCase):
     def test_youtube_extract_player_info(self):
@@ -95,35 +115,54 @@ def setUp(self):
             os.mkdir(self.TESTDATA_DIR)
 
 
-def make_tfunc(url, sig_input, expected_sig):
-    m = re.match(r'.*-([a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.[a-z]+$', url)
-    assert m, '%r should follow URL format' % url
-    test_id = m.group(1)
+def t_factory(name, sig_func, url_pattern):
+    def make_tfunc(url, sig_input, expected_sig):
+        m = url_pattern.match(url)
+        assert m, '%r should follow URL format' % url
+        test_id = m.group('id')
+
+        def test_func(self):
+            basename = 'player-{0}-{1}.js'.format(name, test_id)
+            fn = os.path.join(self.TESTDATA_DIR, basename)
+
+            if not os.path.exists(fn):
+                compat_urlretrieve(url, fn)
+            with io.open(fn, encoding='utf-8') as testf:
+                jscode = testf.read()
+            self.assertEqual(sig_func(jscode, sig_input), expected_sig)
+
+        test_func.__name__ = str('test_{0}_js_{1}'.format(name, test_id))
+        setattr(TestSignature, test_func.__name__, test_func)
+    return make_tfunc
+
 
-    def test_func(self):
-        basename = 'player-%s.js' % test_id
-        fn = os.path.join(self.TESTDATA_DIR, basename)
+def signature(jscode, sig_input):
+    func = YoutubeIE(FakeYDL())._parse_sig_js(jscode)
+    src_sig = (
+        compat_str(string.printable[:sig_input])
+        if isinstance(sig_input, int) else sig_input)
+    return func(src_sig)
 
-        if not os.path.exists(fn):
-            compat_urlretrieve(url, fn)
 
-        ydl = FakeYDL()
-        ie = YoutubeIE(ydl)
-        with io.open(fn, encoding='utf-8') as testf:
-            jscode = testf.read()
-        func = ie._parse_sig_js(jscode)
-        src_sig = (
-            compat_str(string.printable[:sig_input])
-            if isinstance(sig_input, int) else sig_input)
-        got_sig = func(src_sig)
-        self.assertEqual(got_sig, expected_sig)
+def n_sig(jscode, sig_input):
+    # Pending implementation of _extract_n_function_name() or similar in
+    # youtube.py, hard-code here
+    # funcname = YoutubeIE(FakeYDL())._extract_n_function_name(jscode)
+    import re
+    funcname = re.search(r'[=(,&|](\w+)\(\w+\),\w+\.set\("n",', jscode)
+    funcname = funcname and funcname.group(1)
+    return JSInterpreter(jscode).call_function(funcname, sig_input)
 
-    test_func.__name__ = str('test_signature_js_' + test_id)
-    setattr(TestSignature, test_func.__name__, test_func)
 
+make_sig_test = t_factory(
+    'signature', signature, re.compile(r'.*-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.[a-z]+$'))
+for test_spec in _SIG_TESTS:
+    make_sig_test(*test_spec)
 
-for test_spec in _TESTS:
-    make_tfunc(*test_spec)
+make_nsig_test = t_factory(
+    'nsig', n_sig, re.compile(r'.+/player/(?P<id>[a-zA-Z0-9_-]+)/.+.js$'))
+for test_spec in _NSIG_TESTS:
+    make_nsig_test(*test_spec)
 
 
 if __name__ == '__main__':
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index c75cf45b954..a2306557b4a 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -9,7 +9,8 @@
     remove_quotes,
 )
 from .compat import (
-    compat_collections_abc
+    compat_collections_abc,
+    compat_str,
 )
 MutableMapping = compat_collections_abc.MutableMapping
 
@@ -372,7 +373,7 @@ def eval_method():
                 # nonlocal member
                 member = nl.member
                 if variable == 'String':
-                    obj = str
+                    obj = compat_str
                 elif variable in local_vars:
                     obj = local_vars[variable]
                 else:
@@ -391,7 +392,7 @@ def eval_method():
                     self.interpret_expression(v, local_vars, allow_recursion)
                     for v in self._separate(arg_str)]
 
-                if obj == str:
+                if obj == compat_str:
                     if member == 'fromCharCode':
                         assertion(argvals, 'takes one or more arguments')
                         return ''.join(map(chr, argvals))
@@ -533,7 +534,7 @@ def extract_function_from_code(self, argnames, code, *global_stack):
             name = self._named_object(
                 local_vars,
                 self.extract_function_from_code(
-                    [str.strip(x) for x in mobj.group('args').split(',')],
+                    [x.strip() for x in mobj.group('args').split(',')],
                     body, local_vars, *global_stack))
             code = code[:start] + name + remaining
         return self.build_function(argnames, code, local_vars, *global_stack)

From c3b0045720dbd47edfc0c30eec15a6f068836a2f Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 10 Dec 2021 19:14:54 +0000
Subject: [PATCH 19/20] Refactor JSInterpreter._separate

yt-dlp/yt-dlp/@06dfe0a, improve _MATCHING_PARENS
---
 youtube_dl/jsinterp.py | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index a2306557b4a..8eaa911cdab 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -36,6 +36,8 @@ class Nonlocal:
 
 _NAME_RE = r'[a-zA-Z_$][a-zA-Z_$0-9]*'
 
+_MATCHING_PARENS = dict(zip(*zip('()', '{}', '[]')))
+
 
 class JS_Break(ExtractorError):
     def __init__(self):
@@ -100,26 +102,24 @@ def _named_object(self, namespace, obj):
     def _separate(expr, delim=',', max_split=None):
         if not expr:
             return
-        parens = {'(': 0, '{': 0, '[': 0, ']': 0, '}': 0, ')': 0}
-        start, splits, pos, max_pos = 0, 0, 0, len(delim) - 1
+        counters = {k: 0 for k in _MATCHING_PARENS.values()}
+        start, splits, pos, delim_len = 0, 0, 0, len(delim) - 1
         for idx, char in enumerate(expr):
-            if char in parens:
-                parens[char] += 1
-            is_in_parens = (parens['['] - parens[']']
-                            or parens['('] - parens[')']
-                            or parens['{'] - parens['}'])
-            if char == delim[pos] and not is_in_parens:
-                if pos == max_pos:
-                    pos = 0
-                    yield expr[start: idx - max_pos]
-                    start = idx + 1
-                    splits += 1
-                    if max_split and splits >= max_split:
-                        break
-                else:
-                    pos += 1
-            else:
+            if char in _MATCHING_PARENS:
+                counters[_MATCHING_PARENS[char]] += 1
+            elif char in counters:
+                counters[char] -= 1
+            if char != delim[pos] or any(counters.values()):
                 pos = 0
+                continue
+            elif pos != delim_len:
+                pos += 1
+                continue
+            yield expr[start: idx - delim_len]
+            start, pos = idx + 1, 0
+            splits += 1
+            if max_split and splits >= max_split:
+                break
         yield expr[start:]
 
     @staticmethod

From 905d1d281ddfa5d183fc445010d350cefc6a58ec Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 17 Dec 2021 14:36:42 +0000
Subject: [PATCH 20/20] Implement n-param descrambling using JSInterp

---
 youtube_dl/extractor/youtube.py | 326 +++++---------------------------
 1 file changed, 46 insertions(+), 280 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 6f3162decd6..63918924df8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1385,292 +1385,62 @@ def _extract_player_url(self, webpage):
                 'https://www.youtube.com', player_url)
         return player_url
 
-    # Based on an equivalent function [2] in the youtube.lua script from VLC
-    # Many thanks to @linkfanel [3]
-    # NB This code could fail if YT should revise the player code and would then have
-    # to be reworked (thankless task previously undertaken at [1] and [2])
+    # from yt-dlp
+    # See also:
     # 1. https://github.com/ytdl-org/youtube-dl/issues/29326#issuecomment-894619419
     # 2. https://code.videolan.org/videolan/vlc/-/blob/4fb284e5af69aa9ac2100ccbdd3b88debec9987f/share/lua/playlist/youtube.lua#L116
     # 3. https://github.com/ytdl-org/youtube-dl/issues/30097#issuecomment-950157377
-    def _n_descramble(self, n_param, js):
+    def _extract_n_function_name(self, jscode):
+        return self._search_regex(
+            (r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]{3})\([a-zA-Z0-9]\)',),
+            jscode, 'Initial JS player n function name', group='nfunc')
+
+    def _extract_n_function(self, video_id, player_url):
+        player_id = self._extract_player_info(player_url)
+        func_code = self._downloader.cache.load('youtube-nsig', player_id)
+
+        if func_code:
+            jsi = JSInterpreter(func_code)
+        else:
+            player_id = self._extract_player_info(player_url)
+            jscode = self._get_player_code(video_id, player_url, player_id)
+            funcname = self._extract_n_function_name(jscode)
+            jsi = JSInterpreter(jscode)
+            func_code = jsi.extract_function_code(funcname)
+            self._downloader.cache.store('youtube-nsig', player_id, func_code)
+
+        if self._downloader.params.get('youtube_print_sig_code'):
+            self.to_screen('Extracted nsig function from {0}:\n{1}\n'.format(player_id, func_code[1]))
+
+        return lambda s: jsi.extract_function_from_code(*func_code)([s])
+
+    def _n_descramble(self, n_param, player_url, video_id):
         """Compute the response to YT's "n" parameter challenge
 
         Args:
-        n_param -- challenge string that is the value of the
-                   URL's "n" query parameter
-        js      -- text of the JS player code that includes the
-                   challenge response algorithm
+        n_param     -- challenge string that is the value of the
+                       URL's "n" query parameter
+        player_url  -- URL of YT player JS
+        video_id
         """
-        if not js:
-            return
-
-        # helper functions (part 1)
-        def isiterable(x):
-            try:
-                return x.__getitem__ and True
-            except AttributeError:
-                return False
 
-        def find_first(pattern, string, flags=0, groups=1):
-            pattern = re.compile(pattern, flags)
-            return next(map(lambda m: m.groups() if groups is True else m.group(groups),
-                            pattern.finditer(string)),
-                        (None, ) * pattern.groups if groups is True else None)
-
-        # Look for the descrambler function's name
-        # a.D&&(b=a.get("n"))&&(b=lha(b),a.set("n",b))}};
-        descrambler = find_first(r'[=(,&|](\w+)\(\w+\),\w+\.set\("n",', js)
-        if not descrambler:
-            self.report_warning("Couldn't extract YouTube video throttling parameter descrambling function name")
-            return
-        # Fetch the code of the descrambler function
-        # lha=function(a){var b=a.split(""),c=[310282131,"KLf3",b,null,function(d,e){d.push(e)},-45817231, [data and transformations...] ,1248130556];c[3]=c;c[15]=c;c[18]=c;try{c[40](c[14],c[2]),c[25](c[48]),c[21](c[32],c[23]), [scripted calls...] ,c[25](c[33],c[3])}catch(d){return"enhanced_except_4ZMBnuz-_w8_"+a}return b.join("")};
-        code = find_first(r'(?s)%s=function\([^)]+\)\{(.+?)\};' % (descrambler, ), js)
-        if not code:
-            self.report_warning("Couldn't extract YouTube video throttling parameter descrambling code")
-            return
-        # Split code into two main sections: 1/ data and transformations,
-        # and 2/ a script of calls
-        datac, xc, script = find_first(r'(?s)c=\[(.+)\];(.+?);try\{(.+)\}catch\(', code, groups=True)
-        if not datac or not script:
-            self.report_warning("Couldn't extract YouTube video throttling parameter descrambling rules")
-            return
-        if xc:
-            # ad hoc update of c[] array after declaration (player 2dfe380c, ...)
-            xc = [int(x) for x in re.findall(r'c\[(\d+)\]=c\b', xc)]
-            if not xc:
-                self.report_warning("Couldn't extract YouTube video throttling parameter additional rules")
-        else:
-            xc = []
-
-        # Split "n" parameter into a table as descrambling operates on it
-        # as one of several arrays - in Python just copy it as a list
-        n = list(n_param)
-        # Helper: table_len = function() ... end - in Python just use len
-
-        # Common routine shared by the compound transformations,
-        # compounding the "n" parameter with an input string,
-        # character by character using a Base64 alphabet.
-        # d.forEach(function(l,m,n){this.push(n[m]=h[(h.indexOf(l)-h.indexOf(this[m])+m-32+f--)%h.length])},e.split(""))
-        def compound(ntab, strg, alphabet, charcode):
-            if ntab != n or type(strg) != compat_str:
-                return True
-            inp = list(strg)
-            llen = len(alphabet)
-            ntab_copy = ntab[:]
-            for i, c in enumerate(ntab_copy):
-                if type(c) != compat_str:
-                    return True
-                pos1 = alphabet.find(c)
-                pos2 = alphabet.find(inp[i])
-                if pos1 < 0 or pos2 < 0:
-                    return True
-                pos = (pos1 - pos2 + charcode - 32) % llen
-                newc = alphabet[pos]
-                ntab[i] = newc
-                inp.append(newc)
-
-        # The data section contains among others function code for a number
-        # of transformations, most of which are basic array operations.
-        # We can match these functions' code to identify them, and emulate
-        # the corresponding transformations.
-
-        # helper fns (in-place)
-        def swap(s, i, j):
-            x = s[i]
-            s[i] = s[j]
-            s[j] = x
-
-        def rotate(s, i):
-            tmp = s[:]
-            tmp[i:] = s
-            tmp[:i] = tmp[len(s):]
-            s[:] = tmp[:len(s)]
-
-        def remove(s, i):
-            del s[i]
-
-        b64_alphabets = [
-            "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-_",
-            "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_",
-        ]
+        sig_id = ('nsig_value', n_param)
+        if sig_id in self._player_cache:
+            return self._player_cache[sig_id]
 
-        # Compounding functions use a subfunction, so we need to be
-        # more specific in how much parsed data we consume.
-        cp_skip = r'(?s)^.*?\},e\.split\(""\)\)},\s*(.*)$'
-        def_skip = r"(?s)^.*?\},\s*(.*)$"
-
-        trans = (
-            # fn_name, fn, fn_detect_pattern, skip_re
-            ('reverse', lambda tab, _: tab.reverse(),
-                        # noqa: E127
-                        # function(d){d.reverse()}
-                        # function(d){for(var e=d.length;e;)d.push(d.splice(--e,1)[0])}
-                        r"^function\(d\)",
-                        def_skip),
-            ('append', lambda tab, val: tab.append(val),
-                       # noqa: E127
-                       # function(d,e){d.push(e)}
-                       r"^function\(d,e\){d\.push\(e\)\},",
-                       def_skip),
-            ('remove', lambda tab, i: remove(tab, i % len(tab)) if type(i) == int else True,
-                       # noqa: E127
-                       # function(d,e){e=(e%d.length+d.length)%d.length;d.splice(e,1)}
-                       r"^[^}]+?;d\.splice\(e,1\)\},",
-                       def_skip),
-            ('swap', lambda tab, i: swap(tab, 0, i % len(tab)) if type(i) == int else True,
-                     # noqa: E127
-                     # function(d,e){e=(e%d.length+d.length)%d.length;var f=d[0];d[0]=d[e];d[e]=f}
-                     # function(d,e){e=(e%d.length+d.length)%d.length;d.splice(0,1,d.splice(e,1,d[0])[0])}
-                     r"^[^}]+?;(?:var\sf=d\[0\];d\[0\]=d\[e\];d\[e\]=f|d\.splice\(0,1,d\.splice\(e,1,d\[0\]\)\[0\]\))\},",
-                     def_skip),
-            ('rotate', lambda tab, shift: rotate(tab, shift % len(tab)) if type(shift) == int else True,
-                       # noqa: E127
-                       # function(d,e){for(e=(e%d.length+d.length)%d.length;e--;)d.unshift(d.pop())}
-                       # function(d,e){e=(e%d.length+d.length)%d.length;d.splice(-e).reverse().forEach(function(f){d.unshift(f)})}
-                       r"^[^}]+?d\.unshift\((?:d\.pop\(\)|f\)\})\)},",
-                       def_skip),
-            ('alphabet1', lambda: b64_alphabets[0],
-                          # noqa: E127
-                          # function(){for(var d=64,e=[];++d-e.length-32;){switch(d){case 91:d=44;continue;case 123:d=65;break;case 65:d-=18;continue;case 58:d=96;continue;case 46:d=95}e.push(String.fromCharCode(d))}return e}
-                          r"^[^}]+?case\s58:d=96;.+?\}return",
-                          def_skip),
-            ('alphabet2', lambda: b64_alphabets[1],
-                          # noqa: E127
-                          # function(){for(var d=64,e=[];++d-e.length-32;)switch(d){case 46:d=95;default:e.push(String.fromCharCode(d));case 94:case 95:case 96:break;case 123:d-=76;case 92:case 93:continue;case 58:d=44;case 91:}return e}
-                          # function(){for(var d=64,e=[];++d-e.length-32;){switch(d){case 58:d-=14;case 91:case 92:case 93:continue;case 123:d=47;case 94:case 95:case 96:continue;case 46:d=95}e.push(String.fromCharCode(d))}return e}
-                          r"^[^}]+?case\s58:d(?:=44;|-=14;[^}]+\})[^}]+\}return",
-                          def_skip),
-            ('compound', lambda tab, s, alphabet: compound(tab, s, alphabet, 96),
-                         # noqa: E127
-                         # function(d,e,f){var h=f.length;d.forEach(function(l,m,n){this.push(n[m]=f[(f.indexOf(l)-f.indexOf(this[m])+m+h--)%f.length])},e.split(""))}
-                         r"^function\(\w,\w,\w\)",
-                         cp_skip),
-
-            # Compound transformations first build a variation of a
-            # Base64 alphabet, then in a common section, compound the
-            # "n" parameter with an input string, character by character.
-            ('compound1', lambda tab, s: compound(tab, s, b64_alphabets[0], 96),
-                          # noqa: E127
-                          # function(d,e){for(var f=64,h=[];++f-h.length-32;)switch(f){case 58:f=96;continue;case 91:f=44;break;case 65:f=47;continue;case 46:f=153;case 123:f-=58;default:h.push(String.fromCharCode(f))} [ compound... ] }
-                          r"^[^}]+?case\s58:f=96;",
-                          cp_skip),
-            ('compound2', lambda tab, s: compound(tab, s, b64_alphabets[1], 96),
-                          # noqa: E127
-                          # function(d,e){for(var f=64,h=[];++f-h.length-32;){switch(f){case 58:f-=14;case 91:case 92:case 93:continue;case 123:f=47;case 94:case 95:case 96:continue;case 46:f=95}h.push(String.fromCharCode(f))} [ compound... ] }
-                          # function(d,e){for(var f=64,h=[];++f-h.length-32;)switch(f){case 46:f=95;default:h.push(String.fromCharCode(f));case 94:case 95:case 96:break;case 123:f-=76;case 92:case 93:continue;case 58:f=44;case 91:} [ compound... ] }
-                          r"^[^}]+?case\s58:f(?:-=14|=44);",
-                          cp_skip),
-            # Fallback
-            ('unid', lambda _, __: self.report_warning("Couldn't apply unidentified YouTube video throttling parameter transformation, aborting descrambling") or True,
-                     # noqa: E127
-                     None,
-                     def_skip),
-        )
-        # The data section actually mixes input data, reference to the
-        # "n" parameter array, and self-reference to its own array, with
-        # transformation functions used to modify itself. We parse it
-        # as such into a table.
-        data = []
-        datac += ","
-        while datac:
-            # Transformation functions
-            if re.match(r"^function\(", datac):
-                name, el, _, skip = next(
-                    itertools.dropwhile(
-                        lambda x: x[2] is not None and not re.match(x[2], datac), trans))
-                datac = find_first(skip, datac)
-            # String input data
-            elif re.match(r'^"[^"]*",', datac):
-                el, datac = find_first(r'(?s)^"([^"]*)",\s*(.*)$', datac, groups=True)
-            # Integer input data
-            elif re.match(r'^-?\d+(?:E\d+)?,', datac):
-                el, datac = find_first(r"(?s)^(.*?),\s*(.*)$", datac, groups=True)
-                el = int(float(el) if 'E' in el else el)
-            # Reference to "n" parameter array
-            elif re.match('^b,', datac):
-                el = n
-                datac = find_first(r"(?s)^b,\s*(.*)$", datac)
-            # Replaced by self-reference to data array after its declaration
-            elif re.match('^null,', datac):
-                el = data
-                datac = find_first(r"(?s)^null,\s*(.*)$", datac)
-            else:
-                self.report_warning("Couldn't parse unidentified YouTube video throttling parameter descrambling data"
-                                    '\nNear: "%s"' % datac[:64])
-                el = False
-                # Lua tables can't contain nil values: Python can, but still use False
-                datac = find_first(r"(?s)^[^,]*?,\s*(.*)$", datac)
-            data.append(el)
-
-        # Additional rules
-        for idx in xc:
-            data[idx] = data
-
-        # Debugging helper to print data array elements
-        def prd(el, tab=None):
-            if not el:
-                return "???"
-            elif el == n:
-                return "n"
-            elif el == data:
-                return "data"
-            elif type(el) == compat_str:
-                return '"%s"' % (el, )
-            elif type(el) == int:
-                if isiterable(tab):
-                    return "%d -> %d" % (el, el % len(tab), )
-                return "%d" % (el, )
-            else:
-                for tr in trans:
-                    if el == tr[1]:
-                        return tr[0]
-            return repr(el)
-
-        # The script section contains a series of calls to elements of
-        # the data section array onto other elements of it: calls to
-        # transformations, with a reference to the data array itself or
-        # the "n" parameter array as first argument, and often input data
-        # as a second argument. We parse and emulate those calls to follow
-        # the descrambling script.
-        # c[40](c[14],c[2]),c[25](c[48]),c[21](c[32],c[23]), [...]
-        for ifunc, itab, iarg, iextra in map(lambda m: m.groups(),
-                                             # c[m](c[mm]{,c[mmm]{,c[mmmm]}})
-                                             re.finditer(
-                                                 r"c\[(\d+)\]\(c\[(\d+)\](?:,\s*c\[(\d+)\])?(?:,\s*c\[(\d+)\]\(\))?[^)]*?\)",
-                                                 script)):
-            tab = data[int(itab)]
-            arg = iarg and data[int(iarg)]
-            if iextra:
-                alphabet = data[int(iextra)]()
-                func = lambda t, s: data[int(ifunc)](t, s, alphabet)
-            else:
-                func = data[int(ifunc)]
-
-            # Uncomment to debug transformation chain
-            # nprev = ''.join(n)
-            # dprev = ' '.join(map(prd, data))
-            # print(''.join(('"n" parameter transformation: ', prd(func), "(", prd(tab), (", " + prd(arg, tab)) if arg else '', ") ", ifunc, "(", itab, (", " + iarg) if iarg else "", ")")))
-            if not callable(func) or not isiterable(tab) or func(tab, arg):
-                self.report_warning("Invalid data type encountered during YouTube video throttling parameter descrambling transformation chain, aborting"
-                                    "\nCouldn't descramble YouTube throttling URL parameter: data transfer will be throttled")
-                self.report_warning("Couldn't process youtube video URL, please check for updates to this script")
-                break
-            # Uncomment to debug transformation chain
-            # nnew = ''.join(n)
-            # if nprev != nnew:
-            #     print('from: ' + nprev + "\nto:   " + nnew)
-            # dnew = ' '.join(map(prd, data))
-            # if dprev != dnew:
-            #     print('from: ' + dprev + "\nto:   " + dnew)
-        return ''.join(n)
+        try:
+            player_id = ('nsig', player_url)
+            if player_id not in self._player_cache:
+                self._player_cache[player_id] = self._extract_n_function(video_id, player_url)
+            func = self._player_cache[player_id]
+            self._player_cache[sig_id] = func(n_param)
+            if self._downloader.params.get('verbose', False):
+                self._downloader.to_screen('[debug] [%s] %s' % (self.IE_NAME, 'Decrypted nsig {0} => {1}'.format(n_param, self._player_cache[sig_id])))
+            return self._player_cache[sig_id]
+        except Exception as e:
+            raise ExtractorError(traceback.format_exc(), cause=e, video_id=video_id)
 
     def _unthrottle_format_urls(self, video_id, player_url, formats):
-        if not player_url:
-            return
-        player_id = self._extract_player_info(player_url)
-        code = self._get_player_code(video_id, player_url, player_id)
-        n_cache = {}
         for fmt in formats:
             parsed_fmt_url = compat_urlparse.urlparse(fmt['url'])
             qs = compat_urlparse.parse_qs(parsed_fmt_url.query)
@@ -1678,11 +1448,7 @@ def _unthrottle_format_urls(self, video_id, player_url, formats):
             if not n_param:
                 continue
             n_param = n_param[-1]
-            n_response = n_cache.get(n_param)
-            if not n_response:
-                n_response = self._n_descramble(n_param, code)
-                if n_response:
-                    n_cache[n_param] = n_response
+            n_response = self._n_descramble(n_param, player_url, video_id)
             if n_response:
                 qs['n'] = [n_response]
                 fmt['url'] = compat_urlparse.urlunparse(