Direct cipher signature & n-transform functions to circumvent throttling.

lib/sig.js

@@ -1,245 +1,120 @@
 const querystring = require('querystring');
 const Cache = require('./cache');
 const utils = require('./utils');
+const vm = require('vm');
 
-
-// A shared cache to keep track of html5player.js tokens.
+// A shared cache to keep track of html5player js functions.
 exports.cache = new Cache();
 
-
 /**
- * Extract signature deciphering tokens from html5player file.
+ * Extract signature deciphering and n parameter transform functions from html5player file.
  *
  * @param {string} html5playerfile
  * @param {Object} options
  * @returns {Promise<Array.<string>>}
  */
-exports.getTokens = (html5playerfile, options) => exports.cache.getOrSet(html5playerfile, async() => {
+exports.getFunctions = (html5playerfile, options) => exports.cache.getOrSet(html5playerfile, async() => {
   const body = await utils.exposedMiniget(html5playerfile, options).text();
-  const tokens = exports.extractActions(body);
-  if (!tokens || !tokens.length) {
-    throw Error('Could not extract signature deciphering actions');
+  const functions = exports.extractFunctions(body);
+  if (!functions || !functions.length) {
+    throw Error('Could not extract functions');
   }
-  exports.cache.set(html5playerfile, tokens);
-  return tokens;
+  exports.cache.set(html5playerfile, functions);
+  return functions;
 });
 
-
-/**
- * Decipher a signature based on action tokens.
- *
- * @param {Array.<string>} tokens
- * @param {string} sig
- * @returns {string}
- */
-exports.decipher = (tokens, sig) => {
-  sig = sig.split('');
-  for (let i = 0, len = tokens.length; i < len; i++) {
-    let token = tokens[i], pos;
-    switch (token[0]) {
-      case 'r':
-        sig = sig.reverse();
-        break;
-      case 'w':
-        pos = ~~token.slice(1);
-        sig = swapHeadAndPosition(sig, pos);
-        break;
-      case 's':
-        pos = ~~token.slice(1);
-        sig = sig.slice(pos);
-        break;
-      case 'p':
-        pos = ~~token.slice(1);
-        sig.splice(0, pos);
-        break;
-    }
-  }
-  return sig.join('');
-};
-
-
-/**
- * Swaps the first element of an array with one of given position.
- *
- * @param {Array.<Object>} arr
- * @param {number} position
- * @returns {Array.<Object>}
- */
-const swapHeadAndPosition = (arr, position) => {
-  const first = arr[0];
-  arr[0] = arr[position % arr.length];
-  arr[position] = first;
-  return arr;
-};
-
-
-const jsVarStr = '[a-zA-Z_\\$][a-zA-Z_0-9]*';
-const jsSingleQuoteStr = `'[^'\\\\]*(:?\\\\[\\s\\S][^'\\\\]*)*'`;
-const jsDoubleQuoteStr = `"[^"\\\\]*(:?\\\\[\\s\\S][^"\\\\]*)*"`;
-const jsQuoteStr = `(?:${jsSingleQuoteStr}|${jsDoubleQuoteStr})`;
-const jsKeyStr = `(?:${jsVarStr}|${jsQuoteStr})`;
-const jsPropStr = `(?:\\.${jsVarStr}|\\[${jsQuoteStr}\\])`;
-const jsEmptyStr = `(?:''|"")`;
-const reverseStr = ':function\\(a\\)\\{' +
-  '(?:return )?a\\.reverse\\(\\)' +
-'\\}';
-const sliceStr = ':function\\(a,b\\)\\{' +
-  'return a\\.slice\\(b\\)' +
-'\\}';
-const spliceStr = ':function\\(a,b\\)\\{' +
-  'a\\.splice\\(0,b\\)' +
-'\\}';
-const swapStr = ':function\\(a,b\\)\\{' +
-  'var c=a\\[0\\];a\\[0\\]=a\\[b(?:%a\\.length)?\\];a\\[b(?:%a\\.length)?\\]=c(?:;return a)?' +
-'\\}';
-const actionsObjRegexp = new RegExp(
-  `var (${jsVarStr})=\\{((?:(?:${
-    jsKeyStr}${reverseStr}|${
-    jsKeyStr}${sliceStr}|${
-    jsKeyStr}${spliceStr}|${
-    jsKeyStr}${swapStr
-  }),?\\r?\\n?)+)\\};`);
-const actionsFuncRegexp = new RegExp(`${`function(?: ${jsVarStr})?\\(a\\)\\{` +
-    `a=a\\.split\\(${jsEmptyStr}\\);\\s*` +
-    `((?:(?:a=)?${jsVarStr}`}${
-  jsPropStr
-}\\(a,\\d+\\);)+)` +
-    `return a\\.join\\(${jsEmptyStr}\\)` +
-  `\\}`);
-const reverseRegexp = new RegExp(`(?:^|,)(${jsKeyStr})${reverseStr}`, 'm');
-const sliceRegexp = new RegExp(`(?:^|,)(${jsKeyStr})${sliceStr}`, 'm');
-const spliceRegexp = new RegExp(`(?:^|,)(${jsKeyStr})${spliceStr}`, 'm');
-const swapRegexp = new RegExp(`(?:^|,)(${jsKeyStr})${swapStr}`, 'm');
-
-
 /**
- * Extracts the actions that should be taken to decipher a signature.
- *
- * This searches for a function that performs string manipulations on
- * the signature. We already know what the 3 possible changes to a signature
- * are in order to decipher it. There is
- *
- * * Reversing the string.
- * * Removing a number of characters from the beginning.
- * * Swapping the first character with another position.
- *
- * Note, `Array#slice()` used to be used instead of `Array#splice()`,
- * it's kept in case we encounter any older html5player files.
- *
- * After retrieving the function that does this, we can see what actions
- * it takes on a signature.
+ * Extracts the actions that should be taken to decipher a signature
+ * and tranform the n parameter
  *
  * @param {string} body
  * @returns {Array.<string>}
  */
-exports.extractActions = body => {
-  const objResult = actionsObjRegexp.exec(body);
-  const funcResult = actionsFuncRegexp.exec(body);
-  if (!objResult || !funcResult) { return null; }
-
-  const obj = objResult[1].replace(/\$/g, '\\$');
-  const objBody = objResult[2].replace(/\$/g, '\\$');
-  const funcBody = funcResult[1].replace(/\$/g, '\\$');
-
-  let result = reverseRegexp.exec(objBody);
-  const reverseKey = result && result[1]
-    .replace(/\$/g, '\\$')
-    .replace(/\$|^'|^"|'$|"$/g, '');
-  result = sliceRegexp.exec(objBody);
-  const sliceKey = result && result[1]
-    .replace(/\$/g, '\\$')
-    .replace(/\$|^'|^"|'$|"$/g, '');
-  result = spliceRegexp.exec(objBody);
-  const spliceKey = result && result[1]
-    .replace(/\$/g, '\\$')
-    .replace(/\$|^'|^"|'$|"$/g, '');
-  result = swapRegexp.exec(objBody);
-  const swapKey = result && result[1]
-    .replace(/\$/g, '\\$')
-    .replace(/\$|^'|^"|'$|"$/g, '');
-
-  const keys = `(${[reverseKey, sliceKey, spliceKey, swapKey].join('|')})`;
-  const myreg = `(?:a=)?${obj
-  }(?:\\.${keys}|\\['${keys}'\\]|\\["${keys}"\\])` +
-    `\\(a,(\\d+)\\)`;
-  const tokenizeRegexp = new RegExp(myreg, 'g');
-  const tokens = [];
-  while ((result = tokenizeRegexp.exec(funcBody)) !== null) {
-    let key = result[1] || result[2] || result[3];
-    switch (key) {
-      case swapKey:
-        tokens.push(`w${result[4]}`);
-        break;
-      case reverseKey:
-        tokens.push('r');
-        break;
-      case sliceKey:
-        tokens.push(`s${result[4]}`);
-        break;
-      case spliceKey:
-        tokens.push(`p${result[4]}`);
-        break;
+exports.extractFunctions = body => {
+  const functions = [];
+  const extractManipulations = caller => {
+    const functionName = utils.between(caller, `a=a.split("");`, `.`);
+    if (!functionName) return '';
+    const functionStart = `var ${functionName}={`;
+    const ndx = body.indexOf(functionStart);
+    if (ndx < 0) return '';
+    const subBody = body.slice(ndx + functionStart.length - 1);
+    return `var ${functionName}=${utils.cutAfterJSON(subBody)}`;
+  };
+  const extractDecipher = () => {
+    const functionName = utils.between(body, `a.set("alr","yes");c&&(c=`, `(decodeURIC`);
+    if (functionName && functionName.length) {
+      const functionStart = `${functionName}=function(a)`;
+      const ndx = body.indexOf(functionStart);
+      if (ndx >= 0) {
+        const subBody = body.slice(ndx + functionStart.length);
+        let functionBody = `var ${functionStart}${utils.cutAfterJSON(subBody)}`;
+        functionBody = `${extractManipulations(functionBody)};${functionBody};${functionName}(sig);`;
+        functions.push(functionBody);
+      }
     }
-  }
-  return tokens;
+  };
+  const extractNCode = () => {
+    const functionName = utils.between(body, `&&(b=a.get("n"))&&(b=`, `(b)`);
+    if (functionName && functionName.length) {
+      const functionStart = `${functionName}=function(a)`;
+      const ndx = body.indexOf(functionStart);
+      if (ndx >= 0) {
+        const subBody = body.slice(ndx + functionStart.length);
+        const functionBody = `var ${functionStart}${utils.cutAfterJSON(subBody)};${functionName}(ncode);`;
+        functions.push(functionBody);
+      }
+    }
+  };
+  extractDecipher();
+  extractNCode();
+  return functions;
 };
 
-
 /**
+ * Apply decipher and n-transform to individual format
+ *
  * @param {Object} format
- * @param {string} sig
+ * @param {vm.Script} decipherScript
+ * @param {vm.Script} nTransformScript
  */
-exports.setDownloadURL = (format, sig) => {
-  let decodedUrl;
-  if (format.url) {
-    decodedUrl = format.url;
-  } else {
-    return;
-  }
-
-  try {
-    decodedUrl = decodeURIComponent(decodedUrl);
-  } catch (err) {
-    return;
-  }
-
-  // Make some adjustments to the final url.
-  const parsedUrl = new URL(decodedUrl);
-
-  // This is needed for a speedier download.
-  // See https://github.com/fent/node-ytdl-core/issues/127
-  parsedUrl.searchParams.set('ratebypass', 'yes');
-
-  if (sig) {
-    // When YouTube provides a `sp` parameter the signature `sig` must go
-    // into the parameter it specifies.
-    // See https://github.com/fent/node-ytdl-core/issues/417
-    parsedUrl.searchParams.set(format.sp || 'signature', sig);
-  }
-
-  format.url = parsedUrl.toString();
+exports.setDownloadURL = (format, decipherScript, nTransformScript) => {
+  const decipher = url => {
+    const args = querystring.parse(url);
+    if (!args.s || !decipherScript) return args.url;
+    const components = new URL(decodeURIComponent(args.url));
+    components.searchParams.set(args.sp ? args.sp : 'signature',
+      decipherScript.runInNewContext({ sig: decodeURIComponent(args.s) }));
+    return components.toString();
+  };
+  const ncode = url => {
+    const components = new URL(decodeURIComponent(url));
+    const n = components.searchParams.get('n');
+    if (!n || !nTransformScript) return url;
+    components.searchParams.set('n', nTransformScript.runInNewContext({ ncode: n }));
+    return components.toString();
+  };
+  const cipher = !format.url;
+  const url = format.url || format.signatureCipher || format.cipher;
+  format.url = cipher ? ncode(decipher(url)) : ncode(url);
+  delete format.signatureCipher;
+  delete format.cipher;
 };
 
-
 /**
- * Applies `sig.decipher()` to all format URL's.
+ * Applies decipher and n parameter transforms to all format URL's.
  *
  * @param {Array.<Object>} formats
  * @param {string} html5player
  * @param {Object} options
  */
 exports.decipherFormats = async(formats, html5player, options) => {
   let decipheredFormats = {};
-  let tokens = await exports.getTokens(html5player, options);
+  let functions = await exports.getFunctions(html5player, options);
+  const decipherScript = functions.length ? new vm.Script(functions[0]) : null;
+  const nTransformScript = functions.length > 1 ? new vm.Script(functions[1]) : null;
   formats.forEach(format => {
-    let cipher = format.signatureCipher || format.cipher;
-    if (cipher) {
-      Object.assign(format, querystring.parse(cipher));
-      delete format.signatureCipher;
-      delete format.cipher;
-    }
-    const sig = tokens && format.s ? exports.decipher(tokens, format.s) : null;
-    exports.setDownloadURL(format, sig);
+    exports.setDownloadURL(format, decipherScript, nTransformScript);
     decipheredFormats[format.url] = format;
   });
   return decipheredFormats;