Skip to content

Commit

Permalink
fix: use backup endpoints when one fails
Browse files Browse the repository at this point in the history 
closes #774
  • Loading branch information
@fent
fent committed Nov 16, 2020
1 parent 8967bca commit 5a9a1f4
Show file tree
Hide file tree
Showing 9 changed files with 211 additions and 136 deletions.
183 changes: 130 additions & 53 deletions lib/info.js
View file
Expand Up @@ -11,10 +11,6 @@ const Cache = require('./cache');


const VIDEO_URL = 'https://www.youtube.com/watch?v=';
const EMBED_URL = 'https://www.youtube.com/embed/';
const VIDEO_EURL = 'https://youtube.googleapis.com/v/';
const INFO_HOST = 'www.youtube.com';
const INFO_PATH = '/get_video_info';


// Cached for storing basic/full info.
Expand All @@ -23,6 +19,12 @@ exports.cookieCache = new Cache(1000 * 60 * 60 * 24);
exports.watchPageCache = new Cache();


// Special error class used to determine if an error is unrecoverable,
// as in, ytdl-core should not try again to fetch the video metadata.
// In this case, the video is usually unavailable in some way.
class UnrecoverableError extends Error {}


/**
* Gets info from a video without getting additional formats.
*
Expand All @@ -32,37 +34,16 @@ exports.watchPageCache = new Cache();
*/
exports.getBasicInfo = async(id, options) => {
const retryOptions = Object.assign({}, miniget.defaultOptions, options.requestOptions);
let info = await retryFn(getJSONWatchPage, [id, options], retryOptions);
let player_response =
(info.player && info.player.args && info.player.args.player_response) ||
info.player_response || info.playerResponse;
player_response = parseJSON('watch.json `player_response`', player_response);
let html5player = info.player && info.player.assets && info.player.assets.js;

let playErr = utils.playError(player_response, ['ERROR']);
let privateErr = privateVideoError(player_response);
if (playErr || privateErr) {
throw playErr || privateErr;
}

let age_restricted = false;
if (!player_response || (!player_response.streamingData && !isRental(player_response))) {
// If the video page doesn't work, maybe because it has mature content.
// and requires an account logged in to view, try the embed page.
let [embedded_player_response, embedbody] = await retryFn(getEmbedPage, [id, options], retryOptions);
player_response = embedded_player_response;
html5player = html5player || getHTML5player(embedbody);
age_restricted = true;
}

if (!player_response || (!player_response.streamingData && !isRental(player_response))) {
player_response = await retryFn(getVideoInfoPage, [id, options, info], retryOptions);
}
const isValid = info =>
info.player_response && (info.player_response.streamingData || isRental(info.player_response));
let info = await pipeline([id, options], retryOptions, isValid, [
getJSONWatchPage,
getEmbedPage,
getVideoInfoPage,
]);

Object.assign(info, {
player_response,
html5player,
formats: parseFormats(player_response),
formats: parseFormats(info.player_response),
related_videos: extras.getRelatedVideos(info),
});

Expand All @@ -72,7 +53,6 @@ exports.getBasicInfo = async(id, options) => {
media: extras.getMedia(info),
likes: extras.getLikes(info),
dislikes: extras.getDislikes(info),
age_restricted,

// Give the standard link to the video.
video_url: VIDEO_URL + id,
Expand All @@ -85,12 +65,11 @@ exports.getBasicInfo = async(id, options) => {
return info;
};


const privateVideoError = player_response => {
let playability = player_response.playabilityStatus;
if (playability.status === 'LOGIN_REQUIRED' && playability.messages &&
playability.messages.filter(m => /This is a private video/.test(m)).length) {
return Error(playability.reason || (playability.messages && playability.messages[0]));
return new UnrecoverableError(playability.reason || (playability.messages && playability.messages[0]));
} else {
return null;
}
Expand Down Expand Up @@ -123,17 +102,86 @@ const getIdentityToken = (id, options, key, throwIfNotFound) =>
let page = await getWatchPage(id, options);
let match = page.match(/(["'])ID_TOKEN\1[:,]\s?"([^"]+)"/);
if (!match && throwIfNotFound) {
throw Error('Cookie header used in request, but unable to find YouTube identity token');
throw new UnrecoverableError('Cookie header used in request, but unable to find YouTube identity token');
}
return match && match[2];
});


const retryFn = async(fn, args, options) => {
/**
* Goes through each endpoint in the pipeline, retrying on failure if the error is recoverable.
* If unable to succeed with one endpoint, moves onto the next one.
*
* @param {Array.<Object>} args
* @param {Object} retryOptions
* @param {Function} isValid
* @param {Array.<Function>} endpoints
* @returns {[Object, Object, Object]}
*/
const pipeline = async(args, retryOptions, isValid, endpoints) => {
let info;
for (let func of endpoints) {
try {
const newInfo = await retryFunc(func, args, retryOptions);
if (newInfo.player_response) {
if (newInfo.player_response.videoDetails) {
newInfo.player_response.videoDetails = assign(
info && info.player_response && info.player_response.videoDetails,
newInfo.player_response.videoDetails);
}
newInfo.player_response = assign(info && info.player_response, newInfo.player_response);
}
info = assign(info, newInfo);
if (isValid(info)) {
break;
}
} catch (err) {
if (err instanceof UnrecoverableError || func === endpoints[endpoints.length - 1]) {
throw err;
}
// Unable to find video metadata... so try next endpoint.
}
}
return info;
};


/**
* Like Object.assign(), but ignores `null` and `undefined` from `source`.
*
* @param {Object} target
* @param {Object} source
* @returns {Object}
*/
const assign = (target, source) => {
if (!target || !source) { return target || source; }
for (let [key, value] of Object.entries(source)) {
if (value !== null && value !== undefined) {
target[key] = value;
}
}
return target;
};


/**
* Given a function, calls it with `args` until it's successful,
* or until it encounters an unrecoverable error.
* Currently, any error from miniget is considered unrecoverable. Errors such as
* too many redirects, invalid URL, status code 404, status code 502.
*
* @param {Function} func
* @param {Array.<Object>} args
* @param {Object} options
* @param {number} options.maxRetries
* @param {Object} options.backoff
* @param {number} options.backoff.inc
*/
const retryFunc = async(func, args, options) => {
let currentTry = 0, result;
while (currentTry <= options.maxRetries) {
try {
result = await fn(...args);
result = await func(...args);
break;
} catch (err) {
if (err instanceof miniget.MinigetError || currentTry >= options.maxRetries) {
Expand Down Expand Up @@ -192,26 +240,55 @@ const getJSONWatchPage = async(id, options) => {
throw Error('Unable to retrieve video metadata');
}
let info = parsedBody.reduce((part, curr) => Object.assign(curr, part), {});
let player_response =
(info.player && info.player.args && info.player.args.player_response) ||
info.player_response || info.playerResponse;
info.player_response = parseJSON('watch.json `player_response`', player_response);
info.player_response.videoDetails = Object.assign({}, info.player_response.videoDetails, { age_restricted: false });
info.html5player = info.player && info.player.assets && info.player.assets.js;

let playErr = utils.playError(info.player_response, ['ERROR'], UnrecoverableError);
let privateErr = privateVideoError(info.player_response);
if (playErr || privateErr) {
throw playErr || privateErr;
}

return info;
};


/**
* If the video page doesn't work, maybe because it has mature content.
* and requires an account logged in to view, try the embed page.
*
* @param {string} id
* @param {Object} options
* @returns {string}
*/
const EMBED_URL = 'https://www.youtube.com/embed/';
const getEmbedURL = (id, options) => `${EMBED_URL + id}?hl=${options.lang || 'en'}`;
const getEmbedPage = async(id, options) => {
const embedUrl = getEmbedURL(id, options);
let body = await miniget(embedUrl, options.requestOptions).text();
let jsonStr = utils.between(body, /(['"])PLAYER_(CONFIG|VARS)\1:\s?/, '</script>');
let config;
if (!jsonStr) {
throw Error('Could not find player config');
}
config = parseJSON('embed config', utils.cutAfterJSON(jsonStr));
let player_response = (config.args && (config.args.player_response || config.args.embedded_player_response)) ||
let config = parseJSON('embed config', utils.cutAfterJSON(jsonStr));
let info = config.args || config;
let player_response = (info && (info.player_response || info.embedded_player_response)) ||
config.embedded_player_response;
return [parseJSON('embed `player_response`', player_response), body];
info.player_response = parseJSON('embed `player_response`', player_response);
info.player_response.videoDetails = Object.assign({}, info.player_response.videoDetails, { age_restricted: true });
info.html5player = getHTML5player(body);
return info;
};


const getVideoInfoPage = async(id, options, info) => {
const INFO_HOST = 'www.youtube.com';
const INFO_PATH = '/get_video_info';
const VIDEO_EURL = 'https://youtube.googleapis.com/v/';
const getVideoInfoPage = async(id, options) => {
const url = urllib.format({
protocol: 'https',
host: INFO_HOST,
Expand All @@ -222,12 +299,12 @@ const getVideoInfoPage = async(id, options, info) => {
ps: 'default',
gl: 'US',
hl: options.lang || 'en',
sts: info.sts,
},
});
let morebody = await miniget(url, options.requestOptions).text();
let moreinfo = querystring.parse(morebody);
return parseJSON('get_video_info `player_response`', moreinfo.player_response || info.playerResponse);
let body = await miniget(url, options.requestOptions).text();
let info = querystring.parse(body);
info.player_response = parseJSON('get_video_info `player_response`', info.player_response);
return info;
};


Expand Down Expand Up @@ -352,17 +429,17 @@ const getM3U8 = async(url, options) => {

// Cache get info functions.
// In case a user wants to get a video's info before downloading.
for (let fnName of ['getBasicInfo', 'getInfo']) {
for (let funcName of ['getBasicInfo', 'getInfo']) {
/**
* @param {string} link
* @param {Object} options
* @returns {Promise<Object>}
*/
const fn = exports[fnName];
exports[fnName] = (link, options = {}) => {
const func = exports[funcName];
exports[funcName] = (link, options = {}) => {
let id = urlUtils.getVideoID(link);
const key = [fnName, id, options.lang].join('-');
return exports.cache.getOrSet(key, () => fn(id, options));
const key = [funcName, id, options.lang].join('-');
return exports.cache.getOrSet(key, () => func(id, options));
};
}

Expand Down
5 changes: 3 additions & 2 deletions lib/utils.js
View file
Expand Up @@ -104,12 +104,13 @@ exports.cutAfterJSON = mixedJson => {
*
* @param {Object} player_response
* @param {Array.<string>} statuses
* @param {Error} ErrorType
* @returns {!Error}
*/
exports.playError = (player_response, statuses) => {
exports.playError = (player_response, statuses, ErrorType = Error) => {
let playability = player_response.playabilityStatus;
if (playability && statuses.includes(playability.status)) {
return Error(playability.reason || (playability.messages && playability.messages[0]));
return new ErrorType(playability.reason || (playability.messages && playability.messages[0]));
}
return null;
};
19 changes: 7 additions & 12 deletions test/files/refresh.js
View file
Expand Up @@ -64,19 +64,9 @@ const videos = [
{
id: '_HSylqgVYQI',
type: 'regular',
keep: ['video.flv', 'watch-reload-now.json', 'watch-reload-now-2.json'],
keep: ['video.flv', 'watch-reload-now-2.json'],
saveInfo: true,
transform: [
{
page: 'watch.json',
saveAs: 'bad-config',
fn: body => body.replace('[', '{]'),
},
{
page: 'watch.json',
saveAs: 'bad-player-response',
fn: body => body.replace('"(player(?:_r|R)esponse)":"{', '"$1":"'),
},
{
page: 'watch.json',
saveAs: 'no-extras',
Expand Down Expand Up @@ -126,7 +116,7 @@ const videos = [
id: 'LuZu9N53Vd0',
type: 'age-restricted',
saveInfo: true,
keep: ['embed-player-vars.html'],
keep: ['embed-player-vars.html', 'watch-backup.html', 'watch-reload-now.json'],
transform: [
{
page: 'embed.html',
Expand All @@ -148,6 +138,11 @@ const videos = [
saveAs: 'no-player-response',
fn: body => body.replace(/player_response/g, 'no'),
},
{
page: 'watch.json',
saveAs: 'bad-config',
fn: body => body.replace('[', '{]'),
},
],
},
{
Expand Down
2 changes: 2 additions & 0 deletions test/files/videos/age-restricted/watch-backup.html
View file
@@ -0,0 +1,2 @@
<!DOCTYPE html> <html lang="en" dir="ltr" data-cast-api-enabled="true">
<script src="/s/player/c926146c/player_ias.vflset/en_US/base.js" name="player_ias/base" ></script>

0 comments on commit 5a9a1f4

Please sign in to comment.