From e1e1af14f6ae5c6489de9f147921acc8be241d31 Mon Sep 17 00:00:00 2001 From: fent <933490+fent@users.noreply.github.com> Date: Sun, 15 Nov 2020 23:24:58 -0700 Subject: [PATCH] fix: retry endpoint on status code 500 previously, it would give up on that endpoint and move on to the next one. most of the time, a status code 500, a server side error, is a one-off. --- lib/info.js | 2 +- package-lock.json | 6 +- package.json | 2 +- test/files/refresh.js | 7 +- .../videos/regular/watch-reload-now.json | 1 + test/info-test.js | 79 +++++++++++++++---- 6 files changed, 72 insertions(+), 25 deletions(-) create mode 100644 test/files/videos/regular/watch-reload-now.json diff --git a/lib/info.js b/lib/info.js index 0d9a36d0..fe9376fe 100644 --- a/lib/info.js +++ b/lib/info.js @@ -184,7 +184,7 @@ const retryFunc = async(func, args, options) => { result = await func(...args); break; } catch (err) { - if (err instanceof miniget.MinigetError || currentTry >= options.maxRetries) { + if ((err instanceof miniget.MinigetError && err.statusCode < 500) || currentTry >= options.maxRetries) { throw err; } let wait = Math.min(++currentTry * options.backoff.inc, options.backoff.max); diff --git a/package-lock.json b/package-lock.json index 97462f42..ed609af6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2699,9 +2699,9 @@ "dev": true }, "miniget": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/miniget/-/miniget-3.0.0.tgz", - "integrity": "sha512-s+gfqBEEGyoCf+1OG/6OKJDLXUubASlcVoBeXpftT0uyWha2G1qnxE63XLoD8F4Wq+KsTK70raxPoFkOmrzW5g==" + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/miniget/-/miniget-3.1.0.tgz", + "integrity": "sha512-cf18w/CV7RIgGbsi8jg5gmrrrgziLPk3KzC3bWAZW58Rrunlii75eSKPZVAoG7XGA86AYitzY+8Z0j4pafydmw==" }, "minimatch": { "version": "3.0.4", diff --git a/package.json b/package.json index 2e82e216..1578aa56 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "dependencies": { "html-entities": "^1.3.1", "m3u8stream": "^0.8.1", - "miniget": "^3.0.0", + "miniget": "^3.1.0", "sax": "^1.1.3" }, "devDependencies": { diff --git a/test/files/refresh.js b/test/files/refresh.js index f4eb1d64..b25bd5e5 100644 --- a/test/files/refresh.js +++ b/test/files/refresh.js @@ -64,16 +64,13 @@ const videos = [ { id: '_HSylqgVYQI', type: 'regular', - keep: ['video.flv', 'watch-reload-now-2.json'], + keep: ['video.flv', 'watch-reload-now.json', 'watch-reload-now-2.json'], saveInfo: true, transform: [ { page: 'watch.json', saveAs: 'no-extras', - fn: body => { - body = body.replace('playerMicroformatRenderer', ''); - return body; - }, + fn: body => body.replace('playerMicroformatRenderer', ''), }, { page: 'watch.html', diff --git a/test/files/videos/regular/watch-reload-now.json b/test/files/videos/regular/watch-reload-now.json new file mode 100644 index 00000000..e4c3251a --- /dev/null +++ b/test/files/videos/regular/watch-reload-now.json @@ -0,0 +1 @@ +{"reload":"now"} diff --git a/test/info-test.js b/test/info-test.js index ea252f54..77d926be 100644 --- a/test/info-test.js +++ b/test/info-test.js @@ -345,7 +345,6 @@ describe('ytdl.getInfo()', () => { const id = 'LuZu9N53Vd0'; const scope = nock(id, 'age-restricted', { watchJson: [true, 200, 'bad-config'], - watchHtml: false, }); let info = await ytdl.getInfo(id); scope.done(); @@ -371,29 +370,79 @@ describe('ytdl.getInfo()', () => { }); describe('When watch page gives back `{"reload":"now"}`', () => { - it('Uses backup embed.html page', async() => { - const id = 'LuZu9N53Vd0'; - const scope = nock(id, 'age-restricted', { + it('Retries the request', async() => { + const id = '_HSylqgVYQI'; + const scope1 = nock(id, 'regular', { watchJson: [true, 200, 'reload-now'], }); - const scope2 = nock(id, 'age-restricted', { - watchJson: [true, 200, 'reload-now'], - embed: false, - get_video_info: false, + const scope2 = nock(id, 'regular', { + watchHtml: false, player: false, }); - let info = await ytdl.getInfo(id, { - requestOptions: { - maxRetries: 1, - backoff: { inc: 0 }, - }, + let info = await ytdl.getInfo(id, { requestOptions: { maxRetries: 1 } }); + scope1.done(); + scope2.done(); + assert.ok(info.formats.length); + assert.ok(info.formats[0].url); + }); + + describe('Too many times', () => { + it('Uses backup embed.html page', async() => { + const id = 'LuZu9N53Vd0'; + const scope = nock(id, 'age-restricted', { + watchJson: [true, 200, 'reload-now'], + }); + const scope2 = nock(id, 'age-restricted', { + watchJson: [true, 200, 'reload-now'], + embed: false, + get_video_info: false, + player: false, + }); + let info = await ytdl.getInfo(id, { + requestOptions: { + maxRetries: 1, + backoff: { inc: 0 }, + }, + }); + scope.done(); + scope2.done(); + assert.ok(info.html5player); + assert.ok(info.formats.length); + assert.ok(info.formats[0].url); }); - scope.done(); + }); + }); + + describe('When an endpoint gives back a 500 server error', () => { + it('Retries the request', async() => { + const id = '_HSylqgVYQI'; + const scope1 = nock(id, 'regular', { + watchJson: [true, 502], + }); + const scope2 = nock(id, 'regular', { + watchHtml: false, + player: false, + }); + let info = await ytdl.getInfo(id, { requestOptions: { maxRetries: 1 } }); + scope1.done(); scope2.done(); - assert.ok(info.html5player); assert.ok(info.formats.length); assert.ok(info.formats[0].url); }); + + describe('Too many times', () => { + it('Uses the next endpoint as backup', async() => { + const id = 'LuZu9N53Vd0'; + const scope = nock(id, 'age-restricted', { + watchJson: [true, 502], + }); + let info = await ytdl.getInfo(id); + scope.done(); + assert.ok(info.html5player); + assert.ok(info.formats.length); + assert.ok(info.formats[0].url); + }); + }); }); });