Skip to content

Commit

Permalink
feat: support download tarball from other_urls query params (#269)
Browse files Browse the repository at this point in the history 
relate to cnpm/cnpmjs.org#1344
  • Loading branch information
@fengmk2
fengmk2 authored Jun 5, 2018
1 parent 9bbeb01 commit b486e06
Show file tree
Hide file tree
Showing 4 changed files with 65 additions and 22 deletions.
33 changes: 18 additions & 15 deletions lib/download/npm.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,30 +160,34 @@ function* download(pkg, options) {

let lastErr;
let count = 0;
const tarballUrls = utils.parseTarballUrls(pkg.dist.tarball);
let tarballUrlIndex = 0;
let tarballUrl;
while (count < 3) {
tarballUrl = tarballUrls[tarballUrlIndex++];
if (!tarballUrl) {
tarballUrlIndex = 0;
tarballUrl = tarballUrls[tarballUrlIndex];
}
try {
const stream = yield getTarballStream(pkg, options);
const stream = yield getTarballStream(tarballUrl, pkg, options);
let useTarFormat = false;
if (count === 1 && lastErr && lastErr.code === 'Z_DATA_ERROR') {
options.console.warn(`[${pkg.name}@${pkg.version}] tgz format ungzip error, try to use tar format`);
options.console.warn(`[${pkg.name}@${pkg.version}] format ungzip error, try to use tar format`);
useTarFormat = true;
}
yield checkShasumAndUngzip(ungzipDir, stream, pkg, useTarFormat);
lastErr = null;
break;
} catch (err) {
lastErr = err;
// retry download on shasum error or server 50x error
if (err.name === 'ShasumNotMatchError' || err.code === 'Z_DATA_ERROR' || err.status >= 500) {
count++;
options.console.warn(`[${pkg.name}@${pkg.version}] download %s: %s, fail count: %s`,
err.name, err.message, count);
// sleep for a while to wait for server become normal
if (count < 3) {
yield utils.sleep(count * 500);
}
} else {
break;
count++;
options.console.warn(`[${pkg.name}@${pkg.version}] download %s: %s, fail count: %s`,
err.name, err.message, count);
// retry download on any error
// sleep for a while to wait for server become normal
if (count < 3) {
yield utils.sleep(count * 500);
}
}
}
Expand Down Expand Up @@ -269,8 +273,7 @@ function* download(pkg, options) {
};
}

function* getTarballStream(pkg, options) {
let tarballUrl = pkg.dist.tarball;
function* getTarballStream(tarballUrl, pkg, options) {
let formatRedirectUrl;
if (options.formatNpmTarbalUrl) {
tarballUrl = options.formatNpmTarbalUrl(tarballUrl);
Expand Down
15 changes: 15 additions & 0 deletions lib/utils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ const debug = require('debug')('npminstall:utils');
const fs = require('mz/fs');
const path = require('path');
const cp = require('child_process');
const urlparse = require('url').parse;
const querystring = require('querystring');

require('./patch_buffer');
const tar = require('tar');
Expand Down Expand Up @@ -138,6 +140,19 @@ exports.formatPackageUrl = function formatUrl(registry, name) {
return url.format(parsed);
};

exports.parseTarballUrls = function parseTarballUrls(tarball) {
const urls = [ tarball ];
const parsed = urlparse(tarball);
const query = parsed.query && querystring.parse(parsed.query);
if (query && query.other_urls) {
const otherUrls = query.other_urls.split(',');
for (const url of otherUrls) {
urls.push(url);
}
}
return urls;
};

/*
* Runs an npm script.
*/
Expand Down
14 changes: 7 additions & 7 deletions test/download.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,9 @@ describe('test/download.test.js', () => {
it('should throw error when status === 404', function* () {
const request = urllib.request;
mm(urllib, 'request', function* (url, options) {
if (url.endsWith('.tgz')) {
mm.restore();
}
// if (url.endsWith('.tgz')) {
// mm.restore();
// }
const result = yield request.call(urllib, url, options);
if (url.endsWith('.tgz')) {
result.status = 404;
Expand All @@ -53,9 +53,9 @@ describe('test/download.test.js', () => {
it('should throw error when status === 206', function* () {
const request = urllib.request;
mm(urllib, 'request', function* (url, options) {
if (url.endsWith('.tgz')) {
mm.restore();
}
// if (url.endsWith('.tgz')) {
// mm.restore();
// }
const result = yield request.call(urllib, url, options);
if (url.endsWith('.tgz')) {
result.status = 206;
Expand All @@ -81,7 +81,7 @@ describe('test/download.test.js', () => {
describe('mock tarball error', () => {
it('should throw sha1 error', function* () {
this.timeout = 15000;
const registry = process.env.npm_registry || 'https://registry.cnpmjs.org';
const registry = process.env.npm_registry || 'https://registry.npm.taobao.org';
const res = yield urllib.request(`${registry}/pedding`, { dataType: 'json', timeout: 10000 });
const pkg = res.data;
pkg.versions['1.0.0'].dist.shasum = '00098d60307b4ef7240c3d693cb20a9473c111';
Expand Down
25 changes: 25 additions & 0 deletions test/utils.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,4 +102,29 @@ describe('test/utils.test.js', () => {
]) === '1.0.2');
});
});

describe('parseTarballUrls()', () => {
it('should return one url', () => {
assert.deepEqual(utils.parseTarballUrls('https://registry.npmjs.org/node/-/node-10.3.0.tgz'), [
'https://registry.npmjs.org/node/-/node-10.3.0.tgz',
]);
assert.deepEqual(utils.parseTarballUrls('https://registry.npmjs.org/node/-/node-10.3.0.tgz?other_urls='), [
'https://registry.npmjs.org/node/-/node-10.3.0.tgz?other_urls=',
]);
});

it('should return multi urls', () => {
assert.deepEqual(utils.parseTarballUrls('http://foo-us1.oss.com/@cnpmtest/download-test-module/-/@cnpmtest/download-test-module-1.0.0.tgz?other_urls=http%3A%2F%2Fdefault.oss.com%2F%40cnpmtest%2Fdownload-test-module%2F-%2F%40cnpmtest%2Fdownload-test-module-1.0.0.tgz%2Chttp%3A%2F%2Fbackup.oss.com%2F%40cnpmtest%2Fdownload-test-module%2F-%2F%40cnpmtest%2Fdownload-test-module-1.0.0.tgz'), [
'http://foo-us1.oss.com/@cnpmtest/download-test-module/-/@cnpmtest/download-test-module-1.0.0.tgz?other_urls=http%3A%2F%2Fdefault.oss.com%2F%40cnpmtest%2Fdownload-test-module%2F-%2F%40cnpmtest%2Fdownload-test-module-1.0.0.tgz%2Chttp%3A%2F%2Fbackup.oss.com%2F%40cnpmtest%2Fdownload-test-module%2F-%2F%40cnpmtest%2Fdownload-test-module-1.0.0.tgz',
'http://default.oss.com/@cnpmtest/download-test-module/-/@cnpmtest/download-test-module-1.0.0.tgz',
'http://backup.oss.com/@cnpmtest/download-test-module/-/@cnpmtest/download-test-module-1.0.0.tgz',
]);

assert.deepEqual(utils.parseTarballUrls('http://foo-us1.oss.com/@cnpmtest/download-test-module/-/@cnpmtest/download-test-module-1.0.0.tgz?foo=bar&other_urls=http%3A%2F%2Fdefault.oss.com%2F%40cnpmtest%2Fdownload-test-module%2F-%2F%40cnpmtest%2Fdownload-test-module-1.0.0.tgz%2Chttp%3A%2F%2Fbackup.oss.com%2F%40cnpmtest%2Fdownload-test-module%2F-%2F%40cnpmtest%2Fdownload-test-module-1.0.0.tgz'), [
'http://foo-us1.oss.com/@cnpmtest/download-test-module/-/@cnpmtest/download-test-module-1.0.0.tgz?foo=bar&other_urls=http%3A%2F%2Fdefault.oss.com%2F%40cnpmtest%2Fdownload-test-module%2F-%2F%40cnpmtest%2Fdownload-test-module-1.0.0.tgz%2Chttp%3A%2F%2Fbackup.oss.com%2F%40cnpmtest%2Fdownload-test-module%2F-%2F%40cnpmtest%2Fdownload-test-module-1.0.0.tgz',
'http://default.oss.com/@cnpmtest/download-test-module/-/@cnpmtest/download-test-module-1.0.0.tgz',
'http://backup.oss.com/@cnpmtest/download-test-module/-/@cnpmtest/download-test-module-1.0.0.tgz',
]);
});
});
});

0 comments on commit b486e06

Please sign in to comment.