From 89f48cc93f205d66d85e0175129f4379b5bc69ec Mon Sep 17 00:00:00 2001 From: fengmk2 Date: Thu, 19 Jul 2018 13:46:32 +0800 Subject: [PATCH] feat: support high speed store for tarball download (#276) --- .gitignore | 1 + bin/install.js | 7 +++++ lib/download/npm.js | 29 +++++++++++++++---- lib/local_install.js | 7 +++-- package.json | 5 ++-- test/all.js | 6 ++-- test/fixtures/high-speed-store/package.json | 7 +++++ test/fixtures/high-speed-store/store.js | 25 ++++++++++++++++ test/high-speed-store.test.js | 32 +++++++++++++++++++++ test/tarball-url-mapping.test.js | 2 +- test/use-exists-version.test.js | 2 +- 11 files changed, 110 insertions(+), 13 deletions(-) create mode 100644 test/fixtures/high-speed-store/package.json create mode 100644 test/fixtures/high-speed-store/store.js create mode 100644 test/high-speed-store.test.js diff --git a/.gitignore b/.gitignore index bbc79c58..0e9e90d5 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,4 @@ test/fixtures/css-loader-example2/node_modules test/fixtures/css-loader-example2/webpack*.json !test/fixtures/warn-node-modules-exists/node_modules !test/fixtures/ignore-warn-node-modules-exists/node_modules +test/fixtures/high-speed-store/tmp diff --git a/bin/install.js b/bin/install.js index 7c19daf0..57d249a9 100755 --- a/bin/install.js +++ b/bin/install.js @@ -27,6 +27,8 @@ const argv = parseArgs(orignalArgv, { // {"http://a.com":"http://b.com"} 'tarball-url-mapping', 'proxy', + // --high-speed-store=filepath + 'high-speed-store', ], boolean: [ 'version', @@ -106,6 +108,7 @@ Options: --cache-strict: use disk cache even on production env. --fix-bug-versions: auto fix bug version of package. --prune: prune unnecessary files from ./node_modules, such as markdown, typescript source files, and so on. + --high-speed-store: specify high speed store script to cache tgz files, and so on. Should export '* getStream(url)' function. ` ); process.exit(0); @@ -248,6 +251,10 @@ co(function* () { }; } + if (argv['high-speed-store']) { + config.highSpeedStore = require(argv['high-speed-store']); + } + // -g install to npm's global prefix if (argv.global) { // support custom prefix for global install diff --git a/lib/download/npm.js b/lib/download/npm.js index 369eda37..2fbaa0a4 100644 --- a/lib/download/npm.js +++ b/lib/download/npm.js @@ -308,6 +308,23 @@ function* getTarballStream(tarballUrl, pkg, options) { }; } + // high speed store + if (options.highSpeedStore) { + try { + const stream = yield options.highSpeedStore.getStream(tarballUrl); + if (stream) { + // record size + stream.on('data', chunk => { + options.totalTarballSize += chunk.length; + }); + return stream; + } + } catch (err) { + options.console.warn('[npminstall:download:npm] highSpeedStore.get %s error: %s', tarballUrl, err); + options.console.warn(err.stack); + } + } + if (!options.cacheDir || utils.isSudo()) { // sudo don't touch the cacheDir // production mode @@ -344,11 +361,13 @@ function* getTarballStream(tarballUrl, pkg, options) { let exists = yield fs.exists(tarballFile); if (!exists) { // try to remove expired tmp dirs - // last year - const lastYearTmpDir = path.join(options.cacheDir, '.tmp', moment().subtract(1, 'years').format('YYYY')); - // last month - const lastMonthTmpDir = path.join(options.cacheDir, '.tmp', moment().subtract(1, 'months').format('YYYY/MM')); - const dirs = [ lastYearTmpDir, lastMonthTmpDir ]; + const dirs = []; + for (let i = 1; i <= 3; i++) { + // year + dirs.push(path.join(options.cacheDir, '.tmp', moment().subtract(i, 'years').format('YYYY'))); + // month + dirs.push(path.join(options.cacheDir, '.tmp', moment().subtract(i, 'months').format('YYYY/MM'))); + } for (const dir of dirs) { try { yield utils.rimraf(dir); diff --git a/lib/local_install.js b/lib/local_install.js index b7e5063d..0da3935d 100644 --- a/lib/local_install.js +++ b/lib/local_install.js @@ -136,6 +136,7 @@ function* _install(options) { } yield parallel(tasks, 10); + options.downloadFinished = Date.now(); options.spinner && options.spinner.succeed(`Installed ${tasks.length} packages`); // link every packages' latest version to target directory @@ -469,15 +470,17 @@ function recordPackageVersions(options) { function finishInstall(options) { const totalUse = Date.now() - options.start; + const downloadUse = options.downloadFinished - options.start; const totalSize = options.totalTarballSize + options.totalJSONSize; - const avgSpeed = totalSize / totalUse * 1000; + const avgSpeed = totalSize / downloadUse * 1000; const logArguments = [ - chalk[options.detail ? 'green' : 'white']('All packages installed (%s%s%s%sused %s, speed %s/s, json %s(%s), tarball %s%s)'), + chalk[options.detail ? 'green' : 'white']('All packages installed (%s%s%s%sused %s(network %s), speed %s/s, json %s(%s), tarball %s%s)'), options.registryPackages ? `${options.registryPackages} packages installed from npm registry, ` : '', options.remotePackages ? `${options.remotePackages} packages installed from remote url, ` : '', options.localPackages ? `${options.localPackages} packages installed from local file, ` : '', options.gitPackages ? `${options.gitPackages} packages installed from git, ` : '', ms(totalUse), + ms(downloadUse), bytes(avgSpeed), options.totalJSONCount, bytes(options.totalJSONSize), diff --git a/package.json b/package.json index 09381e7a..7bff5ede 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,8 @@ "mkdirp": "^0.5.1", "moment": "^2.18.1", "ms": "^2.0.0", - "mz": "^2.6.0", + "mz": "^2.7.0", + "mz-modules": "^2.1.0", "node-gyp": "^3.7.0", "node-homedir": "^1.0.0", "normalize-git-url": "^3.0.2", @@ -50,7 +51,7 @@ "semver": "^5.3.0", "tar": "^4.0.1", "urllib": "^2.24.0", - "utility": "^1.12.0", + "utility": "^1.14.0", "uuid": "^3.0.1" }, "devDependencies": { diff --git a/test/all.js b/test/all.js index 7f4ff6f4..9431c540 100644 --- a/test/all.js +++ b/test/all.js @@ -4,11 +4,10 @@ const co = require('co'); const path = require('path'); const rimraf = require('rimraf'); const semver = require('semver'); -const npminstall = require('./npminstall'); const spawn = require('child_process').spawn; +const npminstall = require('./npminstall'); const names = [ - 'strongloop', 'express', 'koa', 'browserify', 'pm2', 'grunt-cli', @@ -32,6 +31,9 @@ const semvers = { '>= 6': [ 'egg', ], + '<= 8': [ + 'strongloop', + ], }; for (const version in semvers) { diff --git a/test/fixtures/high-speed-store/package.json b/test/fixtures/high-speed-store/package.json new file mode 100644 index 00000000..23ad8dc8 --- /dev/null +++ b/test/fixtures/high-speed-store/package.json @@ -0,0 +1,7 @@ +{ + "name": "high-speed-store", + "devDependencies": { + "@types/react": "15.0.4", + "@types/react-dom": "0.14.18" + } +} diff --git a/test/fixtures/high-speed-store/store.js b/test/fixtures/high-speed-store/store.js new file mode 100644 index 00000000..7f9c772a --- /dev/null +++ b/test/fixtures/high-speed-store/store.js @@ -0,0 +1,25 @@ +'use strict'; + +const co = require('co'); +const path = require('path'); +const fs = require('mz/fs'); +const mkdirp = require('mz-modules/mkdirp'); +const utility = require('utility'); +const urllib = require('urllib'); + +exports.getStream = function* (url) { + const dir = path.join(__dirname, 'tmp'); + yield mkdirp(dir); + const file = path.join(dir, utility.md5(url) + path.extname(url)); + if (yield fs.exists(file)) { + return fs.createReadStream(file); + } + + const writeStream = fs.createWriteStream(file); + yield urllib.request(url, { + writeStream, + timeout: 10000, + followRedirect: true, + }); + return fs.createReadStream(file); +}; diff --git a/test/high-speed-store.test.js b/test/high-speed-store.test.js new file mode 100644 index 00000000..24922f35 --- /dev/null +++ b/test/high-speed-store.test.js @@ -0,0 +1,32 @@ +'use strict'; + +const coffee = require('coffee'); +const rimraf = require('rimraf'); +const path = require('path'); +const assert = require('assert'); +const fs = require('fs'); + +describe('test/high-speed-store.test.js', () => { + const cwd = path.join(__dirname, 'fixtures', 'high-speed-store'); + const storeScript = path.join(cwd, 'store.js'); + const bin = path.join(__dirname, '../bin/install.js'); + + function cleanup() { + rimraf.sync(path.join(cwd, 'node_modules')); + rimraf.sync(path.join(cwd, 'tmp')); + } + + beforeEach(cleanup); + afterEach(cleanup); + + it('should get tarball stream from store', function* () { + yield coffee.fork(bin, [ '-d', `--high-speed-store=${storeScript}` ], { cwd }) + .debug() + .expect('code', 0) + .expect('stdout', /All packages installed/) + .end(); + const pkg = require(path.join(cwd, 'node_modules/@types/react-dom/node_modules/@types/react/package.json')); + assert(pkg.version === '15.0.4'); + assert(fs.readdirSync(path.join(cwd, 'tmp')).length === 2); + }); +}); diff --git a/test/tarball-url-mapping.test.js b/test/tarball-url-mapping.test.js index b770295c..15dc427f 100644 --- a/test/tarball-url-mapping.test.js +++ b/test/tarball-url-mapping.test.js @@ -5,7 +5,7 @@ const rimraf = require('rimraf'); const mkdirp = require('mkdirp'); const path = require('path'); -describe('tarball-url-mapping.test.js', () => { +describe('test/tarball-url-mapping.test.js', () => { const tmp = path.join(__dirname, 'fixtures', 'tmp'); const bin = path.join(__dirname, '../bin/install.js'); diff --git a/test/use-exists-version.test.js b/test/use-exists-version.test.js index 15a7aa76..54dd17fa 100644 --- a/test/use-exists-version.test.js +++ b/test/use-exists-version.test.js @@ -5,7 +5,7 @@ const rimraf = require('rimraf'); const path = require('path'); const assert = require('assert'); -describe('use-exists-version.test.js', () => { +describe('test/use-exists-version.test.js', () => { const tmp = path.join(__dirname, 'fixtures', 'try-to-use-one-version'); const bin = path.join(__dirname, '../bin/install.js');