diff --git a/lib/fetchers/registry/check-warning-header.js b/lib/fetchers/registry/check-warning-header.js deleted file mode 100644 index b17a233..0000000 --- a/lib/fetchers/registry/check-warning-header.js +++ /dev/null @@ -1,39 +0,0 @@ -'use strict' - -const LRU = require('lru-cache') - -const WARNING_REGEXP = /^\s*(\d{3})\s+(\S+)\s+"(.*)"\s+"([^"]+)"/ -const BAD_HOSTS = new LRU({ max: 50 }) - -module.exports = checkWarnings -function checkWarnings (res, registry, opts) { - if (res.headers.has('warning') && !BAD_HOSTS.has(registry)) { - const warnings = {} - res.headers.raw()['warning'].forEach(w => { - const match = w.match(WARNING_REGEXP) - if (match) { - warnings[match[1]] = { - code: match[1], - host: match[2], - message: match[3], - date: new Date(match[4]) - } - } - }) - BAD_HOSTS.set(registry, true) - if (warnings['199']) { - if (warnings['199'].message.match(/ENOTFOUND/)) { - opts.log.warn('registry', `Using stale data from ${registry} because the host is inaccessible -- are you offline?`) - } else { - opts.log.warn('registry', `Unexpected warning for ${registry}: ${warnings['199'].message}`) - } - } - if (warnings['111']) { - // 111 Revalidation failed -- we're using stale data - opts.log.warn( - 'registry', - `Using stale package data from ${registry} due to a request error during revalidation.` - ) - } - } -} diff --git a/lib/fetchers/registry/fetch.js b/lib/fetchers/registry/fetch.js deleted file mode 100644 index adb959e..0000000 --- a/lib/fetchers/registry/fetch.js +++ /dev/null @@ -1,111 +0,0 @@ -'use strict' - -const BB = require('bluebird') -const Buffer = require('safe-buffer').Buffer - -const checkWarnings = require('./check-warning-header') -const fetch = require('make-fetch-happen') -const optCheck = require('../../util/opt-check.js') -const registryKey = require('./registry-key') -const url = require('url') - -module.exports = regFetch -function regFetch (uri, registry, opts) { - opts = optCheck(opts) - const startTime = Date.now() - return fetch(uri, { - agent: opts.agent, - algorithms: opts.algorithms, - cache: getCacheMode(opts), - cacheManager: opts.cache, - ca: opts.ca, - cert: opts.cert, - headers: getHeaders(uri, registry, opts), - integrity: opts.integrity, - key: opts.key, - localAddress: opts.localAddress, - maxSockets: opts.maxSockets, - memoize: opts.memoize, - noProxy: opts.noProxy, - Promise: BB, - proxy: opts.proxy, - referer: opts.refer, - retry: opts.retry, - strictSSL: !!opts.strictSSL, - timeout: opts.timeout, - uid: opts.uid, - gid: opts.gid - }).then(res => { - if (res.headers.has('npm-notice') && !res.headers.has('x-local-cache')) { - opts.log.warn('notice', res.headers.get('npm-notice')) - } - checkWarnings(res, registry, opts) - if (res.status >= 400) { - const err = new Error(`${res.status} ${res.statusText}: ${ - opts.spec ? opts.spec : uri - }`) - err.code = `E${res.status}` - err.uri = uri - err.response = res - err.spec = opts.spec - logRequest(uri, res, startTime, opts) - throw err - } else { - res.body.on('end', () => logRequest(uri, res, startTime, opts)) - return res - } - }) -} - -function logRequest (uri, res, startTime, opts) { - const elapsedTime = Date.now() - startTime - const attempt = res.headers.get('x-fetch-attempts') - const attemptStr = attempt && attempt > 1 ? ` attempt #${attempt}` : '' - const cacheStr = res.headers.get('x-local-cache') ? ' (from cache)' : '' - opts.log.http( - 'fetch', - `GET ${res.status} ${uri} ${elapsedTime}ms${attemptStr}${cacheStr}` - ) -} - -function getCacheMode (opts) { - return opts.offline - ? 'only-if-cached' - : opts.preferOffline - ? 'force-cache' - : opts.preferOnline - ? 'no-cache' - : 'default' -} - -function getHeaders (uri, registry, opts) { - const headers = Object.assign({ - 'npm-in-ci': opts.isFromCI, - 'npm-scope': opts.projectScope, - 'npm-session': opts.npmSession, - 'user-agent': opts.userAgent, - 'referer': opts.refer - }, opts.headers) - // check for auth settings specific to this registry - let auth = ( - opts.auth && - opts.auth[registryKey(registry)] - ) || opts.auth - // If a tarball is hosted on a different place than the manifest, only send - // credentials on `alwaysAuth` - const shouldAuth = auth && ( - auth.alwaysAuth || - url.parse(uri).host === url.parse(registry).host - ) - if (shouldAuth && auth.token) { - headers.authorization = `Bearer ${auth.token}` - } else if (shouldAuth && auth.username && auth.password) { - const encoded = Buffer.from( - `${auth.username}:${auth.password}`, 'utf8' - ).toString('base64') - headers.authorization = `Basic ${encoded}` - } else if (shouldAuth && auth._auth) { - headers.authorization = `Basic ${auth._auth}` - } - return headers -} diff --git a/lib/fetchers/registry/manifest.js b/lib/fetchers/registry/manifest.js index 4e5a801..8b91c34 100644 --- a/lib/fetchers/registry/manifest.js +++ b/lib/fetchers/registry/manifest.js @@ -2,13 +2,11 @@ const BB = require('bluebird') -const fetch = require('./fetch') +const fetch = require('npm-registry-fetch') const LRU = require('lru-cache') const optCheck = require('../../util/opt-check') const pickManifest = require('npm-pick-manifest') -const pickRegistry = require('./pick-registry') const ssri = require('ssri') -const url = require('url') // Corgis are cute. 🐕🐶 const CORGI_DOC = 'application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8, */*' @@ -18,21 +16,14 @@ module.exports = manifest function manifest (spec, opts) { opts = optCheck(opts) - const registry = pickRegistry(spec, opts) - const uri = metadataUrl(registry, spec.escapedName) + const registry = fetch.pickRegistry(spec, opts) + const uri = registry.replace(/\/?$/, '/') + spec.escapedName return getManifest(uri, registry, spec, opts).then(manifest => { return annotateManifest(uri, registry, manifest) }) } -function metadataUrl (registry, name) { - const normalized = registry.slice(-1) !== '/' - ? registry + '/' - : registry - return url.resolve(normalized, name) -} - function getManifest (uri, registry, spec, opts) { return fetchPackument(uri, spec, registry, opts).then(packument => { try { @@ -78,7 +69,7 @@ function fetchPackument (uri, spec, registry, opts) { return BB.resolve(mem.get(uri)) } - return fetch(uri, registry, Object.assign({ + return fetch(uri, opts.concat({ headers: { 'pacote-req-type': 'packument', 'pacote-pkg-id': `registry:${manifest.name}`, diff --git a/lib/fetchers/registry/pick-registry.js b/lib/fetchers/registry/pick-registry.js deleted file mode 100644 index f326950..0000000 --- a/lib/fetchers/registry/pick-registry.js +++ /dev/null @@ -1,17 +0,0 @@ -'use strict' - -module.exports = pickRegistry -function pickRegistry (spec, opts) { - let registry = spec.scope && opts.scopeTargets[spec.scope] - - if (!registry && opts.scope) { - const prefix = opts.scope[0] === '@' ? '' : '@' - registry = opts.scopeTargets[prefix + opts.scope] - } - - if (!registry) { - registry = opts.registry - } - - return registry -} diff --git a/lib/fetchers/registry/registry-key.js b/lib/fetchers/registry/registry-key.js deleted file mode 100644 index f53e9a9..0000000 --- a/lib/fetchers/registry/registry-key.js +++ /dev/null @@ -1,16 +0,0 @@ -'use strict' - -const url = require('url') - -// Called a nerf dart in the main codebase. Used as a "safe" -// key when fetching registry info from config. -module.exports = registryKey -function registryKey (registry) { - const parsed = url.parse(registry) - const formatted = url.format({ - host: parsed.host, - pathname: parsed.pathname, - slashes: parsed.slashes - }) - return url.resolve(formatted, '.') -} diff --git a/lib/fetchers/registry/tarball.js b/lib/fetchers/registry/tarball.js index b4bd874..26f844a 100644 --- a/lib/fetchers/registry/tarball.js +++ b/lib/fetchers/registry/tarball.js @@ -2,18 +2,17 @@ const BB = require('bluebird') -const fetch = require('./fetch') +const fetch = require('npm-registry-fetch') const manifest = require('./manifest') const optCheck = require('../../util/opt-check') const PassThrough = require('stream').PassThrough -const pickRegistry = require('./pick-registry') const ssri = require('ssri') const url = require('url') module.exports = tarball function tarball (spec, opts) { opts = optCheck(opts) - const registry = pickRegistry(spec, opts) + const registry = fetch.pickRegistry(spec, opts) const stream = new PassThrough() let mani if ( @@ -50,9 +49,9 @@ function fromManifest (manifest, spec, opts) { opts = optCheck(opts) if (spec.scope) { opts = opts.concat({scope: spec.scope}) } const stream = new PassThrough() - const registry = pickRegistry(spec, opts) + const registry = fetch.pickRegistry(spec, opts) const uri = getTarballUrl(spec, registry, manifest, opts) - fetch(uri, registry, Object.assign({ + fetch(uri, opts.concat({ headers: { 'pacote-req-type': 'tarball', 'pacote-pkg-id': `registry:${manifest.name}@${uri}` diff --git a/lib/finalize-manifest.js b/lib/finalize-manifest.js index 1189535..66715ff 100644 --- a/lib/finalize-manifest.js +++ b/lib/finalize-manifest.js @@ -42,7 +42,7 @@ function finalizeManifest (pkg, spec, opts) { : BB.resolve(null) return cachedManifest.then(cached => { - if (cached && cached.metadata.manifest) { + if (cached && cached.metadata && cached.metadata.manifest) { return new Manifest(cached.metadata.manifest) } else { return tarballedProps(pkg, spec, opts).then(props => { @@ -55,11 +55,13 @@ function finalizeManifest (pkg, spec, opts) { return manifest } else { return cacache.put( - opts.cache, cacheKey, '.', opts.concat({metadata: { - id: manifest._id, - manifest, - type: 'finalized-manifest' - }}) + opts.cache, cacheKey, '.', { + metadata: { + id: manifest._id, + manifest, + type: 'finalized-manifest' + } + } ).then(() => manifest) } }) diff --git a/lib/util/git.js b/lib/util/git.js index fd22532..59ce491 100644 --- a/lib/util/git.js +++ b/lib/util/git.js @@ -194,7 +194,12 @@ function execGit (gitArgs, gitOpts, opts) { throw err } }) - }, opts.retry) + }, opts.retry != null ? opts.retry : { + retries: opts['fetch-retries'], + factor: opts['fetch-retry-factor'], + maxTimeout: opts['fetch-retry-maxtimeout'], + minTimeout: opts['fetch-retry-mintimeout'] + }) }) } diff --git a/lib/util/opt-check.js b/lib/util/opt-check.js index c919c02..914972b 100644 --- a/lib/util/opt-check.js +++ b/lib/util/opt-check.js @@ -1,118 +1,41 @@ 'use strict' const figgyPudding = require('figgy-pudding') -const pkg = require('../../package.json') const silentlog = require('./silentlog') const AUTH_REGEX = /^(?:.*:)?(token|_authToken|username|_password|password|email|always-auth|_auth|otp)$/ const SCOPE_REGISTRY_REGEX = /@.*:registry$/gi module.exports = figgyPudding({ annotate: {}, - auth: {}, + cache: {}, defaultTag: 'tag', dirPacker: {}, - fullMetadata: 'full-metadata', - 'full-metadata': { - default: false - }, - includeDeprecated: { - default: true - }, - memoize: {}, - preferOnline: 'prefer-online', - preferOffline: 'prefer-offline', - resolved: {}, - scopeTargets: { - default: {} - }, - tag: { - default: 'latest' - }, - where: {}, - - uid: {}, - gid: {}, dmode: {}, fmode: {}, - umask: {}, - agent: {}, - algorithms: { - default: ['sha1'] - }, - body: {}, - ca: {}, - cache: {}, - cert: {}, - 'fetch-retries': {}, - 'fetch-retry-factor': {}, - 'fetch-retry-maxtimeout': {}, - 'fetch-retry-mintimeout': {}, - headers: {}, + 'fetch-retries': {default: 2}, + 'fetch-retry-factor': {default: 10}, + 'fetch-retry-maxtimeout': {default: 60000}, + 'fetch-retry-mintimeout': {default: 10000}, + fullMetadata: 'full-metadata', + 'full-metadata': {default: false}, + gid: {}, + includeDeprecated: {default: true}, integrity: {}, - 'is-from-ci': 'isFromCI', - isFromCI: { - default () { - return ( - process.env['CI'] === 'true' || - process.env['TDDIUM'] || - process.env['JENKINS_URL'] || - process.env['bamboo.buildKey'] || - process.env['GO_PIPELINE_NAME'] - ) - } - }, - key: {}, - 'local-address': {}, - localAddress: 'local-address', - log: { - default: silentlog - }, - maxSockets: 'maxsockets', - 'max-sockets': 'maxsockets', - maxsockets: { - default: 12 - }, - method: { - default: 'GET' - }, - noProxy: 'noproxy', - noproxy: {}, - 'npm-session': 'npmSession', - npmSession: {}, + log: {default: silentlog}, + memoize: {}, offline: {}, - otp: {}, + preferOffline: 'prefer-offline', 'prefer-offline': {}, + preferOnline: 'prefer-online', 'prefer-online': {}, - projectScope: {}, - 'project-scope': 'projectScope', - Promise: {}, - proxy: {}, - query: {}, - refer: {}, - referer: 'refer', - registry: { - default: 'https://registry.npmjs.org/' - }, + registry: {default: 'https://registry.npmjs.org/'}, + resolved: {}, retry: {}, scope: {}, - spec: {}, - strictSSL: 'strict-ssl', - 'strict-ssl': {}, - timeout: {}, - userAgent: 'user-agent', - 'user-agent': { - default: `${ - pkg.name - }@${ - pkg.version - }/node@${ - process.version - }+${ - process.arch - } (${ - process.platform - })` - } + tag: {default: 'latest'}, + uid: {}, + umask: {}, + where: {} }, { other (key) { return key.match(AUTH_REGEX) || key.match(SCOPE_REGISTRY_REGEX) diff --git a/test/finalize-manifest.js b/test/finalize-manifest.js index 7fd1865..753fef0 100644 --- a/test/finalize-manifest.js +++ b/test/finalize-manifest.js @@ -3,6 +3,7 @@ const BB = require('bluebird') const cacache = require('cacache') +const npa = require('npm-package-arg') const npmlog = require('npmlog') const path = require('path') const ssri = require('ssri') @@ -120,10 +121,7 @@ test('fills in shrinkwrap if missing', t => { 'npm-shrinkwrap.json': sr }).then(tarData => { tnock(t, OPTS.registry).get('/' + tarballPath).reply(200, tarData) - return finalizeManifest(base, { - name: base.name, - type: 'range' - }, OPTS).then(manifest => { + return finalizeManifest(base, npa(base.name), OPTS).then(manifest => { t.deepEqual(manifest._shrinkwrap, sr, 'shrinkwrap successfully added') }) }) @@ -147,10 +145,7 @@ test('fills in integrity hash if missing', t => { }).then(tarData => { const integrity = ssri.fromData(tarData, {algorithms: ['sha512']}).toString() tnock(t, OPTS.registry).get('/' + tarballPath).reply(200, tarData) - return finalizeManifest(base, { - name: base.name, - type: 'range' - }, OPTS).then(manifest => { + return finalizeManifest(base, npa(base.name), OPTS).then(manifest => { t.deepEqual(manifest._integrity, integrity, 'integrity hash successfully added') }) }) @@ -180,10 +175,7 @@ test('fills in `bin` if `directories.bin` string', t => { 'foo/my/nope': 'uhhh' }).then(tarData => { tnock(t, OPTS.registry).get('/' + tarballPath).reply(200, tarData) - return finalizeManifest(base, { - name: base.name, - type: 'range' - }, OPTS).then(manifest => { + return finalizeManifest(base, npa(base.name), OPTS).then(manifest => { t.deepEqual(manifest.bin, { 'x.js': path.join('foo', 'my', 'bin', 'x.js'), 'y': path.join('foo', 'my', 'bin', 'y'), @@ -206,10 +198,7 @@ test('fills in `bin` if original was an array', t => { _resolved: OPTS.registry + tarballPath, _hasShrinkwrap: false } - return finalizeManifest(base, { - name: base.name, - type: 'range' - }, OPTS).then(manifest => { + return finalizeManifest(base, npa(base.name), OPTS).then(manifest => { t.deepEqual(manifest.bin, { 'bin1': path.join('foo', 'my', 'bin1'), 'bin2.js': path.join('foo', 'bin2.js') @@ -236,10 +225,9 @@ test('uses package.json as base if passed null', t => { 'foo/x': 'x()' }).then(tarData => { tnock(t, OPTS.registry).get('/' + tarballPath).reply(200, tarData) - return finalizeManifest(null, { - fetchSpec: OPTS.registry + tarballPath, - type: 'remote' - }, OPTS).then(manifest => { + return finalizeManifest( + null, npa(`${base.name}@${OPTS.registry}${tarballPath}`), OPTS + ).then(manifest => { t.deepEqual(manifest, { name: base.name, version: base.version, @@ -281,13 +269,12 @@ test('errors if unable to get a valid default package.json', t => { 'foo/x': 'x()' }).then(tarData => { tnock(t, OPTS.registry).get('/' + tarballPath).reply(200, tarData) - return finalizeManifest(null, { - fetchSpec: OPTS.registry + tarballPath, - type: 'remote' - }, OPTS).then(manifest => { + return finalizeManifest( + null, npa(`${base.name}@${OPTS.registry}${tarballPath}`), OPTS + ).then(manifest => { throw new Error(`Was not supposed to succeed: ${JSON.stringify(manifest)}`) }, err => { - t.equal(err.code, 'ENOPACKAGEJSON') + t.equal(err.code, 'ENOPACKAGEJSON', 'got correct error code') }) }) }) @@ -305,33 +292,23 @@ test('caches finalized manifests', t => { name: base.name, version: base.version } - const opts = Object.create(OPTS) + const opts = Object.assign({}, OPTS) opts.cache = CACHE return makeTarball({ 'package.json': base, 'npm-shrinkwrap.json': sr }).then(tarData => { tnock(t, OPTS.registry).get('/' + tarballPath).reply(200, tarData) - return finalizeManifest(base, { - name: base.name, - type: 'range' - }, opts).then(manifest1 => { + return finalizeManifest(base, npa(base.name), opts).then(manifest1 => { base._integrity = manifest1._integrity return cacache.ls(CACHE, opts).then(entries => { - const promises = [] Object.keys(entries).forEach(k => { - if (!k.match(/^pacote:range-manifest/)) { - promises.push(cacache.put(CACHE, k, '', opts)) - } else { + if (k.match(/^pacote:.*-manifest/)) { t.ok(true, 'manifest entry exists in cache: ' + k) } }) - return BB.all(promises) }).then(() => { - return finalizeManifest(base, { - name: base.name, - type: 'range' - }, opts) + return finalizeManifest(base, npa(base.name), opts) }).then(manifest2 => { t.deepEqual(manifest2, manifest1, 'got cached manifest') }) diff --git a/test/registry-key.js b/test/registry-key.js deleted file mode 100644 index 6629e2b..0000000 --- a/test/registry-key.js +++ /dev/null @@ -1,7 +0,0 @@ -'use strict' - -const tap = require('tap') -const registryKey = require('../lib/fetchers/registry/registry-key') - -tap.equals(registryKey('https://somedomain/somepath/'), '//somedomain/somepath/', 'registryKey should keep URL path') -tap.equals(registryKey('https://somedomain/somepath/morepath'), '//somedomain/somepath/', 'registryKey should strip trailing path segment') diff --git a/test/registry.manifest.js b/test/registry.manifest.js index 0d1759e..3273f8a 100644 --- a/test/registry.manifest.js +++ b/test/registry.manifest.js @@ -164,11 +164,7 @@ test('sends auth token if passed in opts', t => { const opts = { log: OPTS.log, registry: OPTS.registry, - auth: { - '//mock.reg/': { - token: TOKEN - } - } + '//mock.reg/:_authToken': TOKEN } const srv = tnock(t, OPTS.registry) @@ -193,9 +189,7 @@ test('treats options as optional', t => { test('uses scope from spec for registry lookup', t => { const opts = { - scopeTargets: { - '@myscope': OPTS.registry - }, + '@myscope:registry': OPTS.registry, // package scope takes priority scope: '@otherscope' } @@ -214,9 +208,7 @@ test('uses scope opt for registry lookup', t => { return BB.join( manifest('foo@1.2.3', { - scopeTargets: { - '@myscope': OPTS.registry - }, + '@myscope:registry': OPTS.registry, scope: '@myscope', // scope option takes priority registry: 'nope' @@ -224,9 +216,7 @@ test('uses scope opt for registry lookup', t => { t.deepEqual(pkg, PKG, 'used scope to pick registry') }), manifest('bar@latest', { - scopeTargets: { - '@myscope': OPTS.registry - }, + '@myscope:registry': OPTS.registry, scope: 'myscope' // @ auto-inserted }).then(pkg => { t.deepEqual(pkg, PKG, 'scope @ was auto-inserted') @@ -247,14 +237,8 @@ test('supports scoped auth', t => { const TOKEN = 'deadbeef' const opts = { scope: 'myscope', - scopeTargets: { - '@myscope': OPTS.registry - }, - auth: { - '//mock.reg/': { - token: TOKEN - } - } + '@myscope:registry': OPTS.registry, + '//mock.reg/:_authToken': TOKEN } const srv = tnock(t, OPTS.registry) srv.get( @@ -271,10 +255,8 @@ test('sends auth token if passed in global opts', t => { const TOKEN = 'deadbeef' const opts = { registry: OPTS.registry, - auth: { - alwaysAuth: true, - token: TOKEN - } + 'always-auth': true, + token: TOKEN } const srv = tnock(t, OPTS.registry) @@ -292,10 +274,8 @@ test('sends basic authorization if alwaysAuth and _auth', t => { const TOKEN = 'deadbeef' const opts = { registry: OPTS.registry, - auth: { - alwaysAuth: true, - _auth: TOKEN - } + alwaysAuth: true, + _auth: TOKEN } const srv = tnock(t, OPTS.registry)