Merge 9dc845a into 2333a17

README.md

@@ -72,6 +72,7 @@ fields. The object has the following shape:
   "bundleDependencies": false || [PkgName],
   "bin": { BinName: Path },
   "_resolved": TarballSource, // different for each package type
+  "_tarballStandardPath": Boolean, // true if the tarball path follows standard convention for registries, and thus can be guessed correctly.
   "_integrity": SubresourceIntegrityHash,
   "_shrinkwrap": null || ShrinkwrapJsonObj
 }
@@ -246,6 +247,16 @@ specifiers. If true, deprecations do not affect version selection.
 If `true`, the full packument will be fetched when doing metadata requests. By
 defaul, `pacote` only fetches the summarized packuments, also called "corgis".
 
+##### <a name="opts-guess-resolved"></a> `opts.guessResolved`
+
+* Type: Boolean
+* Default: false
+
+If true, pacote will guess the resolved URL for `version` registry specs if
+`opts.resolved` is omitted. It will only fall back to manifest checking if the
+guessed URL 404s. Enabling this can be a breaking change because it may add
+spurious 404s.
+
 ##### <a name="opts-tag"></a> `opts.tag`
 
 * Alias: `opts.defaultTag`

lib/fetchers/registry/manifest.js

@@ -55,8 +55,19 @@ function annotateManifest (spec, manifest, opts) {
   manifest._resolved = (
     manifest.dist && manifest.dist.tarball
   )
+  const registry = fetch.pickRegistry(spec, opts)
+  const standardPath = `/${spec.name}/-/${spec.name.replace(/^@[^/]+./, '')}-${manifest.version}.tgz`
+  // Manifests with `dist.tarball` that matches the standard registry.npmjs.org
+  // paths get tagged as such. This will allow them to be omitted by pkglock.
+  manifest._tarballStandardPath = (
+    manifest._resolved &&
+    manifest._resolved.indexOf(registry) === 0 &&
+    manifest._resolved.slice(
+      registry.replace(/\/$/, '').length
+    ) === standardPath
+  ) || false
+
   if (!manifest._resolved) {
-    const registry = fetch.pickRegistry(spec, opts)
     const uri = registry.replace(/\/?$/, '/') + spec.escapedName
 
     const err = new Error(
@@ -66,8 +77,7 @@ function annotateManifest (spec, manifest, opts) {
     err.manifest = manifest
     if (!manifest._warnings) { manifest._warnings = [] }
     manifest._warnings.push(err.message)
-    manifest._resolved =
-    `${registry}/${manifest.name}/-/${manifest.name}-${manifest.version}.tgz`
+    manifest._resolved = `${registry}${standardPath}`
   }
   return manifest
 }

lib/fetchers/registry/tarball.js

@@ -28,6 +28,22 @@ function tarball (spec, opts) {
       _resolved: opts.resolved,
       _fakeChild: true
     })
+  } else if (!opts.resolved && spec.type === 'version' && opts.guessResolved) {
+    // If no resolved field, make it try a standard-format URL.
+    const guessed = `${
+      registry.replace(/\/+$/, '')
+    }/${spec.name}/-/${
+      // scoped packages get the scope removed from tarball name
+      spec.name.replace(/^@[^/]+\//, '')
+    }-${spec.fetchSpec}.tgz`
+    mani = BB.resolve({
+      name: spec.name,
+      version: spec.fetchSpec,
+      _integrity: opts.integrity,
+      _resolved: guessed,
+      _guessedResolved: true,
+      _fakeChild: true
+    })
   } else {
     // We can't trust opts.resolved if it's going to a separate host.
     mani = manifest(spec, opts)
@@ -46,35 +62,44 @@ function tarball (spec, opts) {
 }
 
 module.exports.fromManifest = fromManifest
-function fromManifest (manifest, spec, opts) {
+function fromManifest (mani, spec, opts) {
   opts = optCheck(opts)
   if (spec.scope) { opts = opts.concat({scope: spec.scope}) }
   const stream = new PassThrough()
   const registry = fetch.pickRegistry(spec, opts)
-  const uri = getTarballUrl(spec, registry, manifest, opts)
-  fetch(uri, opts.concat({
+  const uri = getTarballUrl(spec, registry, mani, opts)
+  const reqOpts = opts.concat({
     headers: {
       'pacote-req-type': 'tarball',
-      'pacote-pkg-id': `registry:${manifest.name}@${uri}`
+      'pacote-pkg-id': `registry:${mani.name}@${uri}`
     },
-    integrity: manifest._integrity,
+    integrity: mani._integrity,
     algorithms: [
-      manifest._integrity
-        ? ssri.parse(manifest._integrity).pickAlgorithm()
+      mani._integrity
+        ? ssri.parse(mani._integrity).pickAlgorithm()
         : 'sha1'
     ],
     spec
-  }, opts))
-    .then(res => {
-      const hash = res.headers.get('x-local-cache-hash')
-      if (hash) {
-        stream.emit('integrity', decodeURIComponent(hash))
-      }
-      res.body.on('error', err => stream.emit('error', err))
-      res.body.pipe(stream)
-      return null
-    })
-    .catch(err => stream.emit('error', err))
+  }, opts)
+  fetch(uri, reqOpts).catch(err => {
+    if (err.code === 'E404' && mani._guessedResolved) {
+      // Retry if we guessed the tarball url.
+      return manifest(spec, opts).then(mani => {
+        const newUri = getTarballUrl(spec, registry, mani, reqOpts)
+        return fetch(newUri, opts)
+      })
+    } else {
+      throw err
+    }
+  }).then(res => {
+    const hash = res.headers.get('x-local-cache-hash')
+    if (hash) {
+      stream.emit('integrity', decodeURIComponent(hash))
+    }
+    res.body.on('error', err => stream.emit('error', err))
+    res.body.pipe(stream)
+    return null
+  }, err => stream.emit('error', err))
   return stream
 }
 

lib/finalize-manifest.js

@@ -94,6 +94,7 @@ function Manifest (pkg, fromTarball, fullMetadata) {
 
   // These depend entirely on each handler
   this._resolved = pkg._resolved
+  this._tarballStandardPath = pkg._tarballStandardPath || false
 
   // Not all handlers (or registries) provide these out of the box,
   // and if they don't, we need to extract and read the tarball ourselves.

lib/util/opt-check.js

@@ -19,6 +19,7 @@ module.exports = figgyPudding({
   fullMetadata: 'full-metadata',
   'full-metadata': {default: false},
   gid: {},
+  guessResolved: {default: false},
   includeDeprecated: {default: true},
   'include-deprecated': 'includeDeprecated',
   integrity: {},

test/directory.js

@@ -58,6 +58,7 @@ test('supports directory deps', t => {
       peerDependencies: {},
       deprecated: false,
       _resolved: path.resolve(PKG).replace(/\\/g, '/'),
+      _tarballStandardPath: false,
       _shasum: null,
       _integrity: null,
       _shrinkwrap: sr,

test/finalize-manifest.js

@@ -43,6 +43,7 @@ test('returns a manifest with the right fields', t => {
     bin: './foo.js',
     _resolved: 'resolved.to.this',
     _integrity: 'sha1-deadbeefc0ffeebad1dea',
+    _tarballStandardPath: false,
     _shasum: 'deadbeef1',
     _hasShrinkwrap: false,
     deprecated: 'foo'
@@ -65,6 +66,7 @@ test('returns a manifest with the right fields', t => {
       _shasum: 'deadbeef1',
       _resolved: 'resolved.to.this',
       _integrity: 'sha1-deadbeefc0ffeebad1dea',
+      _tarballStandardPath: false,
       _shrinkwrap: null,
       deprecated: 'foo',
       _id: 'testing@1.2.3'
@@ -94,6 +96,7 @@ test('defaults all field to expected types + values', t => {
       bundleDependencies: false, // because npm does boolean checks on this
       peerDependencies: {},
       bin: null,
+      _tarballStandardPath: false,
       _resolved: base._resolved,
       _integrity: base._integrity,
       _shasum: base._shasum,
@@ -220,6 +223,7 @@ test('fills in `bin` if original was an array', t => {
     },
     _integrity: 'sha1-deadbeefc0ffeebad1dea',
     _shasum: '75e69d6de79f7347df79e6da77575e',
+    _tarballStandardPath: false,
     _resolved: OPTS.registry + tarballPath,
     _hasShrinkwrap: false
   }
@@ -264,6 +268,7 @@ test('uses package.json as base if passed null', t => {
         devDependencies: {},
         bundleDependencies: false,
         peerDependencies: {},
+        _tarballStandardPath: false,
         _resolved: OPTS.registry + tarballPath,
         deprecated: false,
         _integrity: ssri.fromData(tarData, {algorithms: ['sha512']}).toString(),

test/prefetch.js

@@ -19,6 +19,7 @@ const BASE = {
   name: 'foo',
   version: '1.0.0',
   _hasShrinkwrap: false,
+  _tarballStandardPath: false,
   _resolved: 'https://foo.bar/x.tgz',
   dist: {
     tarball: 'https://foo.bar/x.tgz'