Skip to content
This repository has been archived by the owner on Jul 3, 2019. It is now read-only.

Commit

Permalink
feat(manifest): switch to cacache for caching
Browse files Browse the repository at this point in the history 
Fixes: #51

BREAKING CHANGE: Toplevel APIs now return Promises instead of using callbacks.
  • Loading branch information
@zkat
zkat committed Mar 5, 2017
1 parent f3e224f commit 12917fd
Show file tree
Hide file tree
Showing 22 changed files with 804 additions and 1,133 deletions.
58 changes: 44 additions & 14 deletions README.md
View file
Expand Up @@ -20,44 +20,61 @@ needed to reduce excess operations, using [`cacache`](https://npm.im/cacache).
* [API](#api)
* [`manifest`](#manifest)
* [`extract`](#extract)
* [`prefetch`](#prefetch)
* [`options`](#options)

### Example

```javascript
const pacote = require('pacote')

pacote.manifest('pacote@^1', function (err, pkg) {
pacote.manifest('pacote@^1').then(pkg => {
console.log('package manifest for registry pkg:', pkg)
// { "name": "pacote", "version": "1.0.0", ... }
})

pacote.extract('http://hi.com/pkg.tgz', './here', function (err) {
pacote.extract('http://hi.com/pkg.tgz', './here').then(() => {
console.log('remote tarball contents extracted to ./here')
})
```

### Features

* simple interface to common package-related actions.
* supports all package types npm does.
* fast cache
* Handles all package types [npm](https://npm.im/npm) does
* [high-performance, reliably, verified local cache](https://npm.im/cacache)
* offline mode
* authentication support (private git, private npm registries, etc)
* npm-compatible for all relevant operations
* github, gitlab, and bitbucket-aware
* version/tag aware when fetching from git repositories.
* caches git repositories
* semver range support for git dependencies

### Contributing

The pacote team enthusiastically welcomes contributions and project participation! There's a bunch of things you can do if you want to contribute! The [Contributor Guide](CONTRIBUTING.md) has all the information you need for everything from reporting bugs to contributing entire new features. Please don't hesitate to jump in if you'd like to, or even ask us questions if something isn't clear.

### API

#### <a name="manifest"></a> `> pacote.manifest(spec, [opts], cb)`
#### <a name="manifest"></a> `> pacote.manifest(spec, [opts])`

Fetches the *manifest* for a package, aka `package.json`.
Fetches the *manifest* for a package. Manifest objects are similar and based
on the `package.json` for that package, but with pre-processed and limited
fields. The object has the following shape:

```javascript
{
"name": PkgName,
"version": SemverString,
"dependencies": { PkgName: SemverString },
"optionalDependencies": { PkgName: SemverString },
"devDependencies": { PkgName: SemverString },
"peerDependencies": { PkgName: SemverString },
"bundleDependencies": false || [PkgName],
"bin": { BinName: Path },
"_resolved": TarballSource, // different for each package type
"_shasum": TarballSha1Sum,
"_sha512sum": TarballSha512Sum,
"_shrinkwrap": null || ShrinkwrapJsonObj
}
```

Note that depending on the spec type, some additional fields might be present.
For example, packages from `registry.npmjs.org` have additional metadata
Expand All @@ -66,12 +83,12 @@ appended by the registry.
##### Example

```javascript
pacote.manifest('pacote@1.0.0', function (err, pkgJson) {
// fetched `package.json` data from the registry (or cache, if cached)
pacote.manifest('pacote@1.0.0').then(pkgJson => {
// fetched `package.json` data from the registry
})
```

#### <a name="extract"></a> `> pacote.extract(spec, destination, [opts], cb)`
#### <a name="extract"></a> `> pacote.extract(spec, destination, [opts])`

Extracts package data identified by `<spec>` into a directory named
`<destination>`, which will be created if it does not already exist.
Expand All @@ -85,12 +102,25 @@ tarball.
```javascript
pacote.extract('pacote@1.0.0', './woot', {
digest: 'deadbeef'
}, function (err) {
}).then(() => {
// Succeeds as long as `pacote@1.0.0` still exists somewhere. Network and
// other operations are bypassed entirely if `digest` is present in the cache.
})
```

#### <a name="prefetch"></a> `> pacote.prefetch(spec, [opts])`

Fetches package data identified by `<spec>`, usually for the purpose of warming
up the local package cache (with `opts.cache`). It does not return anything.

##### Example

```javascript
pacote.prefetch('pacote@1.0.0', { cache: './my-cache' }).then(() => {
// ./my-cache now has both the manifest and tarball for `pacote@1.0.0`.
})
```

#### <a name="options"></a> `> options`

##### `opts.digest`
Expand Down
50 changes: 23 additions & 27 deletions extract.js
View file
@@ -1,48 +1,44 @@
'use strict'

var cache = require('./lib/cache')
var extractStream = require('./lib/extract-stream')
var pipe = require('mississippi').pipe
var optCheck = require('./lib/util/opt-check')
var rps = require('realize-package-specifier')
const BB = require('bluebird')

const cache = require('./lib/cache')
const extractStream = require('./lib/extract-stream')
const pipe = BB.promisify(require('mississippi').pipe)
const optCheck = require('./lib/util/opt-check')
const rps = BB.promisify(require('realize-package-specifier'))

module.exports = extract
function extract (spec, dest, opts, cb) {
if (!cb) {
cb = opts
opts = null
}
function extract (spec, dest, opts) {
opts = optCheck(opts)
if (opts.digest) {
opts.log.silly('extract', 'trying ', spec, ' digest:', opts.digest)
extractByDigest(dest, opts, function (err) {
return extractByDigest(
dest, opts
).catch(err => {
if (err && err.code === 'ENOENT') {
opts.log.silly('extract', 'digest for', spec, 'not present. Using manifest.')
return extractByManifest(spec, dest, opts, cb)
return extractByManifest(spec, dest, opts)
} else {
return cb(err)
throw err
}
})
} else {
opts.log.silly('extract', 'no digest provided for ', spec, '- extracting by manifest')
extractByManifest(spec, dest, opts, cb)
return extractByManifest(spec, dest, opts)
}
}

function extractByDigest (dest, opts, cb) {
var xtractor = extractStream(dest, opts)
var cached = cache.get.stream.byDigest(opts.cache, opts.digest, opts)
pipe(cached, xtractor, cb)
function extractByDigest (dest, opts) {
const xtractor = extractStream(dest, opts)
const cached = cache.get.stream.byDigest(opts.cache, opts.digest, opts)
return pipe(cached, xtractor)
}

function extractByManifest (spec, dest, opts, cb) {
var xtractor = extractStream(dest, opts)
rps(spec, function (err, res) {
if (err) { return cb(err) }
var tarball = require('./lib/handlers/' + res.type + '/tarball')
pipe(tarball(res, opts), xtractor, function (err) {
opts.log.silly('extract', 'extraction finished for', spec)
cb(err)
})
function extractByManifest (spec, dest, opts) {
const xtractor = extractStream(dest, opts)
return rps(spec).then(res => {
const tarball = require('./lib/handlers/' + res.type + '/tarball')
return pipe(tarball(res, opts), xtractor)
})
}
155 changes: 1 addition & 154 deletions lib/cache/index.js
View file
Expand Up @@ -2,159 +2,6 @@

var cacache = require('cacache')
var cacheKey = require('./cache-key')
var finished = require('mississippi').finished
var through = require('mississippi').through
var pipe = require('mississippi').pipe
var pipeline = require('mississippi').pipeline

var HASH = 'sha1'

module.exports = cacache
module.exports.key = cacheKey

var MEMOIZED = {}

module.exports._clearMemoized = clearMemoized
function clearMemoized () {
var old = MEMOIZED
MEMOIZED = {}
return old
}

module.exports.memoize = memoize
function memoize (key, data) {
MEMOIZED[key] = data
}

module.exports.readMemoized = readMemoized
function readMemoized (key) {
return MEMOIZED[key]
}

module.exports.put = putData
function putData (cache, key, data, opts, cb) {
if (!cb) {
cb = opts
opts = null
}
opts = opts || {}
var put = putStream(cache, key, opts)
var dummy = through()
pipe(dummy, put, function (err) {
cb(err)
})
dummy.write(data)
dummy.end()
}

module.exports.put.stream = putStream
function putStream (cache, key, opts) {
// This is the most ridiculous thing, but cacache
// needs an API overhaul, and I need to think longer
// about whether to emit the index entry...
var dest = cacache.put.stream(cache, key, opts)
var lol = through()
var piped = pipeline(lol, dest)
var meta
dest.on('metadata', function (m) {
meta = m.metadata
piped.emit('metadata', m.metadata)
})
dest.on('digest', function (d) {
piped.emit('digest', d)
})
dest.on('end', function () {
opts.log.silly('cache.put', 'finished writing cache data for', key)
})
if (opts.memoize) {
opts.log.silly('cache.put', 'memoization of', key, 'requested')
var data = ''
lol.on('data', function (d) {
data += d
})
finished(piped, function (err) {
if (err) { return }
opts.log.silly('cache.put', key, 'memoized')
MEMOIZED[cache + ':' + key] = {
data: data,
meta: meta || null
}
})
}
return piped
}

module.exports.get = getData
function getData (cache, key, opts, cb) {
if (!cb) {
cb = opts
opts = null
}
opts = opts || {}
var meta
var data = ''
var stream = getStream(
false, cache, key, opts
).on('metadata', function (m) {
meta = m
}).on('data', function (d) {
data += d
})
finished(stream, function (err) {
cb(err, data, meta)
})
}

module.exports.get.stream = function (cache, key, opts) {
return getStream(false, cache, key, opts)
}
module.exports.get.stream.byDigest = function (cache, digest, opts) {
return getStream(true, cache, digest, opts)
}
function getStream (byDigest, cache, key, opts) {
opts = opts || {}
var stream
if (!cache || !key) {
throw new Error('cache and key are required')
}
var memoed = MEMOIZED[cache + ':' + key]
if (memoed) {
opts.log && opts.log.silly('cache.get', key, 'already memoized.')
stream = through()
stream.on('newListener', function (ev, cb) {
ev === 'metadata' && cb(memoed.meta)
})
stream.write(memoed.data)
stream.end()
return stream
} else {
var dest = through()
var meta
var getter = byDigest ? cacache.get.stream.byDigest : cacache.get.stream
var src = getter(cache, key, {
hashAlgorithm: HASH
}).on('metadata', function (m) {
meta = m.metadata
piped.emit('metadata', m.metadata)
}).on('digest', function (d) {
piped.emit('digest', d)
})
var piped = pipeline(src, dest)
if (opts.memoize) {
var data = ''
piped.on('data', function (d) {
data += d
})
finished(piped, function (err) {
if (err) { return }
MEMOIZED[cache + ':' + key] = {
data: data,
meta: meta || null
}
})
}
return piped
}
}

module.exports.get.info = cacache.get.info
module.exports.get.hasContent = require('cacache/lib/content/read').hasContent

0 comments on commit 12917fd

Please sign in to comment.