diff --git a/lib/finalize-manifest.js b/lib/finalize-manifest.js index 132ba73..36a5588 100644 --- a/lib/finalize-manifest.js +++ b/lib/finalize-manifest.js @@ -30,14 +30,13 @@ function finalizeManifest (pkg, spec, opts) { if (err) { return cb(err) } // normalize should not add any fields, and once // makeManifest completes, it should never be modified. - var result = pkg ? new Manifest(pkg, props) : new Manifest(props) + var result = pkg && pkg.name + ? new Manifest(pkg, props) + : new Manifest(props) if (opts.cache) { opts.metadata = result cache.put( - opts.cache, - finalKey(result, spec), - '.', - opts + opts.cache, key || finalKey(result, spec), '.', opts ).then(() => cb(null, result), cb) } else { cb(null, result) @@ -76,7 +75,7 @@ function Manifest (pkg, fromTarball) { if (this.bin && Array.isArray(this.bin)) { // Code yanked from read-package-json. - const m = pkg.directories && pkg.directories.bin || '.' + const m = (pkg.directories && pkg.directories.bin) || '.' this.bin = this.bin.reduce((acc, mf) => { if (mf && mf.charAt(0) !== '.') { const f = path.basename(mf) @@ -105,17 +104,17 @@ function Manifest (pkg, fromTarball) { function tarballedProps (pkg, spec, opts, cb) { cb = dezalgo(cb) const extraProps = {} - const needsShrinkwrap = (!pkg || + const needsShrinkwrap = (!pkg || ( pkg._hasShrinkwrap !== false && !pkg._shrinkwrap - ) - const needsBin = !!(!pkg || + )) + const needsBin = !!(!pkg || ( !pkg.bin && pkg.directories && pkg.directories.bin - ) + )) const needsShasum = !pkg || !pkg._shasum - const needsManifest = !pkg + const needsManifest = !pkg || !pkg.name if (!needsShrinkwrap && !needsBin && !needsShasum && !needsManifest) { opts.log.silly('finalize-manifest', 'Skipping tarball extraction -- nothing needed.') return cb(null, extraProps) @@ -207,11 +206,16 @@ function tarballedProps (pkg, spec, opts, cb) { } function finalKey (pkg, spec) { - return ( - pkg && (pkg._sha512sum || pkg._shasum) && - cache.key( - `${spec.type}-manifest`, - `${pkg._sha512sum || pkg._shasum}-${pkg._resolved}` + if (pkg && pkg._uniqueResolved) { + // git packages have a unique, identifiable id, but no tar sha + return cache.key(`${spec.type}-manifest`, pkg._uniqueResolved) + } else { + return ( + pkg && (pkg._sha512sum || pkg._shasum) && + cache.key( + `${spec.type}-manifest`, + `${pkg._sha512sum || pkg._shasum}-${pkg._resolved}` + ) ) - ) + } } diff --git a/lib/handlers/git/manifest.js b/lib/handlers/git/manifest.js new file mode 100644 index 0000000..d3c73d1 --- /dev/null +++ b/lib/handlers/git/manifest.js @@ -0,0 +1,63 @@ +'use strict' + +const BB = require('bluebird') + +const git = require('../../util/git') +const normalizeGitUrl = require('normalize-git-url') +const optCheck = require('../../util/opt-check') +const pickManifest = require('../../registry/pick-manifest') +const semver = require('semver') + +module.exports = manifest +function manifest (spec, opts) { + opts = optCheck(opts) + // TODO - spec.hosted url stuff should be done in hosted handler + const normed = normalizeGitUrl(spec.hosted ? spec.hosted.ssh : spec.spec) + const rawRef = decodeURIComponent(normed.branch) + return resolve( + normed.url, rawRef, spec.name, opts + ).then(ref => { + if (ref) { + return { + _repo: normed.url, + _resolved: `${normed.url}#${ref.sha}`, + _spec: spec, + _ref: ref, + _rawRef: rawRef, + _uniqueResolved: `${normed.url}#${ref.sha}` + } + } else { + // We're SOL and need a full clone :( + // + // If we're confident enough that `rawRef` is a commit SHA, + // then we can at least get `finalize-manifest` to cache its result. + return { + _repo: normed.url, + _rawRef: rawRef, + _resolved: rawRef.match(/^[a-f0-9]{40}$/) && `${normed.url}#${rawRef}`, + _uniqueResolved: rawRef.match(/^[a-f0-9]{40}$/) && `${normed.url}#${rawRef}` + } + } + }) +} + +function resolve (url, rawRef, name, opts) { + const semverMatch = rawRef.match(/^semver:v?(.*)/) + const isSemver = semverMatch && semver.validRange(semverMatch[1]) + return git.revs(url, opts).then(remoteRefs => { + return isSemver + ? pickManifest({ + versions: remoteRefs.versions, + 'dist-tags': remoteRefs['dist-tags'], + name: name + }, { + type: 'range', + spec: semverMatch[1] + }, opts) + : remoteRefs + ? BB.resolve( + remoteRefs.refs[rawRef] || remoteRefs.refs[remoteRefs.shas[rawRef]] + ) + : null + }) +} diff --git a/lib/handlers/git/tarball.js b/lib/handlers/git/tarball.js new file mode 100644 index 0000000..3e82f3e --- /dev/null +++ b/lib/handlers/git/tarball.js @@ -0,0 +1,119 @@ +'use strict' + +const BB = require('bluebird') + +const cache = require('../../cache') +const git = require('../../util/git') +const mkdirp = BB.promisify(require('mkdirp')) +const optCheck = require('../../util/opt-check') +const osenv = require('osenv') +const path = require('path') +const pipe = BB.promisify(require('mississippi').pipe) +const rimraf = BB.promisify(require('rimraf')) +const tar = require('tar-fs') +const through = require('mississippi').through +const to = require('mississippi').to +const uniqueFilename = require('unique-filename') + +const gitManifest = require('./manifest') + +module.exports = tarball +function tarball (spec, opts) { + opts = optCheck(opts) + let streamErr = null + const stream = through().on('error', e => { streamErr = e }) + gitManifest(spec, opts).then(manifest => { + if (streamErr) { throw streamErr } + return pipe(fromManifest(manifest, spec, opts), stream) + }) + return stream +} + +module.exports.fromManifest = fromManifest +function fromManifest (manifest, spec, opts) { + opts = optCheck(opts) + let streamError + const stream = through().on('error', e => { streamError = e }) + const cacheStream = ( + opts.cache && + cache.get.stream( + opts.cache, cache.key('git-clone', manifest._resolved), opts + ) + ) + cacheStream.pipe(stream) + cacheStream.on('error', err => { + if (err.code !== 'ENOENT') { + return stream.emit('error', err) + } else { + stream.emit('reset') + withTmp(opts, tmp => { + if (streamError) { throw streamError } + return cloneRepo( + manifest._repo, manifest._ref, manifest._rawRef, tmp, opts + ).then(HEAD => { + if (streamError) { throw streamError } + return packDir(manifest._resolved, tmp, stream, opts) + }) + }).catch(err => stream.emit('error', err)) + } + }) + return stream +} + +function withTmp (opts, cb) { + if (opts.cache) { + // cacache has a special facility for working in a tmp dir + return cache.tmp.withTmp(opts.cache, opts, cb) + } else { + const tmpDir = path.join(osenv.tmpdir(), 'pacote-git-tmp') + const tmpName = uniqueFilename(tmpDir, 'git-clone') + const tmp = mkdirp(tmpName).then(() => tmpName).disposer(rimraf) + return BB.using(tmp, cb) + } +} + +function cloneRepo (repo, resolvedRef, rawRef, tmp, opts) { + // TODO - loop through alternatives for hosted specs + if (resolvedRef) { + return git.shallow(repo, resolvedRef.ref, tmp, opts) + } else { + return git.clone(repo, rawRef, tmp, opts) + } +} + +function packDir (label, tmp, target, opts) { + opts = optCheck(opts) + + // TODO - make a better attempt at a pack + const packer = tar.pack(tmp, { + map: header => { + header.name = 'package/' + header.name + }, + ignore: (name) => { + name.match(/\.git/) + } + }) + + if (!opts.cache) { + return pipe(packer, target) + } else { + const cacher = cache.put.stream( + opts.cache, cache.key('git-clone', label), opts + ) + cacher.once('error', err => packer.emit('error', err)) + target.once('error', err => packer.emit('error', err)) + packer.once('error', err => { + cacher.emit('error', err) + target.emit('error', err) + }) + return pipe(packer, to((chunk, enc, cb) => { + cacher.write(chunk, enc, () => { + target.write(chunk, enc, cb) + }) + }, done => { + cacher.end(() => { + target.end(done) + }) + })) + } +} diff --git a/lib/handlers/hosted/manifest.js b/lib/handlers/hosted/manifest.js new file mode 100644 index 0000000..3ec48ba --- /dev/null +++ b/lib/handlers/hosted/manifest.js @@ -0,0 +1,3 @@ +'use strict' + +module.exports = require('../git/manifest') diff --git a/lib/handlers/hosted/tarball.js b/lib/handlers/hosted/tarball.js new file mode 100644 index 0000000..9bee3d8 --- /dev/null +++ b/lib/handlers/hosted/tarball.js @@ -0,0 +1,3 @@ +'use strict' + +module.exports = require('../git/tarball') diff --git a/lib/util/git.js b/lib/util/git.js new file mode 100644 index 0000000..5c8a671 --- /dev/null +++ b/lib/util/git.js @@ -0,0 +1,160 @@ +'use strict' + +const BB = require('bluebird') + +const cp = require('child_process') +const execFileAsync = BB.promisify(cp.execFile, { + multiArgs: true +}) +const finished = BB.promisify(require('mississippi').finished) +const normalizeGitUrl = require('normalize-git-url') +const optCheck = require('./opt-check') +const path = require('path') +const which = BB.promisify(require('which')) + +module.exports.clone = fullClone +function fullClone (repo, committish, target, opts) { + opts = optCheck(opts) + const normed = normalizeGitUrl(repo) + const gitArgs = [ + 'clone', + '-q', + // Mainly for windows, but no harm done + '-c', 'core.longpaths=true', + normed.url, + target + ] + return execGit(gitArgs, { + cwd: path.dirname(target) + }, opts).then(() => { + return execGit(['checkout', committish, '-c', 'core.longpaths=true'], { + cwd: target + }) + }).then(() => headSha(repo, opts)) +} + +module.exports.shallow = shallowClone +function shallowClone (repo, branch, target, opts) { + opts = optCheck(opts) + const normed = normalizeGitUrl(repo) + const gitArgs = [ + 'clone', + '--depth=1', + '-q', + '-b', branch, + // Mainly for windows, but no harm done + '-c', 'core.longpaths=true', + normed.url, + target + ] + return execGit(gitArgs, { + cwd: path.dirname(target) + }, opts).then(() => headSha(repo, opts)) +} + +function headSha (repo, opts) { + opts = optCheck(opts) + return execGit(['rev-parse', '--revs-only', 'HEAD', repo], {}, opts).spread(stdout => { + return stdout.trim() + }) +} + +module.exports.revs = revs +function revs (repo, opts) { + opts = optCheck(opts) + return spawnGit(['ls-remote', repo, '-t', '-h', '*'], { + env: opts.gitEnv + }, opts).then(child => { + let stdout = '' + child.stdout.on('data', d => { stdout += d }) + return finished(child).then(() => { + return stdout.split('\n').reduce((revs, line) => { + const split = line.split(/\s+/, 2) + if (split.length < 2) { return revs } + const sha = split[0].trim() + const ref = split[1].trim().match(/(?:refs\/[^/]+\/)?(.*)/)[1] + if (!ref) { return revs } // ??? + const type = refType(line) + const doc = {sha, ref, type} + + revs.refs[ref] = doc + // We can check out shallow clones on specific SHAs if we have a ref + if (revs.shas[sha]) { + revs.shas[sha].push(ref) + } else { + revs.shas[sha] = [ref] + } + + if (type === 'tag') { + const match = ref.match(/v?(\d+\.\d+\.\d+)$/) + if (match) { + revs.versions[match[1]] = doc + } + } + + return revs + }, {versions: {}, 'dist-tags': {}, refs: {}, shas: {}}) + }).then(revs => { + if (revs.refs.HEAD) { + const HEAD = revs.refs.HEAD + Object.keys(revs.versions).forEach(v => { + if (v.sha === HEAD.sha) { + revs['dist-tags'].HEAD = v + if (!revs.refs.latest) { + revs['dist-tags'].latest = revs.refs.HEAD + } + } + }) + } + return revs + }) + }) +} + +module.exports._exec = execGit +function execGit (gitArgs, _gitOpts, opts) { + opts = optCheck(opts) + const gitOpts = { + env: opts.gitEnv, + uid: opts.uid, + gid: opts.gid + } + Object.keys(_gitOpts || {}).forEach(k => { + gitOpts[k] = _gitOpts[k] + }) + return which(opts.gitPath).catch(err => { + err.code = 'ENOGIT' + throw err + }).then(gitPath => { + return execFileAsync(gitPath, gitArgs, gitOpts) + }) +} + +module.exports._spawn = spawnGit +function spawnGit (gitArgs, _gitOpts, opts) { + opts = optCheck(opts) + const gitOpts = { + env: opts.gitEnv, + uid: opts.uid, + gid: opts.gid + } + Object.keys(_gitOpts).forEach(k => { + gitOpts[k] = _gitOpts[k] + }) + return which(opts.gitPath).catch(err => { + err.code = 'ENOGIT' + throw err + }).then(gitPath => { + return cp.spawn(gitPath, gitArgs, gitOpts) + }) +} + +function refType (ref) { + return ref.match(/refs\/tags\/.*$/) + ? 'tag' + : ref.match(/refs\/heads\/.*$/) + ? 'branch' + : ref.match(/HEAD$/) + ? 'head' + : 'other' +} diff --git a/lib/util/opt-check.js b/lib/util/opt-check.js index 03e06f6..d588223 100644 --- a/lib/util/opt-check.js +++ b/lib/util/opt-check.js @@ -26,6 +26,8 @@ function PacoteOptions (opts) { this.scope = opts.scope this.where = opts.where + this.gitPath = opts.gitPath || 'git' + this.gitEnv = opts.gitEnv || process.env this.uid = opts.uid this.gid = opts.gid diff --git a/package.json b/package.json index 6c9bbb8..46f6070 100644 --- a/package.json +++ b/package.json @@ -41,22 +41,25 @@ "license": "CC0-1.0", "dependencies": { "bluebird": "^3.5.0", - "cacache": "^6.1.0", + "cacache": "^6.2.0", "checksum-stream": "^1.0.2", "dezalgo": "^1.0.3", "glob": "^7.1.1", "inflight": "^1.0.6", "minimatch": "^3.0.3", "mississippi": "^1.2.0", + "normalize-git-url": "^3.0.2", "normalize-package-data": "^2.3.6", "npm-registry-client": "^7.4.6", + "osenv": "^0.1.4", "promise-inflight": "^1.0.1", "realize-package-specifier": "^3.0.3", "request": "^2.81.0", "semver": "^5.3.0", "slide": "^1.1.6", "tar-fs": "^1.15.1", - "tar-stream": "^1.5.2" + "tar-stream": "^1.5.2", + "which": "^1.2.12" }, "devDependencies": { "mkdirp": "^0.5.1", @@ -66,6 +69,7 @@ "require-inject": "^1.4.0", "rimraf": "^2.5.4", "standard": "^9.0.1", + "tacks": "^1.2.6", "tap": "^10.2.0", "weallbehave": "^1.0.0", "weallcontribute": "^1.0.7" diff --git a/test/util.git.js b/test/util.git.js new file mode 100644 index 0000000..631465f --- /dev/null +++ b/test/util.git.js @@ -0,0 +1,21 @@ +'use strict' + +const BB = require('bluebird') + +const git = require('../lib/util/git') +const gitUtil = require('./util/git') +const Tacks = require('tacks') +const test = require('tap').test +const which = BB.promisify(require('which')) + +const CACHE = require('./util/test-dir')(__filename) + +const systemGit = which.sync('git') + +test('executes git binary', { + skip: !systemGit && 'requires git' +}, t => { + return git._exec(['--version']).spread(stdout => { + t.match(stdout, /^git version/, 'successfully ran git') + }) +}) diff --git a/test/util/git.js b/test/util/git.js new file mode 100644 index 0000000..f725c15 --- /dev/null +++ b/test/util/git.js @@ -0,0 +1,47 @@ +'use strict' + +const BB = require('bluebird') + +const cp = BB.promisifyAll(require('child_process')) +const git = require('../../lib/util/git') +const mkdirp = BB.promisify(require('mkdirp')) + +module.exports = mockRepo +function mockRepo (opts) { + return mkdirp() +} + +module.exports.daemon = daemon +function daemon (opts) { + opts = opts || {} + return BB.fromNode(cb => { + const srv = cp.spawn('git', [ + 'daemon', + '--verbose', + '--listen=localhost', + `--port=${opts.port || 1234}`, + '--reuseaddr', + '--export-all', + '--base-path=.' + ], { + cwd: opts.cwd || process.cwd() + }) + srv.stderr.on('data', d => { + console.warn(d.toString('utf8')) + }) + srv.stdout.on('data', d => { + const str = d.toString('utf8') + const match = str.match(/\[(\d+)\]/) + if (match) { + srv.pid = parseInt(match[1]) + cb(null, srv) + } + }) + srv.once('exit', cb) + srv.once('error', cb) + }).disposer(srv => BB.fromNode(cb => { + srv.on('error', cb) + srv.on('close', cb) + srv.kill() + })) +}