Skip to content

Commit

Permalink
Add support for exotic package types (file, hosted, and tarball)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mgcrea
mgcrea committed Jun 20, 2016
1 parent b91452a commit e8ff294
Show file tree
Hide file tree
Showing 2 changed files with 127 additions and 19 deletions.
2 changes: 2 additions & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,14 @@
"gunzip-maybe": "^1.2.1",
"init-package-json": "^1.9.1",
"lodash.frompairs": "^4.0.1",
"lodash.memoize": "^4.1.0",
"minimist": "^1.2.0",
"mkdirp": "^0.5.1",
"needle": "^1.0.0",
"node-gyp": "^3.3.1",
"node-pre-gyp": "^0.6.26",
"node-uuid": "^1.4.3",
"npm-package-arg": "^4.2.0",
"ora": "^0.2.1",
"rimraf": "^2.5.2",
"rxjs": "^5.0.0-beta.2",
Expand Down
144 changes: 125 additions & 19 deletions src/install.js
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,23 @@
import crypto from 'crypto'
import path from 'path'
import url from 'url'
import {ArrayObservable} from 'rxjs/observable/ArrayObservable'
import {EmptyObservable} from 'rxjs/observable/EmptyObservable'
import {Observable} from 'rxjs/Observable'
import {_finally} from 'rxjs/operator/finally'
import {concatStatic} from 'rxjs/operator/concat'
import {distinctKey} from 'rxjs/operator/distinctKey'
import {expand} from 'rxjs/operator/expand'
import {forkJoin as forkJoinStatic} from 'rxjs/observable/forkJoin'
import {map} from 'rxjs/operator/map'
import {_catch} from 'rxjs/operator/catch'
import {mergeMap} from 'rxjs/operator/mergeMap'
import {retry} from 'rxjs/operator/retry'
import {skip} from 'rxjs/operator/skip'
import needle from 'needle'
import assert from 'assert'
import npa from 'npm-package-arg'
import memoize from 'lodash.memoize'

import * as cache from './cache'
import * as config from './config'
Expand All @@ -25,6 +29,7 @@ import {normalizeBin, parseDependencies} from './pkg_json'
import debuglog from './debuglog'

const log = debuglog('install')
const cachedNpa = memoize(npa)

/**
* properties of project-level `package.json` files that will be checked for
Expand All @@ -51,16 +56,26 @@ export const DEPENDENCY_FIELDS = [

/**
* resolve a dependency's `package.json` file from the local file system.
* @param {String} nodeModules - `node_modules` base directory.
* @param {String} parentTarget - relative parent's node_modules path.
* @param {String} name - name of the dependency.
* @param {String} nodeModules - `node_modules` base directory.
* @param {String} parentTarget - relative parent's node_modules path.
* @param {String} name - name of the dependency.
* @param {String} version - version of the dependency.
* @return {Observable} - observable sequence of `package.json` objects.
*/
export function resolveLocal (nodeModules, parentTarget, name) {
export function resolveLocal (nodeModules, parentTarget, name, version) {
const linkname = path.join(nodeModules, parentTarget, 'node_modules', name)
const fetch = () => EmptyObservable.create()
log(`resolving ${linkname} from node_modules`)

// support `file:` with symlinks
if (version.substr(0, 5) === 'file:') {
log(`resolved ${name}@${version} as local symlink`)
const isScoped = name.charAt(0) === '@'
const src = path.join(parentTarget, isScoped ? '..' : '', version.substr(5))
const dst = path.join('node_modules', parentTarget, 'node_modules', name)
return util.forceSymlink(src, dst)::_finally(progress.complete)
}

return util.readlink(linkname)::mergeMap((rel) => {
const target = path.basename(path.dirname(rel))
const filename = path.join(linkname, 'package.json')
Expand All @@ -72,22 +87,112 @@ export function resolveLocal (nodeModules, parentTarget, name) {
})
}

/**
* resolve a dependency's `package.json` file from a remote registry.
* @param {String} nodeModules - `node_modules` base directory.
* @param {String} parentTarget - relative parent's node_modules path.
* @param {String} name - name of the dependency.
* @param {String} version - version of the dependency.
* @return {Observable} - observable sequence of `package.json` objects.
*/
export function resolveRemote (nodeModules, parentTarget, name, version) {
log(`resolving ${name}@${version} from ${nodeModules} via ${nodeModules}`)
const source = `${name}@${version}`
log(`resolving ${source} from remote registry`)

const parsedSpec = cachedNpa(source)

switch (parsedSpec.type) {
case 'range':
case 'version':
case 'tag':
return resolveFromNpm(nodeModules, parentTarget, parsedSpec)
case 'remote':
return resolveFromTarball(nodeModules, parentTarget, parsedSpec)
case 'hosted':
return resolveFromGitHub(nodeModules, parentTarget, parsedSpec)
default:
throw new Error('Unknown package spec: ' + parsedSpec.type + ' for ' + name)
}
}

/**
* resolve a dependency's `package.json` file from the npm registry.
* @param {String} nodeModules - `node_modules` base directory.
* @param {String} parentTarget - relative parent's node_modules path.
* @param {Object} parsedSpec - parsed package name and specifier.
* @return {Observable} - observable sequence of `package.json` objects.
*/
export function resolveFromNpm (nodeModules, parentTarget, parsedSpec) {
const {raw, name, type, spec} = parsedSpec
log(`resolving ${raw} from npm`)
const options = {...config.httpOptions, retries: config.retries}
return registry.match(name, version, options)::map((pkgJson) => {
return registry.match(name, spec, options)::map((pkgJson) => {
const target = pkgJson.dist.shasum
log(`resolved ${name}@${version} to ${target}`)
return { parentTarget, pkgJson, target, name, fetch }
log(`resolved ${raw} to tarball shasum ${target} from npm`)
return { parentTarget, pkgJson, target, name, type, fetch }
})
}

/**
* resolve a dependency's `package.json` file from an url tarball.
* @param {String} nodeModules - `node_modules` base directory.
* @param {String} parentTarget - relative parent's node_modules path.
* @param {Object} parsedSpec - parsed package name and specifier.
* @return {Observable} - observable sequence of `package.json` objects.
*/
export function resolveFromTarball (nodeModules, parentTarget, parsedSpec) {
const {raw, name, type, spec} = parsedSpec
log(`resolving ${raw} from tarball`)
return Observable.create((observer) => {
// create shasum from url for storage
const hash = crypto.createHash('sha1')
hash.update(raw)
const shasum = hash.digest('hex')
const pkgJson = {name, dist: {tarball: spec, shasum}}
log(`resolved ${raw} to uri shasum ${shasum} from tarball`)
observer.next({ parentTarget, pkgJson, target: shasum, name, type, fetch })
observer.complete()
})
}

/**
* resolve a dependency's `package.json` file from the github registry.
* @param {String} nodeModules - `node_modules` base directory.
* @param {String} parentTarget - relative parent's node_modules path.
* @param {Object} parsedSpec - parsed package name and specifier.
* @return {Observable} - observable sequence of `package.json` objects.
*/
export function resolveFromGitHub (nodeModules, parentTarget, parsedSpec) {
const {raw, name, type, spec, hosted} = parsedSpec
log(`resolving ${raw} from github`)

const hashIndex = spec.indexOf('#')
const ref = hashIndex !== -1 ? spec.substr(hashIndex + 1) : 'master'
const githubUri = spec.substr(7, hashIndex - 7)

// fetch hosted package.json to get package name
const pkgUri = hosted.directUrl
// fetch specified ref current commit to be used as a shasum for storage
// @TODO handle GitHub API rejections
const refUri = url.resolve('https://api.github.com/repos/', githubUri + '/git/refs/heads/' + ref)
const options = {...config.httpOptions, retries: config.retries}
return forkJoinStatic(
registry.fetch(pkgUri, options)::map(({ body }) => JSON.parse(body)),
registry.fetch(refUri, options)::map(({ body }) => body)
)::map(([pkgJson, refJson]) => {
const tarball = url.resolve('https://codeload.github.com/', githubUri + '/tar.gz/' + ref)
const shasum = refJson.object.sha
pkgJson.dist = {tarball, shasum}
log(`resolved ${name}@${ref} to commit shasum ${shasum} from github`)
return { parentTarget, pkgJson, target: shasum, name: pkgJson.name, type, fetch }
}, {})
}

/**
* resolve an individual sub-dependency based on the parent's target and the
* current working directory.
* @param {String} nodeModules - `node_modules` base directory.
* @param {String} parentTarget - target path used for determining the sub-
* @param {String} nodeModules - `node_modules` base directory.
* @param {String} parentTarget - target path used for determining the sub-
* dependency's path.
* @return {Obserable} - observable sequence of `package.json` root documents
* wrapped into dependency objects representing the resolved sub-dependency.
Expand All @@ -98,7 +203,7 @@ export function resolve (nodeModules, parentTarget) {
progress.report(`resolving ${name}@${version}`)
log(`resolving ${name}@${version}`)

return resolveLocal(nodeModules, parentTarget, name)
return resolveLocal(nodeModules, parentTarget, name, version)
::_catch((error) => {
if (error.code !== 'ENOENT') {
throw error
Expand All @@ -112,8 +217,8 @@ export function resolve (nodeModules, parentTarget) {

/**
* resolve all dependencies starting at the current working directory.
* @param {String} nodeModules - `node_modules` base directory.
* @param {Object} [targets=Object.create(null)] - resolved / active targets.
* @param {String} nodeModules - `node_modules` base directory.
* @param {Object} [targets=Object.create(null)] - resolved / active targets.
* @return {Observable} - an observable sequence of resolved dependencies.
*/
export function resolveAll (nodeModules, targets = Object.create(null)) {
Expand Down Expand Up @@ -166,7 +271,7 @@ function getDirectLink (dep) {

/**
* symlink the intermediate results of the underlying observable sequence
* @param {String} nodeModules - `node_modules` base directory.
* @param {String} nodeModules - `node_modules` base directory.
* @return {Observable} - empty observable sequence that will be completed
* once all dependencies have been symlinked.
*/
Expand All @@ -185,15 +290,17 @@ function checkShasum (shasum, expected, tarball) {
`shasum mismatch for ${tarball}: ${shasum} <-> ${expected}`)
}

function download (tarball, expected) {
function download (tarball, expected, type) {
log(`downloading ${tarball}, expecting ${expected}`)
return Observable.create((observer) => {
const errorHandler = (error) => observer.error(error)
const dataHandler = (chunk) => shasum.update(chunk)
const finishHandler = () => {
const actualShasum = shasum.digest('hex')
log(`downloaded ${actualShasum} into ${cached.path}`)
observer.next({ tmpPath: cached.path, shasum: actualShasum })
// only actually check shasum integrity for npm tarballs
const expectedShasum = ['range', 'version', 'tag'].indexOf(type) !== -1 ? actualShasum : expected
observer.next({ tmpPath: cached.path, shasum: expectedShasum })
observer.complete()
}

Expand Down Expand Up @@ -231,7 +338,7 @@ function fixPermissions (target, bin) {
}

function fetch (nodeModules) {
const {target, pkgJson: {name, bin, dist: {tarball, shasum} }} = this
const {target, type, pkgJson: {name, bin, dist: {tarball, shasum}}} = this
const where = path.join(nodeModules, target, 'package')

log(`fetching ${tarball} into ${where}`)
Expand All @@ -245,7 +352,7 @@ function fetch (nodeModules) {
throw error
}
return concatStatic(
download(tarball, shasum),
download(tarball, shasum, type),
cache.extract(where, shasum)
)
})
Expand All @@ -258,4 +365,3 @@ export function fetchAll (nodeModules) {
const fetch = (dep) => dep.fetch(nodeModules)::retry(config.retries)
return this::distinctKey('target')::mergeMap(fetch)
}

0 comments on commit e8ff294

Please sign in to comment.