diff --git a/package-lock.json b/package-lock.json index fcf44ef9..5ad4501a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -107,7 +107,7 @@ "dev": true, "requires": { "babel-runtime": "6.26.0", - "chalk": "2.4.0" + "chalk": "2.4.1" } }, "@commitlint/is-ignored": { @@ -247,7 +247,7 @@ "dev": true, "requires": { "ansi-styles": "3.2.1", - "chalk": "2.4.0", + "chalk": "2.4.1", "strip-ansi": "4.0.0", "supports-color": "5.4.0" } @@ -402,7 +402,7 @@ "integrity": "sha512-EpFH340Kg+zUndVuo6WoJBec1iNscpSUV53jGnLq94UqySp0SQXtHkvP1ZUMVUatvaOBIOtC5Y6EKOGHQtfCGg==", "requires": { "@oclif/linewrap": "1.0.0", - "chalk": "2.4.0" + "chalk": "2.4.1" } }, "@oclif/plugin-help": { @@ -449,7 +449,7 @@ "dev": true, "requires": { "@heroku-cli/color": "1.1.3", - "@oclif/command": "1.4.15", + "@oclif/command": "1.4.18", "string-similarity": "1.2.0" } }, @@ -459,10 +459,10 @@ "integrity": "sha512-4tA2utXhKZHrIlQYLzc+KftzW5U4eXIyAtpVgoON+Czrcmji1d1fO9XrL2190gfbdEx/qCCqIc/3NxZHop7mVQ==", "dev": true, "requires": { - "@oclif/command": "1.4.15", + "@oclif/command": "1.4.18", "@oclif/config": "1.6.13", "@oclif/errors": "1.0.6", - "chalk": "2.4.0", + "chalk": "2.4.1", "debug": "3.1.0", "fs-extra": "5.0.0", "http-call": "5.1.0", @@ -500,7 +500,7 @@ "execa": "0.10.0", "fs-extra": "5.0.0", "npm-run-path": "2.0.2", - "semantic-release": "15.1.7" + "semantic-release": "15.1.11" } }, "@oclif/test": { @@ -736,7 +736,7 @@ "integrity": "sha512-CCXv/Ar8/pcmtBdE1mqiN70BdSfCaqMe8f8GPR24URBei8KE5C8dydivV89uO91hffYLKFh7/m6Xrw/DHpbp0g==", "dev": true, "requires": { - "@types/node": "9.6.6" + "@types/node": "10.0.2" } }, "@types/glob": { @@ -747,7 +747,7 @@ "requires": { "@types/events": "1.2.0", "@types/minimatch": "3.0.3", - "@types/node": "9.6.6" + "@types/node": "10.0.2" } }, "@types/invariant": { @@ -774,7 +774,7 @@ "integrity": "sha512-cy3yebKhrHuOcrJGkfwNHhpTXQLgmXSv1BX+4p32j+VUQ6aP2eJ5cL7OvGcAQx75fCTFaAIIAKewvqL+iwSd4g==", "dev": true, "requires": { - "@types/node": "9.6.6" + "@types/node": "10.0.2" } }, "@types/node": { @@ -1224,6 +1224,11 @@ "regenerator-runtime": "0.11.1" } }, + "babylon": { + "version": "6.18.0", + "resolved": "https://registry.npmjs.org/babylon/-/babylon-6.18.0.tgz", + "integrity": "sha512-q/UEjfGJ2Cm3oKV71DJz9d25TPnq5rhBVL2Q4fA5wcC3jcrdn7+SssEybFIxwAvvP+YCsCYNKughoF33GxgycQ==" + }, "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", @@ -1438,6 +1443,14 @@ "integrity": "sha1-Jw8HbFpywC9bZaR9+Uxf46J4iS8=", "dev": true }, + "builtins": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/builtins/-/builtins-2.0.0.tgz", + "integrity": "sha512-8srrxpDx3a950BHYcbse+xMjupHHECvQYnShkoPz2ZLhTBrk/HQO6nWMh4o4ui8YYp2ourGVYXlGqFm+UYQwmA==", + "requires": { + "semver": "5.5.0" + } + }, "cache-base": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/cache-base/-/cache-base-1.0.1.tgz", @@ -1704,7 +1717,7 @@ "@oclif/screen": "1.0.2", "ansi-styles": "3.2.1", "cardinal": "1.0.0", - "chalk": "2.4.0", + "chalk": "2.4.1", "clean-stack": "1.3.0", "extract-stack": "1.0.0", "fs-extra": "5.0.0", @@ -2372,7 +2385,8 @@ "deep-extend": { "version": "0.4.2", "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.4.2.tgz", - "integrity": "sha1-SLaZwn4zS/ifEIkr5DL25MfTSn8=" + "integrity": "sha1-SLaZwn4zS/ifEIkr5DL25MfTSn8=", + "dev": true }, "deep-is": { "version": "0.1.3", @@ -2671,7 +2685,7 @@ "requires": { "ajv": "5.5.2", "babel-code-frame": "6.26.0", - "chalk": "2.4.0", + "chalk": "2.4.1", "concat-stream": "1.6.2", "cross-spawn": "5.1.0", "debug": "3.1.0", @@ -4107,7 +4121,7 @@ "dev": true, "requires": { "ansi-escapes": "3.1.0", - "chalk": "2.4.0", + "chalk": "2.4.1", "cli-cursor": "2.1.0", "cli-width": "2.2.0", "external-editor": "2.2.0", @@ -5435,6 +5449,11 @@ "integrity": "sha1-O23qo31g+xFnE8RsXxfqGQ7EjWQ=", "dev": true }, + "lodash.unescape": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/lodash.unescape/-/lodash.unescape-4.0.1.tgz", + "integrity": "sha1-vyJJiGzlFM2hEvrpIYzcBlIR/Jw=" + }, "lodash.upperfirst": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/lodash.upperfirst/-/lodash.upperfirst-4.3.1.tgz", @@ -5447,7 +5466,7 @@ "integrity": "sha512-VeIAFslyIerEJLXHziedo2basKbMKtTw3vfn5IzG0XTjhAVEJyNHnL2p7vc+wBDSdQuUpNw3M2u6xb9QsAY5Eg==", "dev": true, "requires": { - "chalk": "2.4.0" + "chalk": "2.4.1" } }, "log-update": { @@ -6170,6 +6189,14 @@ "integrity": "sha1-q4hOjn5X44qUR1POxwb3iNF2i7U=", "dev": true }, + "node-source-walk": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/node-source-walk/-/node-source-walk-3.3.0.tgz", + "integrity": "sha1-rRjjW/2z0Lb34OSv8eePhGo7iHM=", + "requires": { + "babylon": "6.18.0" + } + }, "normalize-package-data": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.4.0.tgz", @@ -6391,7 +6418,7 @@ "dev": true, "requires": { "@oclif/command": "1.4.15", - "chalk": "2.4.0", + "chalk": "2.4.1", "indent-string": "3.2.0", "lodash.template": "4.4.0", "string-width": "2.1.1", @@ -7213,7 +7240,7 @@ "integrity": "sha512-JL39c60XlzCVgNrO+qq68FoNb56w/m7JYvGR2jT5iR1xBrUA3Mfx5Twk5rqTThPmQKMWydGmq8oFtDlxfrmxnQ==", "dev": true, "requires": { - "rc": "1.2.6", + "rc": "1.2.7", "safe-buffer": "5.1.1" } }, @@ -7517,7 +7544,7 @@ "@semantic-release/npm": "3.2.4", "@semantic-release/release-notes-generator": "6.0.10", "aggregate-error": "1.0.0", - "chalk": "2.4.0", + "chalk": "2.4.1", "cosmiconfig": "4.0.0", "debug": "3.1.0", "env-ci": "2.0.1", @@ -8124,7 +8151,7 @@ "requires": { "ajv": "5.5.2", "ajv-keywords": "2.1.1", - "chalk": "2.4.0", + "chalk": "2.4.1", "lodash": "4.17.5", "slice-ansi": "1.0.0", "string-width": "2.1.1" @@ -8373,7 +8400,7 @@ "dev": true, "requires": { "arrify": "1.0.1", - "chalk": "2.4.0", + "chalk": "2.4.1", "diff": "3.5.0", "make-error": "1.3.4", "minimist": "1.2.0", @@ -8395,7 +8422,7 @@ "requires": { "babel-code-frame": "6.26.0", "builtin-modules": "1.1.1", - "chalk": "2.4.0", + "chalk": "2.4.1", "commander": "2.15.1", "diff": "3.5.0", "glob": "7.1.2", @@ -8537,6 +8564,15 @@ "integrity": "sha512-K7g15Bb6Ra4lKf7Iq2l/I5/En+hLIHmxWZGq3D4DIRNFxMNV6j2SHSvDOqs2tGd4UvD/fJvrwopzQXjLrT7Itw==", "dev": true }, + "typescript-eslint-parser": { + "version": "15.0.0", + "resolved": "https://registry.npmjs.org/typescript-eslint-parser/-/typescript-eslint-parser-15.0.0.tgz", + "integrity": "sha512-MMmSBcCaV5je+6DNinfnI2S6uwXSR6/TWR2phyzevqWDQKykobHc+f5eLDqK2sUpdg3T1Msd880o/xnRpAbG9Q==", + "requires": { + "lodash.unescape": "4.0.1", + "semver": "5.5.0" + } + }, "uglify-js": { "version": "2.8.29", "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-2.8.29.tgz", @@ -9060,7 +9096,7 @@ "integrity": "sha512-jzHBTTy8EPI4ImV8dpUMt+Q5zELkSU5xvGpndHcHudQ4tqN6YgIWaCGmRFl+HDchwRUkcgyjQ+n6/w5zlJBCPg==", "dev": true, "requires": { - "chalk": "2.4.0", + "chalk": "2.4.1", "debug": "3.1.0", "diff": "3.5.0", "escape-string-regexp": "1.0.5", @@ -9103,7 +9139,7 @@ "dev": true, "requires": { "async": "2.6.0", - "chalk": "2.4.0", + "chalk": "2.4.1", "cli-table": "0.3.1", "cross-spawn": "5.1.0", "dargs": "5.1.0", diff --git a/package.json b/package.json index 99c7db1b..ae9c41c1 100644 --- a/package.json +++ b/package.json @@ -40,18 +40,21 @@ "@oclif/config": "^1.6.4", "@oclif/errors": "^1.0.4", "@oclif/plugin-help": "^1.2.4", + "builtins": "^2.0.0", "chalk": "^2.4.0", "debug": "^3.1.0", "find-root": "^1.1.0", "glob": "^7.1.2", "invariant": "^2.2.4", "mz": "^2.7.0", + "node-source-walk": "^3.3.0", "pkg-up": "^2.0.0", "rc": "^1.2.6", "semver": "^5.5.0", "string": "^3.3.3", "supports-color": "^5.4.0", - "tslib": "^1.9.0" + "tslib": "^1.9.0", + "typescript-eslint-parser": "^15.0.0" }, "devDependencies": { "@commitlint/cli": "^6.1.3", @@ -113,7 +116,12 @@ "bin": "clark", "plugins": [ "@oclif/plugin-help" - ] + ], + "topics": { + "deps": { + "description": "Work with dependencies" + } + } }, "types": "lib/index.d.ts", "release": { diff --git a/src/commands/deps/generate.ts b/src/commands/deps/generate.ts new file mode 100644 index 00000000..8a234fb9 --- /dev/null +++ b/src/commands/deps/generate.ts @@ -0,0 +1,100 @@ +import {Command, flags} from '@oclif/command'; + +import {format as f} from '../../lib/debug'; +import {generate} from '../../lib/deps'; +import {apply} from '../../lib/packages'; + +/** + * Walk the dependency tree and copy deps from the root package.json to each + * subpackage as appropriate + */ +export default class DepsGenerate extends Command { + /** + * aliases + */ + static aliases = ['deps:generate']; + + /** + * description + */ + static description = 'Generate package depencies'; + + /** + * flags + */ + static flags = { + failFast: flags.boolean({ + description: 'Stop on first failure', + }), + 'fail-fast': flags.boolean({ + description: 'Alias of --failFast', + }), + packageName: flags.string({ + char: 'p', + description: + 'The package for which to generate dependencies. May be specified more than once', + multiple: true, + }), + package: flags.string({ + description: 'alias of --packageName', + multiple: true, + }), + 'package-name': flags.string({ + description: 'alias of --packageName', + multiple: true, + }), + silent: flags.boolean({ + char: 's', + description: 'Indicates nothing should be printed to the stdout', + }), + }; + + /** + * implementation + */ + async run() { + const {flags, args} = this.parse(DepsGenerate); + + flags.packageName = ([] as string[]) + .concat(flags.packageName) + .concat(flags['package-name']) + .concat(flags.package) + .filter(Boolean); + + if (!flags.packageName.length) { + delete flags.packageName; + } + + flags.failFast = flags.failFast || flags['fail-fast']; + + const options = { + ...flags, + ...args, + }; + + await apply( + { + before: (packages) => + f`Generating dependencies for ${packages.length} packages`, + beforeEach: (packageName) => + f`Generating dependencies for ${packageName}`, + afterEach: (packageName, error) => { + if (error) { + return f`Failed to generate deps for ${packageName}`; + } + + return f`Generated deps for ${packageName}`; + }, + after: (packages, errors) => { + if (errors.length) { + return f`Failed to generate deps for ${errors.length} packages`; + } + + return f`Generated dependencies for ${packages.length} packages`; + }, + }, + generate, + options, + ); + } +} diff --git a/src/lib/deps.ts b/src/lib/deps.ts new file mode 100644 index 00000000..1245aafb --- /dev/null +++ b/src/lib/deps.ts @@ -0,0 +1,154 @@ +import {existsSync, readFileSync} from 'fs'; +import {dirname, resolve} from 'path'; + +import {format as f, makeDebug} from './debug'; +import detective from './detective'; +import {findEntryPoints, read, write} from './packages'; +import {read as readProject} from './project'; + +const debug = makeDebug(__filename); + +const extensions = ['js', 'jsx', 'ts', 'tsx', 'mjs']; + +const visited = new Map(); + +interface VersionedDependencies { + [key: string]: string; +} + +/** + * Adds version strings from the root package.json to the passed in set of + * dependencies.s + * @param deps + */ +async function addVersionsToDeps( + deps: string[], +): Promise { + const proj = await readProject(); + return deps.reduce( + (acc, dep) => { + acc[dep] = proj.dependencies[dep]; + return acc; + }, + {} as VersionedDependencies, + ); +} + +/** + * Strips path segments off of require statements + * @param requires + */ +function convertRequiresToDeps(requires: string[]): string[] { + requires = Array.from(new Set(requires)) + .filter((r) => !r.startsWith('.')) + .map((r) => { + // The following block makes sure the dep is a package name and not a file + // reference. Given a require of `@scope/foo/bar/baz`, the following will + // return `@scope/foo`. Given a require of `foo/bar/baz`, the folling will + // return `foo`. + const rr = r.split('/'); + if (rr[0].startsWith('@')) { + return rr.slice(0, 2).join('/'); + } + return rr[0]; + }); + return requires; +} + +/** + * Finds all of the require/import statements for specified file *and its local + * dependencies* + * @param filePath + */ +function findRequires(filePath: string): string[] { + if (visited.has(filePath)) { + debug(f`Already visited ${filePath}`); + return visited.get(filePath); + } + + try { + debug(f`Finding requires for ${filePath}`); + const requires = detective(loadSource(filePath)); + debug(f`Found ${requires.length} requires for ${filePath}`); + + visited.set(filePath, []); + + visited.set( + filePath, + requires.reduce( + (acc, req) => { + debug(f`Found ${req}`); + if (req.startsWith('.')) { + debug(f`${req} is relative, descending`); + const next = findRequires(resolve(dirname(filePath), req)); + return Array.from(new Set([...acc, ...next])); + } + return acc; + }, + [] as string[], + ), + ); + + return requires; + } catch (err) { + if (err.code === 'EISDIR') { + debug(f`${filePath} is a directory, descending`); + return findRequires(resolve(filePath, 'index')); + } + + debug(f`An unexpected error occurred while walking ${filePath}`); + throw err; + } +} + +/** + * Generate dependencies for the specified package by walking the code found at + * its entrypoints. + * @param packageName + */ +export async function generate(packageName: string) { + const deps = await list(packageName); + + const versionedDeps = await addVersionsToDeps(deps); + + const pkg = await read(packageName); + pkg.dependencies = versionedDeps; + await write(packageName, pkg); +} + +/** + * Lists dependencies for the specified package + * @param packageName + */ +export async function list(packageName: string): Promise { + const entrypoints = await findEntryPoints(packageName); + + const requires = entrypoints.reduce( + (acc, entrypoint) => { + return acc.concat(findRequires(entrypoint)); + }, + [] as string[], + ); + + return convertRequiresToDeps(requires).sort(); +} + +/** + * Loads a source file + * @param filePath + */ +function loadSource(filePath: string): string { + if (!existsSync(filePath)) { + for (const ext of extensions) { + const withExt = `${filePath}.${ext}`; + if (existsSync(withExt)) { + return loadSource(withExt); + } + } + + debug(f`Could not find node module identified by ${filePath}`); + throw new Error(`Could not find node module identified by ${filePath}`); + } + + return readFileSync(filePath, 'utf-8'); +} diff --git a/src/lib/detective.ts b/src/lib/detective.ts new file mode 100644 index 00000000..840a0aae --- /dev/null +++ b/src/lib/detective.ts @@ -0,0 +1,91 @@ +import Walker from 'node-source-walk'; +import TypescriptParser from 'typescript-eslint-parser'; + +import {format as f, makeDebug} from './debug'; + +const debug = makeDebug(__filename); + +/** + * Given a piece of sourcecode, locates its import statements + * @param src + */ +export default function detective(src: string) { + if (src === '') { + return []; + } + + try { + debug(f`Attempting to parse src as TypeScript`); + const walker = new Walker({ + ecmaFeatures: { + jsx: true, + }, + parser: TypescriptParser, + }); + const result = walk(walker, src); + debug(f`Successfully parsed src as TypeScript`); + return result; + } catch (err) { + debug(f`Failed to parse src as TypeScript`); + debug(err); + debug(f`Attempting to parse src as ECMAScript`); + const walker = new Walker({ + ecmaFeatures: { + jsx: true, + }, + }); + const result = walk(walker, src); + debug(f`Failed to parse src as TypeScript`); + return result; + } +} + +/** + * placehoder. At some future point this should probably be documented. + */ +type Node = any; + +/** + * Uses walker to search src for import/require statements. + * @param walker + * @param src + */ +function walk(walker: Walker, src: string) { + const dependencies: string[] = []; + walker.walk(src, function(node: Node) { + switch (node.type) { + case 'ImportDeclaration': + if (node.source && node.source.value) { + dependencies.push(node.source.value); + } + break; + case 'ExportNamedDeclaration': + case 'ExportAllDeclaration': + if (node.source && node.source.value) { + dependencies.push(node.source.value); + } + break; + case 'TSExternalModuleReference': + if (node.expression && node.expression.value) { + dependencies.push(node.expression.value); + } + break; + case 'CallExpression': + if (node.callee.type === 'Import' && node.arguments.length) { + dependencies.push(node.arguments[0].value); + } + + if ( + node.callee.name === 'require' && + node.arguments[0].type === 'Literal' + ) { + dependencies.push(node.arguments[0].value); + } + + break; + default: + return; + } + }); + return dependencies; +} diff --git a/src/lib/packages.ts b/src/lib/packages.ts index e3c67b36..df59be05 100644 --- a/src/lib/packages.ts +++ b/src/lib/packages.ts @@ -1,5 +1,5 @@ import {sync as glob} from 'glob'; -import {readFile, writeFile} from 'mz/fs'; +import {exists, existsSync, readFile, writeFile} from 'mz/fs'; import {dirname, resolve} from 'path'; import {load} from './config'; @@ -208,6 +208,74 @@ function filterEnv(env: object): object { }, {}); } +/** + * Lists all the entry points for the specified package + * @param packageName + */ +export async function findEntryPoints(packageName: string): Promise { + const pkg = await read(packageName); + + debug(f`listing entrypoints for ${pkg.name}`); + if (!pkg.name) { + throw new Error('cannot read dependencies for unnamed package'); + } + + let paths = []; + + if (pkg.main) { + debug(f`found main path for ${pkg.name}`); + paths.push(pkg.main); + } + + if (pkg.bin) { + debug(f`found bin entry(s) for ${pkg.name}`); + paths = paths.concat(Object.values(pkg.bin)); + } + + if (pkg.browser) { + debug(f`found browser entry(s) for ${pkg.name}`); + paths = paths.concat( + Object.values(pkg.browser as { + [key: string]: string; + }).filter((p) => p && !p.startsWith('@')), + ); + } + + const packagePath = await getPackagePath(packageName); + const tsconfigPath = resolve(packagePath, 'tsconfig.json'); + + debug('checking if this is a typescript project'); + if (await exists(tsconfigPath)) { + debug('this is a typescript project'); + debug('using tsconfig.json to find all entrypoints'); + const tsconfig = JSON.parse(await readFile(tsconfigPath, 'utf-8')) as { + include: string[]; + }; + + for (const pattern of tsconfig.include) { + paths = paths.concat( + glob(pattern, {cwd: packagePath, nodir: true}).filter( + (p) => p.endsWith('.ts') || p.endsWith('.tsx'), + ), + ); + } + } + + debug(paths); + + const testPattern = /[\.-]spec|test\.[jt]sx?$/; + return ( + paths + .map((p) => resolve(packagePath, p)) + // filter out test files + .filter((p) => !testPattern.test(p)) + // filter out files that don't exist. this may happen, in particular, in + // scenarios where we're relying on tsconfig.json to identify source files + // instead of main/bin to identify built artifacts. + .filter((p) => existsSync(p)) + ); +} + /** * Higher-level version of that "does the right thing" whether packageName is * provided or not. diff --git a/src/types/builtins.d.ts b/src/types/builtins.d.ts new file mode 100644 index 00000000..6adff1c0 --- /dev/null +++ b/src/types/builtins.d.ts @@ -0,0 +1,10 @@ +/** + * Local type defintion for builtins + */ +declare module 'builtins' { + /** + * builtins + */ + function builtins(): string[]; + export = builtins; +} diff --git a/src/types/node-source-walk.d.ts b/src/types/node-source-walk.d.ts new file mode 100644 index 00000000..9a160f6f --- /dev/null +++ b/src/types/node-source-walk.d.ts @@ -0,0 +1,45 @@ +/** + * Local type defintion for node-source-walk + */ +declare module 'node-source-walk' { + /** + * walker options + */ + interface WalkerOptions { + ecmaFeatures: EcmaFeatures; + parser?: Parser; + } + + /** + * ecma features + */ + interface EcmaFeatures { + jsx: true; + } + + /** + * node + */ + type Node = any; + + /** + * parser + */ + type Parser = any; + + /** + * walker + */ + class Walker { + constructor(options?: WalkerOptions); + + /** + * walk + * @param src + * @param callback + */ + walk(src: string, callback: (node: Node) => void): void; + } + + export = Walker; +} diff --git a/src/types/typescript-eslint-parser.d.ts b/src/types/typescript-eslint-parser.d.ts new file mode 100644 index 00000000..43d00801 --- /dev/null +++ b/src/types/typescript-eslint-parser.d.ts @@ -0,0 +1,6 @@ +/** + * Local type defintion for typescript-eslint-parser + */ +declare module 'typescript-eslint-parser' { + +}