diff --git a/package.json b/package.json index d34e1cd886e975..aa827278e642bd 100644 --- a/package.json +++ b/package.json @@ -220,7 +220,6 @@ "ncp": "2.0.0", "node-sass": "3.8.0", "nock": "8.0.0", - "npm": "3.10.10", "portscanner": "1.0.0", "proxyquire": "1.7.10", "react": "15.2.0", diff --git a/tasks/build/notice.js b/tasks/build/notice.js index b546902a96a033..cc99182a414992 100644 --- a/tasks/build/notice.js +++ b/tasks/build/notice.js @@ -1,86 +1,35 @@ -import _ from 'lodash'; -import npm from 'npm'; -import npmLicense from 'license-checker'; -import glob from 'glob'; -import path from 'path'; -import fs from 'fs'; -import { execSync } from 'child_process'; +import { resolve } from 'path'; + +import { + getInstalledPackages, + generateNoticeText, +} from '../lib'; + +async function generate(grunt, directory) { + return await generateNoticeText({ + packages: await getInstalledPackages({ + directory, + licenseOverrides: grunt.config.get('licenses.options.overrides') + }), + nodeDir: grunt.config.get('platforms')[0].nodeDir + }); +} -export default function licenses(grunt) { +export default function (grunt) { grunt.registerTask('_build:notice', 'Adds a notice', function () { const done = this.async(); - const buildPath = path.join(grunt.config.get('buildDir'), 'kibana'); - - function getPackagePaths() { - const packagePaths = {}; - const installedPackages = execSync(`npm ls --parseable --long`, { - cwd: buildPath - }); - installedPackages.toString().trim().split('\n').forEach(pkg => { - const packageDetails = pkg.split(':'); - const [modulePath, packageName] = packageDetails; - const licenses = glob.sync(path.join(modulePath, '*LICENSE*')); - const notices = glob.sync(path.join(modulePath, '*NOTICE*')); - packagePaths[packageName] = { - relative: modulePath.replace(/.*\/kibana\//, ''), - licenses, - notices - }; - }); - return packagePaths; - } - - function combineFiles(filePaths) { - let content = ''; - filePaths.forEach(filePath => { - content += fs.readFileSync(filePath) + '\n'; - }); - return content; - } - - function getNodeInfo() { - const nodeVersion = grunt.config.get('nodeVersion'); - const nodeDir = path.join(grunt.config.get('root'), '.node_binaries', nodeVersion); - const licensePath = path.join(nodeDir, 'linux-x64', 'LICENSE'); - const license = fs.readFileSync(licensePath); - return `This product bundles Node.js.\n\n${license}`; - } - - function getPackageInfo(packages) { - const packagePaths = getPackagePaths(); - const overrides = grunt.config.get('licenses.options.overrides'); - let content = ''; - _.forOwn(packages, (value, key) => { - const licenses = [].concat(overrides.hasOwnProperty(key) ? overrides[key] : value.licenses); - if (!licenses.length || licenses.includes('UNKNOWN')) return grunt.fail.fatal(`Unknown license for ${key}`); - const packagePath = packagePaths[key]; - const readLicenseAndNotice = combineFiles([].concat(packagePath.licenses, packagePath.notices)); - const licenseOverview = licenses.length > 1 ? `the\n"${licenses.join('", ')} licenses` : `a\n"${licenses[0]}" license`; - const licenseAndNotice = readLicenseAndNotice ? `\n${readLicenseAndNotice}` : ` For details, see ${packagePath.relative}/.`; - const combinedText = `This product bundles ${key} which is available under ${licenseOverview}.${licenseAndNotice}\n---\n`; - - content += combinedText; - }); - return content; - } - - function getBaseNotice() { - return fs.readFileSync(path.join(__dirname, 'notice', 'base_notice.txt')); - } - - npmLicense.init({ - start: buildPath, - production: true, - json: true - }, (result, error) => { - if (error) return grunt.fail.fatal(error); - const noticePath = path.join(buildPath, 'NOTICE.txt'); - const fd = fs.openSync(noticePath, 'w'); - fs.appendFileSync(fd, getBaseNotice()); - fs.appendFileSync(fd, getPackageInfo(result)); - fs.appendFileSync(fd, getNodeInfo()); - fs.closeSync(fd); - done(); - }); + const kibanaDir = resolve(grunt.config.get('buildDir'), 'kibana'); + const noticePath = resolve(kibanaDir, 'NOTICE.txt'); + + generate(grunt, kibanaDir).then( + (noticeText) => { + grunt.file.write(noticePath, noticeText); + done(); + }, + (error) => { + grunt.fail.fatal(error); + done(error); + } + ); }); } diff --git a/tasks/config/simplemocha.js b/tasks/config/simplemocha.js index 4c655d9b37473c..6df38f778082a9 100644 --- a/tasks/config/simplemocha.js +++ b/tasks/config/simplemocha.js @@ -12,6 +12,7 @@ module.exports = { src: [ 'test/**/__tests__/**/*.js', 'src/**/__tests__/**/*.js', + 'tasks/**/__tests__/**/*.js', 'test/fixtures/__tests__/*.js', '!src/**/public/**', '!**/_*.js' diff --git a/tasks/lib/index.js b/tasks/lib/index.js new file mode 100644 index 00000000000000..b7af0e2c282352 --- /dev/null +++ b/tasks/lib/index.js @@ -0,0 +1,3 @@ +export { generateNoticeText } from './notice'; +export { getInstalledPackages } from './packages'; +export { assertLicensesValid } from './licenses'; diff --git a/tasks/lib/licenses/__tests__/valid.js b/tasks/lib/licenses/__tests__/valid.js new file mode 100644 index 00000000000000..f31326036f9a4e --- /dev/null +++ b/tasks/lib/licenses/__tests__/valid.js @@ -0,0 +1,62 @@ +import { resolve } from 'path'; + +import expect from 'expect.js'; + +import { assertLicensesValid } from '../valid'; + +const NODE_MODULES = resolve(__dirname, '../../../../node_modules'); + +const PACKAGE = { + name: '@elastic/httpolyglot', + version: '0.1.2-elasticpatch1', + licenses: ['MIT'], + directory: resolve(NODE_MODULES, '@elastic/httpolyglot'), + relative: 'node_modules/@elastic/httpolyglot', +}; + +describe('tasks/lib/licenses', () => { + describe('assertLicensesValid()', () => { + it('returns undefined when package has valid license', () => { + expect(assertLicensesValid({ + packages: [PACKAGE], + validLicenses: [...PACKAGE.licenses] + })).to.be(undefined); + }); + + it('throw an error when the packages license is invalid', () => { + expect(() => { + assertLicensesValid({ + packages: [PACKAGE], + validLicenses: [`not ${PACKAGE.licenses[0]}`] + }); + }).to.throwError(PACKAGE.name); + }); + + it('throws an error when the package has no licenses', () => { + expect(() => { + assertLicensesValid({ + packages: [ + { + ...PACKAGE, + licenses: [] + } + ], + validLicenses: [...PACKAGE.licenses] + }); + }).to.throwError(PACKAGE.name); + }); + + it('includes the relative path to packages in error message', () => { + try { + assertLicensesValid({ + packages: [PACKAGE], + validLicenses: ['none'] + }); + throw new Error('expected assertLicensesValid() to throw'); + } catch (error) { + expect(error.message).to.contain(PACKAGE.relative); + expect(error.message).to.not.contain(PACKAGE.directory); + } + }); + }); +}); diff --git a/tasks/lib/licenses/index.js b/tasks/lib/licenses/index.js new file mode 100644 index 00000000000000..eb0c17b965b5d9 --- /dev/null +++ b/tasks/lib/licenses/index.js @@ -0,0 +1 @@ +export { assertLicensesValid } from './valid'; diff --git a/tasks/lib/licenses/valid.js b/tasks/lib/licenses/valid.js new file mode 100644 index 00000000000000..f5d62ed281e8e7 --- /dev/null +++ b/tasks/lib/licenses/valid.js @@ -0,0 +1,47 @@ +const describeInvalidLicenses = getInvalid => pkg => ( +` + ${pkg.name} + version: ${pkg.version} + all licenses: ${pkg.licenses} + invalid licenses: ${getInvalid(pkg.licenses).join(', ')} + path: ${pkg.relative} +` +); + +/** + * When given a list of packages and the valid license + * options, either throws an error with details about + * violations or returns undefined. + * + * @param {Object} [options={}] + * @property {Array} options.packages List of packages to check, see + * getInstalledPackages() in ../packages + * @property {Array} options.validLicenses + * @return {undefined} + */ +export function assertLicensesValid(options = {}) { + const { + packages, + validLicenses + } = options; + + if (!packages || !validLicenses) { + throw new Error('packages and validLicenses options are required'); + } + + const getInvalid = licenses => ( + licenses.filter(license => !validLicenses.includes(license)) + ); + + const isPackageInvalid = pkg => ( + !pkg.licenses.length || getInvalid(pkg.licenses).length > 0 + ); + + const invalidMsgs = packages + .filter(isPackageInvalid) + .map(describeInvalidLicenses(getInvalid)); + + if (invalidMsgs.length) { + throw new Error(`Non-confirming licenses: ${invalidMsgs.join('')}`); + } +} diff --git a/tasks/lib/notice/__tests__/notice.js b/tasks/lib/notice/__tests__/notice.js new file mode 100644 index 00000000000000..4c8e275210ad7d --- /dev/null +++ b/tasks/lib/notice/__tests__/notice.js @@ -0,0 +1,55 @@ +import { resolve } from 'path'; +import { readFileSync } from 'fs'; + +import expect from 'expect.js'; + +import { generateNoticeText } from '../notice'; + +const NODE_MODULES = resolve(__dirname, '../../../../node_modules'); +const NODE_DIR = resolve(process.execPath, '../..'); +const PACKAGES = [ + { + name: '@elastic/httpolyglot', + version: '0.1.2-elasticpatch1', + licenses: ['MIT'], + directory: resolve(NODE_MODULES, '@elastic/httpolyglot'), + relative: 'node_modules/@elastic/httpolyglot', + }, + { + name: 'aws-sdk', + version: '2.0.31', + licenses: ['Apache 2.0'], + directory: resolve(NODE_MODULES, 'aws-sdk'), + relative: 'node_modules/aws-sdk', + } +]; + +describe('tasks/lib/notice', () => { + describe('generateNoticeText()', () => { + let notice; + before(async () => notice = await generateNoticeText({ + packages: PACKAGES, + nodeDir: NODE_DIR + })); + + it('returns a string', () => { + expect(notice).to.be.a('string'); + }); + + it('includes *NOTICE* files from packages', () => { + expect(notice).to.contain(readFileSync(resolve(NODE_MODULES, 'aws-sdk/NOTICE.txt'), 'utf8')); + }); + + it('includes *LICENSE* files from packages', () => { + expect(notice).to.contain(readFileSync(resolve(NODE_MODULES, '@elastic/httpolyglot/LICENSE'), 'utf8')); + }); + + it('includes the LICENSE file from node', () => { + expect(notice).to.contain(readFileSync(resolve(NODE_DIR, 'LICENSE'), 'utf8')); + }); + + it('includes the base_notice.txt file', () => { + expect(notice).to.contain(readFileSync(resolve(__dirname, '../base_notice.txt'), 'utf8')); + }); + }); +}); diff --git a/tasks/build/notice/base_notice.txt b/tasks/lib/notice/base_notice.txt similarity index 99% rename from tasks/build/notice/base_notice.txt rename to tasks/lib/notice/base_notice.txt index 19f505cdfe61bf..13cb1a0121529b 100644 --- a/tasks/build/notice/base_notice.txt +++ b/tasks/lib/notice/base_notice.txt @@ -56,4 +56,3 @@ THE SOFTWARE. --- This product bundles geohash.js which is available under a "MIT" license. For details, see src/ui/public/utils/decode_geo_hash.js. ---- diff --git a/tasks/lib/notice/bundled_notices.js b/tasks/lib/notice/bundled_notices.js new file mode 100644 index 00000000000000..c573c7a1b7ead4 --- /dev/null +++ b/tasks/lib/notice/bundled_notices.js @@ -0,0 +1,14 @@ +import { resolve } from 'path'; +import { readFile } from 'fs'; + +import { fromNode as fcb } from 'bluebird'; +import glob from 'glob'; + +export async function getBundledNotices(packageDirectory) { + const pattern = resolve(packageDirectory, '*{LICENSE,NOTICE}*'); + const paths = await fcb(cb => glob(pattern, cb)); + return Promise.all(paths.map(async path => ({ + path, + text: await fcb(cb => readFile(path, 'utf8', cb)) + }))); +} diff --git a/tasks/lib/notice/index.js b/tasks/lib/notice/index.js new file mode 100644 index 00000000000000..acd902a3eee02b --- /dev/null +++ b/tasks/lib/notice/index.js @@ -0,0 +1 @@ +export { generateNoticeText } from './notice'; diff --git a/tasks/lib/notice/node_notice.js b/tasks/lib/notice/node_notice.js new file mode 100644 index 00000000000000..1503ff8781172f --- /dev/null +++ b/tasks/lib/notice/node_notice.js @@ -0,0 +1,8 @@ +import { resolve } from 'path'; +import { readFileSync } from 'fs'; + +export function generateNodeNoticeText(nodeDir) { + const licensePath = resolve(nodeDir, 'LICENSE'); + const license = readFileSync(licensePath, 'utf8'); + return `This product bundles Node.js.\n\n${license}`; +} diff --git a/tasks/lib/notice/notice.js b/tasks/lib/notice/notice.js new file mode 100644 index 00000000000000..f57129d6bbdcf5 --- /dev/null +++ b/tasks/lib/notice/notice.js @@ -0,0 +1,33 @@ +import { resolve } from 'path'; +import { readFileSync } from 'fs'; + +import { generatePackageNoticeText } from './package_notice'; +import { generateNodeNoticeText } from './node_notice'; + +const BASE_NOTICE = resolve(__dirname, './base_notice.txt'); + +/** + * When given a list of packages and the directory to the + * node distribution that will be shipping with Kibana, + * generates the text for NOTICE.txt + * + * @param {Object} [options={}] + * @property {Array} options.packages List of packages to check, see + * getInstalledPackages() in ../packages + * @property {string} options.nodeDir The directory containing the version of node.js + * that will ship with Kibana + * @return {undefined} + */ +export async function generateNoticeText(options = {}) { + const { packages, nodeDir } = options; + + const packageNotices = await Promise.all( + packages.map(generatePackageNoticeText) + ); + + return [ + readFileSync(BASE_NOTICE, 'utf8'), + ...packageNotices, + generateNodeNoticeText(nodeDir), + ].join('\n---\n'); +} diff --git a/tasks/lib/notice/package_notice.js b/tasks/lib/notice/package_notice.js new file mode 100644 index 00000000000000..66759fc99747ed --- /dev/null +++ b/tasks/lib/notice/package_notice.js @@ -0,0 +1,22 @@ +import { getBundledNotices } from './bundled_notices'; + +const concatNotices = notices => ( + notices.map(notice => notice.text).join('\n') +); + +export async function generatePackageNoticeText(pkg) { + const bundledNotices = concatNotices(await getBundledNotices(pkg.directory)); + + const intro = `This product bundles ${pkg.name}@${pkg.version}`; + const license = ` which is available under ${ + pkg.licenses.length > 1 + ? `the\n"${pkg.licenses.join('", ')} licenses.` + : `a\n"${pkg.licenses[0]}" license.` + }`; + + const moreInfo = bundledNotices + ? `\n${bundledNotices}\n` + : ` For details, see ${pkg.relative}/.`; + + return `${intro}${license}${moreInfo}`; +} diff --git a/tasks/lib/packages/__tests__/fixtures/fixture1/index.js b/tasks/lib/packages/__tests__/fixtures/fixture1/index.js new file mode 100644 index 00000000000000..24b00f58ebd36b --- /dev/null +++ b/tasks/lib/packages/__tests__/fixtures/fixture1/index.js @@ -0,0 +1 @@ +console.log('I am fixture 1'); diff --git a/tasks/lib/packages/__tests__/fixtures/fixture1/node_modules/dep1/index.js b/tasks/lib/packages/__tests__/fixtures/fixture1/node_modules/dep1/index.js new file mode 100644 index 00000000000000..17aefb3c74b917 --- /dev/null +++ b/tasks/lib/packages/__tests__/fixtures/fixture1/node_modules/dep1/index.js @@ -0,0 +1 @@ +console.log('I am dep 1'); diff --git a/tasks/lib/packages/__tests__/fixtures/fixture1/node_modules/dep1/package.json b/tasks/lib/packages/__tests__/fixtures/fixture1/node_modules/dep1/package.json new file mode 100644 index 00000000000000..a2ccb021abcb97 --- /dev/null +++ b/tasks/lib/packages/__tests__/fixtures/fixture1/node_modules/dep1/package.json @@ -0,0 +1,5 @@ +{ + "name": "dep1", + "version": "0.0.2", + "license": "Apache-2.0" +} diff --git a/tasks/lib/packages/__tests__/fixtures/fixture1/package.json b/tasks/lib/packages/__tests__/fixtures/fixture1/package.json new file mode 100644 index 00000000000000..d2d533591bcfd5 --- /dev/null +++ b/tasks/lib/packages/__tests__/fixtures/fixture1/package.json @@ -0,0 +1,8 @@ +{ + "name": "fixture1", + "version": "0.0.1", + "license": "MIT", + "dependencies": { + "dep1": "0.0.2" + } +} diff --git a/tasks/lib/packages/__tests__/installed_packages.js b/tasks/lib/packages/__tests__/installed_packages.js new file mode 100644 index 00000000000000..4c47412a2a1f29 --- /dev/null +++ b/tasks/lib/packages/__tests__/installed_packages.js @@ -0,0 +1,59 @@ +import { resolve } from 'path'; + +import { uniq } from 'lodash'; +import expect from 'expect.js'; + +import { getInstalledPackages } from '../installed_packages'; + +const KIBANA_ROOT = resolve(__dirname, '../../../../'); +const FIXTURE1_ROOT = resolve(__dirname, 'fixtures/fixture1'); + +describe('tasks/lib/packages', () => { + describe('getInstalledPackages()', function () { + + let kibanaPackages; + let fixture1Packages; + before(async function () { + this.timeout(30 * 1000); + [kibanaPackages, fixture1Packages] = await Promise.all([ + getInstalledPackages({ + directory: KIBANA_ROOT + }), + getInstalledPackages({ + directory: FIXTURE1_ROOT + }), + ]); + }); + + it('requires a directory', async () => { + try { + await getInstalledPackages({}); + throw new Error('expected getInstalledPackages() to reject'); + } catch (err) { + expect(err.message).to.contain('directory'); + } + }); + + it('reads all installed packages of a module', () => { + expect(fixture1Packages).to.eql([ + { + name: 'dep1', + version: '0.0.2', + licenses: [ 'Apache-2.0' ], + directory: resolve(FIXTURE1_ROOT, 'node_modules/dep1'), + relative: 'node_modules/dep1', + } + ]); + }); + + it('returns a single entry for every package/version combo', () => { + const tags = kibanaPackages.map(pkg => `${pkg.name}@${pkg.version}`); + expect(tags).to.eql(uniq(tags)); + }); + + it('does not include root package in the list', async () => { + expect(kibanaPackages.find(pkg => pkg.name === 'kibana')).to.be(undefined); + expect(fixture1Packages.find(pkg => pkg.name === 'fixture1')).to.be(undefined); + }); + }); +}); diff --git a/tasks/lib/packages/index.js b/tasks/lib/packages/index.js new file mode 100644 index 00000000000000..aa3f5195afc4ee --- /dev/null +++ b/tasks/lib/packages/index.js @@ -0,0 +1 @@ +export { getInstalledPackages } from './installed_packages'; diff --git a/tasks/lib/packages/installed_packages.js b/tasks/lib/packages/installed_packages.js new file mode 100644 index 00000000000000..e34feb7478c4cd --- /dev/null +++ b/tasks/lib/packages/installed_packages.js @@ -0,0 +1,55 @@ +import { relative } from 'path'; + +import { callLicenseChecker } from './license_checker'; + +/** + * Get a list of objects with details about each installed + * NPM package. + * + * @param {Object} [options={}] + * @property {String} options.directory root of the project to read + * @property {Boolean} [options.dev=false] should development dependencies be included? + * @property {Object} [options.licenseOverrides] map of `${name}@${version}` to a list of + * license ids to override the automatically + * detected ones + * @return {Array} + */ +export async function getInstalledPackages(options = {}) { + const { + directory, + dev = false, + licenseOverrides = {} + } = options; + + if (!directory) { + throw new Error('You must specify a directory to read installed packages from'); + } + + const licenseInfo = await callLicenseChecker({ directory, dev }); + return Object + .keys(licenseInfo) + .map(key => { + const keyParts = key.split('@'); + const name = keyParts.slice(0, -1).join('@'); + const version = keyParts[keyParts.length - 1]; + const { + licenses: detectedLicenses, + realPath, + } = licenseInfo[key]; + + const licenses = [].concat( + licenseOverrides[key] + ? licenseOverrides[key] + : detectedLicenses + ); + + return { + name, + version, + licenses, + directory: realPath, + relative: relative(directory, realPath) + }; + }) + .filter(pkg => pkg.directory !== directory); +} diff --git a/tasks/lib/packages/license_checker.js b/tasks/lib/packages/license_checker.js new file mode 100644 index 00000000000000..127f9474ca5661 --- /dev/null +++ b/tasks/lib/packages/license_checker.js @@ -0,0 +1,26 @@ +import licenseChecker from 'license-checker'; + +export function callLicenseChecker(options = {}) { + const { + directory, + dev = false + } = options; + + if (!directory) { + throw new Error('You must specify the directory where license checker should start'); + } + + return new Promise((resolve, reject) => { + licenseChecker.init({ + start: directory, + production: !dev, + json: true, + customFormat: { + realPath: true + } + }, (licenseInfo, err) => { + if (err) reject(err); + else resolve(licenseInfo); + }); + }); +} diff --git a/tasks/licenses.js b/tasks/licenses.js index d18c7698cb4257..b1d9de855f03a4 100644 --- a/tasks/licenses.js +++ b/tasks/licenses.js @@ -1,77 +1,29 @@ -import _ from 'lodash'; -import { fromNode } from 'bluebird'; -import npm from 'npm'; -import npmLicense from 'license-checker'; +import { + getInstalledPackages, + assertLicensesValid +} from './lib'; export default function licenses(grunt) { grunt.registerTask('licenses', 'Checks dependency licenses', async function () { - const config = this.options(); const done = this.async(); - const result = []; - const options = { - start: process.cwd(), - production: true, - json: true - }; - - const packages = await fromNode(cb => { - npmLicense.init(options, (result, error) => { - cb(undefined, result); + try { + const options = this.options({ + licenses: [], + overrides: {} }); - }); - - /** - * Licenses for a package by name with overrides - * - * @param {String} name - * @return {Array} - */ - - function licensesForPackage(name) { - let licenses = packages[name].licenses; - - if (config.overrides.hasOwnProperty(name)) { - licenses = config.overrides[name]; - } - - return typeof licenses === 'string' ? [licenses] : licenses; - } - - /** - * Determine if a package has a valid license - * - * @param {String} name - * @return {Boolean} - */ - - function isInvalidLicense(name) { - const licenses = licensesForPackage(name); - // verify all licenses for the package are in the config - return _.intersection(licenses, config.licenses).length < licenses.length; - } - - // Build object containing only invalid packages - const invalidPackages = _.pick(packages, (pkg, name) => { - return isInvalidLicense(name); - }); - - if (Object.keys(invalidPackages).length) { - const util = require('util'); - const execSync = require('child_process').execSync; - const names = Object.keys(invalidPackages); - - // Uses npm ls to create tree for package locations - const tree = execSync(`npm ls ${names.join(' ')}`); - - grunt.log.debug(JSON.stringify(invalidPackages, null, 2)); - grunt.fail.warn( - `Non-confirming licenses:\n ${names.join('\n ')}\n\n${tree}`, - invalidPackages.length - ); + assertLicensesValid({ + packages: await getInstalledPackages({ + directory: grunt.config.get('root'), + licenseOverrides: options.overrides + }), + validLicenses: options.licenses + }); + done(); + } catch (err) { + grunt.fail.fatal(err); + done(err); } - - done(); }); }