Remove use of npm ls in grunt tasks (#11965)

* [grunt/build] refactor _build:notice task to not depend on npm

The _build:notice task used to rely on the output of `npm ls` to determine where modules were defined, but the task now just asks `license-checker` to include the `realPath` of the modules it describes in it's output, which is ultimately the same thing but works with `yarn` too.

* [grunt/licenses] convert to use lib/packages/getInstalledPackages()

* [grunt/notice/generate] test generateNoticeText()

* [grunt/licenses] tested assertLicensesValid()

* [npm] remove npm dev dep

* [tasks/lib/packages] do not include kibana in "installed packages"

* [tasks/lib/notice] join all notices with the same separator

(cherry picked from commit 5c04ff6)

package.json

@@ -265,7 +265,6 @@
     "ncp": "2.0.0",
     "nock": "8.0.0",
     "node-sass": "3.8.0",
-    "npm": "3.10.10",
     "portscanner": "1.0.0",
     "proxyquire": "1.7.10",
     "sass-loader": "4.0.0",

tasks/build/notice.js

@@ -1,94 +1,35 @@
-import _ from 'lodash';
-import npmLicense from 'license-checker';
-import glob from 'glob';
-import path from 'path';
-import fs from 'fs';
-import { execSync } from 'child_process';
+import { resolve } from 'path';
+
+import {
+  getInstalledPackages,
+  generateNoticeText,
+} from '../lib';
+
+async function generate(grunt, directory) {
+  return await generateNoticeText({
+    packages: await getInstalledPackages({
+      directory,
+      licenseOverrides: grunt.config.get('licenses.options.overrides')
+    }),
+    nodeDir: grunt.config.get('platforms')[0].nodeDir
+  });
+}
 
-export default function licenses(grunt) {
+export default function (grunt) {
   grunt.registerTask('_build:notice', 'Adds a notice', function () {
     const done = this.async();
-    const buildPath = path.join(grunt.config.get('buildDir'), 'kibana');
-
-    function getPackagePaths() {
-      const packagePaths = {};
-      const installedPackages = execSync(`npm ls --parseable --long`, {
-        cwd: buildPath
-      });
-      installedPackages.toString().trim().split('\n').forEach(pkg => {
-        let modulePath;
-        let dirPath;
-        let packageName;
-        let drive;
-        const packageDetails = pkg.split(':');
-        if (/^win/.test(process.platform)) {
-          [drive, dirPath, packageName] = packageDetails;
-          modulePath = `${drive}:${dirPath}`;
-        } else {
-          [modulePath, packageName] = packageDetails;
-        }
-        const licenses = glob.sync(path.join(modulePath, '*LICENSE*'));
-        const notices = glob.sync(path.join(modulePath, '*NOTICE*'));
-        packagePaths[packageName] = {
-          relative: modulePath.replace(/.*(\/|\\)kibana(\/|\\)/, ''),
-          licenses,
-          notices
-        };
-      });
-      return packagePaths;
-    }
-
-    function combineFiles(filePaths) {
-      let content = '';
-      filePaths.forEach(filePath => {
-        content += fs.readFileSync(filePath) + '\n';
-      });
-      return content;
-    }
-
-    function getNodeInfo() {
-      const nodeVersion = grunt.config.get('nodeVersion');
-      const nodeDir = path.join(grunt.config.get('root'), '.node_binaries', nodeVersion);
-      const licensePath = path.join(nodeDir, 'linux-x64', 'LICENSE');
-      const license = fs.readFileSync(licensePath);
-      return `This product bundles Node.js.\n\n${license}`;
-    }
-
-    function getPackageInfo(packages) {
-      const packagePaths = getPackagePaths();
-      const overrides = grunt.config.get('licenses.options.overrides');
-      let content = '';
-      _.forOwn(packages, (value, key) => {
-        const licenses = [].concat(overrides.hasOwnProperty(key) ? overrides[key] : value.licenses);
-        if (!licenses.length || licenses.includes('UNKNOWN')) return grunt.fail.fatal(`Unknown license for ${key}`);
-        const packagePath = packagePaths[key];
-        const readLicenseAndNotice = combineFiles([].concat(packagePath.licenses, packagePath.notices));
-        const licenseOverview = licenses.length > 1 ? `the\n"${licenses.join('", ')} licenses` : `a\n"${licenses[0]}" license`;
-        const licenseAndNotice = readLicenseAndNotice ? `\n${readLicenseAndNotice}` : `  For details, see ${packagePath.relative}/.`;
-        const combinedText = `This product bundles ${key} which is available under ${licenseOverview}.${licenseAndNotice}\n---\n`;
-
-        content += combinedText;
-      });
-      return content;
-    }
-
-    function getBaseNotice() {
-      return fs.readFileSync(path.join(__dirname, 'notice', 'base_notice.txt'));
-    }
-
-    npmLicense.init({
-      start: buildPath,
-      production: true,
-      json: true
-    }, (result, error) => {
-      if (error) return grunt.fail.fatal(error);
-      const noticePath = path.join(buildPath, 'NOTICE.txt');
-      const fd = fs.openSync(noticePath, 'w');
-      fs.appendFileSync(fd, getBaseNotice());
-      fs.appendFileSync(fd, getPackageInfo(result));
-      fs.appendFileSync(fd, getNodeInfo());
-      fs.closeSync(fd);
-      done();
-    });
+    const kibanaDir = resolve(grunt.config.get('buildDir'), 'kibana');
+    const noticePath = resolve(kibanaDir, 'NOTICE.txt');
+
+    generate(grunt, kibanaDir).then(
+      (noticeText) => {
+        grunt.file.write(noticePath, noticeText);
+        done();
+      },
+      (error) => {
+        grunt.fail.fatal(error);
+        done(error);
+      }
+    );
   });
 }

tasks/config/simplemocha.js

@@ -11,6 +11,7 @@ module.exports = {
       'test/mocha_setup.js',
       'test/**/__tests__/**/*.js',
       'src/**/__tests__/**/*.js',
+      'tasks/**/__tests__/**/*.js',
       'test/fixtures/__tests__/*.js',
       '!src/**/public/**',
       '!**/_*.js'

tasks/lib/index.js

@@ -0,0 +1,3 @@
+export { generateNoticeText } from './notice';
+export { getInstalledPackages } from './packages';
+export { assertLicensesValid } from './licenses';

tasks/lib/licenses/__tests__/valid.js

@@ -0,0 +1,62 @@
+import { resolve } from 'path';
+
+import expect from 'expect.js';
+
+import { assertLicensesValid } from '../valid';
+
+const NODE_MODULES = resolve(__dirname, '../../../../node_modules');
+
+const PACKAGE = {
+  name: '@elastic/httpolyglot',
+  version: '0.1.2-elasticpatch1',
+  licenses: ['MIT'],
+  directory: resolve(NODE_MODULES, '@elastic/httpolyglot'),
+  relative: 'node_modules/@elastic/httpolyglot',
+};
+
+describe('tasks/lib/licenses', () => {
+  describe('assertLicensesValid()', () => {
+    it('returns undefined when package has valid license', () => {
+      expect(assertLicensesValid({
+        packages: [PACKAGE],
+        validLicenses: [...PACKAGE.licenses]
+      })).to.be(undefined);
+    });
+
+    it('throw an error when the packages license is invalid', () => {
+      expect(() => {
+        assertLicensesValid({
+          packages: [PACKAGE],
+          validLicenses: [`not ${PACKAGE.licenses[0]}`]
+        });
+      }).to.throwError(PACKAGE.name);
+    });
+
+    it('throws an error when the package has no licenses', () => {
+      expect(() => {
+        assertLicensesValid({
+          packages: [
+            {
+              ...PACKAGE,
+              licenses: []
+            }
+          ],
+          validLicenses: [...PACKAGE.licenses]
+        });
+      }).to.throwError(PACKAGE.name);
+    });
+
+    it('includes the relative path to packages in error message', () => {
+      try {
+        assertLicensesValid({
+          packages: [PACKAGE],
+          validLicenses: ['none']
+        });
+        throw new Error('expected assertLicensesValid() to throw');
+      } catch (error) {
+        expect(error.message).to.contain(PACKAGE.relative);
+        expect(error.message).to.not.contain(PACKAGE.directory);
+      }
+    });
+  });
+});

tasks/lib/licenses/index.js

@@ -0,0 +1 @@
+export { assertLicensesValid } from './valid';

tasks/lib/licenses/valid.js

@@ -0,0 +1,47 @@
+const describeInvalidLicenses = getInvalid => pkg => (
+`
+  ${pkg.name}
+    version: ${pkg.version}
+    all licenses: ${pkg.licenses}
+    invalid licenses: ${getInvalid(pkg.licenses).join(', ')}
+    path: ${pkg.relative}
+`
+);
+
+/**
+ *  When given a list of packages and the valid license
+ *  options, either throws an error with details about
+ *  violations or returns undefined.
+ *
+ *  @param  {Object} [options={}]
+ *  @property {Array<Package>} options.packages List of packages to check, see
+ *                                              getInstalledPackages() in ../packages
+ *  @property {Array<string>} options.validLicenses
+ *  @return {undefined}
+ */
+export function assertLicensesValid(options = {}) {
+  const {
+    packages,
+    validLicenses
+  } = options;
+
+  if (!packages || !validLicenses) {
+    throw new Error('packages and validLicenses options are required');
+  }
+
+  const getInvalid = licenses => (
+    licenses.filter(license => !validLicenses.includes(license))
+  );
+
+  const isPackageInvalid = pkg => (
+    !pkg.licenses.length || getInvalid(pkg.licenses).length > 0
+  );
+
+  const invalidMsgs = packages
+    .filter(isPackageInvalid)
+    .map(describeInvalidLicenses(getInvalid));
+
+  if (invalidMsgs.length) {
+    throw new Error(`Non-confirming licenses: ${invalidMsgs.join('')}`);
+  }
+}

tasks/lib/notice/__tests__/notice.js

@@ -0,0 +1,55 @@
+import { resolve } from 'path';
+import { readFileSync } from 'fs';
+
+import expect from 'expect.js';
+
+import { generateNoticeText } from '../notice';
+
+const NODE_MODULES = resolve(__dirname, '../../../../node_modules');
+const NODE_DIR = resolve(process.execPath, '../..');
+const PACKAGES = [
+  {
+    name: '@elastic/httpolyglot',
+    version: '0.1.2-elasticpatch1',
+    licenses: ['MIT'],
+    directory: resolve(NODE_MODULES, '@elastic/httpolyglot'),
+    relative: 'node_modules/@elastic/httpolyglot',
+  },
+  {
+    name: 'aws-sdk',
+    version: '2.0.31',
+    licenses: ['Apache 2.0'],
+    directory: resolve(NODE_MODULES, 'aws-sdk'),
+    relative: 'node_modules/aws-sdk',
+  }
+];
+
+describe('tasks/lib/notice', () => {
+  describe('generateNoticeText()', () => {
+    let notice;
+    before(async () => notice = await generateNoticeText({
+      packages: PACKAGES,
+      nodeDir: NODE_DIR
+    }));
+
+    it('returns a string', () => {
+      expect(notice).to.be.a('string');
+    });
+
+    it('includes *NOTICE* files from packages', () => {
+      expect(notice).to.contain(readFileSync(resolve(NODE_MODULES, 'aws-sdk/NOTICE.txt'), 'utf8'));
+    });
+
+    it('includes *LICENSE* files from packages', () => {
+      expect(notice).to.contain(readFileSync(resolve(NODE_MODULES, '@elastic/httpolyglot/LICENSE'), 'utf8'));
+    });
+
+    it('includes the LICENSE file from node', () => {
+      expect(notice).to.contain(readFileSync(resolve(NODE_DIR, 'LICENSE'), 'utf8'));
+    });
+
+    it('includes the base_notice.txt file', () => {
+      expect(notice).to.contain(readFileSync(resolve(__dirname, '../base_notice.txt'), 'utf8'));
+    });
+  });
+});

tasks/build/notice/base_notice.txt → tasks/lib/notice/base_notice.txt

@@ -56,4 +56,3 @@ THE SOFTWARE.
 ---
 This product bundles geohash.js which is available under a
 "MIT" license.  For details, see src/ui/public/utils/decode_geo_hash.js.
----

tasks/lib/notice/bundled_notices.js

@@ -0,0 +1,14 @@
+import { resolve } from 'path';
+import { readFile } from 'fs';
+
+import { fromNode as fcb } from 'bluebird';
+import glob from 'glob';
+
+export async function getBundledNotices(packageDirectory) {
+  const pattern = resolve(packageDirectory, '*{LICENSE,NOTICE}*');
+  const paths = await fcb(cb => glob(pattern, cb));
+  return Promise.all(paths.map(async path => ({
+    path,
+    text: await fcb(cb => readFile(path, 'utf8', cb))
+  })));
+}

tasks/lib/notice/index.js

@@ -0,0 +1 @@
+export { generateNoticeText } from './notice';

tasks/lib/notice/node_notice.js

@@ -0,0 +1,8 @@
+import { resolve } from 'path';
+import { readFileSync } from 'fs';
+
+export function generateNodeNoticeText(nodeDir) {
+  const licensePath = resolve(nodeDir, 'LICENSE');
+  const license = readFileSync(licensePath, 'utf8');
+  return `This product bundles Node.js.\n\n${license}`;
+}

tasks/lib/notice/notice.js

@@ -0,0 +1,33 @@
+import { resolve } from 'path';
+import { readFileSync } from 'fs';
+
+import { generatePackageNoticeText } from './package_notice';
+import { generateNodeNoticeText } from './node_notice';
+
+const BASE_NOTICE = resolve(__dirname, './base_notice.txt');
+
+/**
+ *  When given a list of packages and the directory to the
+ *  node distribution that will be shipping with Kibana,
+ *  generates the text for NOTICE.txt
+ *
+ *  @param  {Object} [options={}]
+ *  @property {Array<Package>} options.packages List of packages to check, see
+ *                                              getInstalledPackages() in ../packages
+ *  @property {string} options.nodeDir The directory containing the version of node.js
+ *                                     that will ship with Kibana
+ *  @return {undefined}
+ */
+export async function generateNoticeText(options = {}) {
+  const { packages, nodeDir } = options;
+
+  const packageNotices = await Promise.all(
+    packages.map(generatePackageNoticeText)
+  );
+
+  return [
+    readFileSync(BASE_NOTICE, 'utf8'),
+    ...packageNotices,
+    generateNodeNoticeText(nodeDir),
+  ].join('\n---\n');
+}