build new platform plugins with shared deps

package.json

@@ -298,6 +298,7 @@
     "@kbn/eslint-plugin-eslint": "1.0.0",
     "@kbn/expect": "1.0.0",
     "@kbn/plugin-generator": "1.0.0",
+    "@kbn/optimizer": "1.0.0",
     "@kbn/test": "1.0.0",
     "@kbn/utility-types": "1.0.0",
     "@microsoft/api-documenter": "7.7.2",

packages/kbn-interpreter/package.json

@@ -27,7 +27,7 @@
     "del": "^5.1.0",
     "getopts": "^2.2.4",
     "pegjs": "0.10.0",
-    "sass-loader": "^7.3.1",
+    "sass-loader": "^8.0.2",
     "style-loader": "0.23.1",
     "supports-color": "^7.0.0",
     "url-loader": "2.2.0",

packages/kbn-optimizer/README.md

@@ -0,0 +1,8 @@
+# @kbn/optimizer
+
+New version of the Optimizer that ships with Kibana, aiming for the following goals:
+
+ - stop shipping optimizer
+ - build packages to distributable files that don't need to be updated
+ - run each plugin in separate webpack instances with as simple a configuration as possible
+ - support running all webpack instances in watch mode

src/cli/color.js → packages/kbn-optimizer/babel.config.js

@@ -17,8 +17,7 @@
  * under the License.
  */
 
-import chalk from 'chalk';
-
-export const green = chalk.black.bgGreen;
-export const red = chalk.white.bgRed;
-export const yellow = chalk.black.bgYellow;
+module.exports = {
+  presets: ['@kbn/babel-preset/node_preset'],
+  ignore: ['**/*.test.js'],
+};

packages/kbn-optimizer/index.d.ts

@@ -0,0 +1,20 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export * from './src/index';

packages/kbn-optimizer/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@kbn/optimizer",
+  "main": "./target/index.js",
+  "version": "1.0.0",
+  "license": "Apache-2.0",
+  "private": true,
+  "scripts": {
+    "build": "babel src --out-dir target --copy-files --delete-dir-on-start --extensions .ts --ignore *.test.ts --source-maps=inline",
+    "kbn:bootstrap": "yarn build",
+    "kbn:watch": "yarn build --watch"
+  },
+  "dependencies": {
+    "@babel/cli": "^7.5.5",
+    "@kbn/babel-preset": "1.0.0",
+    "@kbn/dev-utils": "1.0.0",
+    "@kbn/ui-shared-deps": "1.0.0",
+    "@types/webpack": "^4.41.0",
+    "babel-loader": "^8.0.6",
+    "clean-webpack-plugin": "^3.0.0",
+    "cpy": "^8.0.0",
+    "css-loader": "^2.1.1",
+    "del": "^5.1.0",
+    "file-loader": "^4.2.0",
+    "istanbul-instrumenter-loader": "^3.0.1",
+    "jest-raw-loader": "^1.0.1",
+    "node-sass": "^4.13.0",
+    "postcss-loader": "^3.0.0",
+    "raw-loader": "^3.1.0",
+    "rxjs": "^6.5.3",
+    "sass-loader": "^8.0.2",
+    "script-loader": "^0.7.2",
+    "style-loader": "^0.23.1",
+    "terser-webpack-plugin": "^2.1.2",
+    "url-loader": "^2.2.0",
+    "val-loader": "^1.1.1",
+    "webpack-merge": "^4.2.2",
+    "webpack": "^4.41.0"
+  }
+}

packages/kbn-optimizer/src/bundle_metadata.ts

@@ -0,0 +1,100 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import Fs from 'fs';
+import Path from 'path';
+
+const DEFAULT_STATE = JSON.stringify({});
+
+interface State {
+  moduleCounts: { [key: string]: number };
+}
+
+/**
+ * Helper to read and update metadata for bundles.
+ *
+ * Currently only tracks the module count of bundles, allowing
+ * us to assign bundles to workers in a way that ensures an even
+ * distribution of modules to be built
+ */
+export class BundleMetadata {
+  private cache: State | undefined = undefined;
+  constructor(private readonly path: string | false) {}
+
+  private getState() {
+    if (!this.cache) {
+      let json;
+      try {
+        if (this.path) {
+          json = Fs.readFileSync(this.path, 'utf8');
+        }
+      } catch (error) {
+        if (error.code !== 'ENOENT') {
+          throw error;
+        }
+      }
+
+      let partialCache: Partial<State>;
+      try {
+        partialCache = JSON.parse(json || DEFAULT_STATE);
+      } catch (error) {
+        partialCache = {};
+      }
+
+      this.cache = {
+        moduleCounts: partialCache.moduleCounts || {},
+      };
+    }
+
+    return this.cache;
+  }
+
+  getModuleCount(bundleId: string) {
+    const state = this.getState();
+
+    return state.moduleCounts[bundleId] || -Infinity;
+  }
+
+  saveModuleCount(bundleId: string, count: number) {
+    const current = this.getState();
+
+    this.setState({
+      ...current,
+      moduleCounts: {
+        ...current.moduleCounts,
+        [bundleId]: count,
+      },
+    });
+  }
+
+  private scheduledWrite = false;
+  private setState(updated: State) {
+    this.cache = updated;
+    if (!this.scheduledWrite && this.path) {
+      const path = this.path;
+      const directory = Path.dirname(path);
+      this.scheduledWrite = true;
+      process.nextTick(() => {
+        this.scheduledWrite = false;
+        Fs.mkdirSync(directory, { recursive: true });
+        Fs.writeFileSync(path, JSON.stringify(this.cache, null, 2));
+      });
+    }
+  }
+}

packages/kbn-optimizer/src/cli.ts

@@ -0,0 +1,97 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import 'source-map-support/register';
+
+import Path from 'path';
+
+import { run, REPO_ROOT, createFlagError } from '@kbn/dev-utils';
+
+import { logOptimizerState } from './log_optimizer_state';
+import { OptimizerConfig } from './optimizer_config';
+import { Optimizer } from './optimizer';
+
+run(
+  async ({ log, flags }) => {
+    const watch = flags.watch ?? false;
+    if (typeof watch !== 'boolean') {
+      throw createFlagError('expected --watch to have no value');
+    }
+
+    const oss = flags.oss ?? false;
+    if (typeof oss !== 'boolean') {
+      throw createFlagError('expected --oss to have no value');
+    }
+
+    const dist = flags.dist ?? false;
+    if (typeof dist !== 'boolean') {
+      throw createFlagError('expected --dist to have no value');
+    }
+
+    const examples = flags.examples ?? false;
+    if (typeof examples !== 'boolean') {
+      throw createFlagError('expected --no-examples to have no value');
+    }
+
+    const workerCount = flags.workers ? Number.parseInt(String(flags.workers), 10) : undefined;
+    if (workerCount !== undefined && (!Number.isFinite(workerCount) || workerCount < 1)) {
+      throw createFlagError('expected --workers to be a number greater than 0');
+    }
+
+    const extraScanDirs = ([] as string[])
+      .concat((flags['scan-dir'] as string | string[]) || [])
+      .map(s => Path.resolve(s));
+
+    if (!extraScanDirs.every(s => typeof s === 'string')) {
+      throw createFlagError('expected --scan-dir to have string values');
+    }
+
+    const config = OptimizerConfig.load({
+      repoRoot: REPO_ROOT,
+      watch,
+      workerCount,
+      oss,
+      dist,
+      examples,
+      extraScanDirs,
+    });
+
+    await new Optimizer(config)
+      .run()
+      .pipe(logOptimizerState(log, config))
+      .toPromise();
+  },
+  {
+    flags: {
+      boolean: ['watch', 'oss', 'examples', 'dist'],
+      string: ['workers', 'scan-dir'],
+      default: {
+        examples: true,
+      },
+      help: `
+        --watch            run the optimizer in watch mode
+        --workers          number of workers to use
+        --oss              only build oss plugins
+        --no-examples      don't build the example plugins
+        --scan-dir         add a directory that should be scanned for plugins (can be specified multiple times)
+        --dist             build new platform plugins in a way that is suitable for inclusion in the Kibana distributable
+      `,
+    },
+  }
+);