[optimizer] extract plugin discovery (#14745)

* [plugins] extract plugin discover from the kibana server

* integrate plugin discovery module with server

* [pluginDiscovery] fully extend config before checking enabled status

* [pluginDiscovery] limit arbitrary defaults in PluginSpec

* [ui/navLink] fix tests

* [ui/injectedVars] fix tests

* [ui/app] fix tests

* [server/plugins] convert init to callPluginHook tests

* [build/verifyTranslations] update verify logic

* [pluginDiscovery] remove rx utils

* fix i18n transaltion key name

* [pluginDiscovery] do kibana version checks as a part of discovery

* [pluginDiscovery/createPacksInDirectory$] clarify error handling

* [eslint] fix lint errors

* [uiApp/modules] ensure load order matches master

* [uiBundle] use known uiExport type for providers

* [uiExports] use the `home` export type

* [uiExports] validate that all uiExport types are known

* [timelion] remove archaic/broken bwc check

* revert some stragler changes

* [pluginSpecs] reformat comments

* [uiBundle] rebel and use more fcb 😬

* correct comment

* [server/waitForPluginsInit] describe queues var

* [server/plugins] prevent multiple calls to next() by using single then()

* [uiApp] remove archaic deprecation warning

* [uiApp] tighten up tests

* [pluginDiscovery/errors] remove $ from symbol var

* [pluginDiscovery/reduceExportSpecs] update docs

* [pluginDiscovery/findPluginSpecs] rightVersion -> isRightVersion

* [pluginDiscovery/findPluginSpecs] fix typos

* [uiApps/getById] use Map() rather than memoize

* save

* [savedObjects/mappings] use uiExports.savedObjectMappings

* [server/mapping/indexMapping] update tests, addRootProperties method removed

* [uiExports] "embeddableHandlers" -> "embeddableFactories"

* [pluginDiscovery] fix pluralization of invalidVersionSpec$

* [pluginDiscover] add README

* [pluginDiscovery/reduceExportSpecs] don't ignore fasly spec values, just undefined

* [ui/exportTypes] use better reducer names

* [ui/uiExports] add README

* fix links

* [pluginDiscovery/readme] expand examples

* [pluginDiscovery/readme] clean up reduceExportSpecs() doc

* [ui/uiExports/readme] cleanup example

* [pluginDiscovery] remove needless use of lodash

* [pluginDiscovery/waitForComplete] use better name

* [pluginDiscovery/findPluginSpecs] use fixtures rather than core_plugins

* [pluginDiscovery/stubSchema] use deafult: false

* [plguinDiscovery/pluginConfig] add tests

* typo

* [uiExports/readme] fix link

* [pluginDiscovery/packAtPath] fail with InvalidPackError if path is not a string

* [pluginDiscovery/packAtPath] rely on error.code to detect missing package.json file

* [pluginDiscovery/packAtPath] only attempt to get pack when observable is subscribed

* [pluginDiscovery/packAtPath] add tests

* [pluginDiscovery/pluginPack] move absolute path checks into fs lib

* [pluginDiscovery/packsInDirectory] fix error type check

* [pluginDiscovery/pluginPack/tests] share some utils

* [pluginDiscovery/packsInDirectory] add tests

* [pluginDiscovery/pluginPack] only cast undefined to array

* [pluginDiscovery/pluginPack] add tests

* [pluginDiscovery/pluginSpec/isVersionCompatible] add tests

* [pluginDiscovery/InvalidPluginError] be less redundant

* [pluginDiscovery/pluginSpec] verify config service is passed to isEnabled()

* [pluginDiscovery/pluginSpec] add tests

* fix "existent" spelling

src/cli/cluster/base_path_proxy.js

@@ -6,7 +6,7 @@ import { map as promiseMap, fromNode } from 'bluebird';
 import { Agent as HttpsAgent } from 'https';
 import { readFileSync } from 'fs';
 
-import Config from '../../server/config/config';
+import { Config } from '../../server/config/config';
 import setupConnection from '../../server/http/setup_connection';
 import registerHapiPlugins from '../../server/http/register_hapi_plugins';
 import setupLogging from '../../server/logging';

src/core_plugins/console/index.js

@@ -31,6 +31,14 @@ export default function (kibana) {
     id: 'console',
     require: [ 'elasticsearch' ],
 
+    isEnabled(config) {
+      // console must be disabled when tribe mode is configured
+      return (
+        config.get('console.enabled') &&
+        !config.get('elasticsearch.tribe.url')
+      );
+    },
+
     config: function (Joi) {
       return Joi.object({
         enabled: Joi.boolean().default(true),
@@ -115,7 +123,7 @@ export default function (kibana) {
         }
       });
 
-      const testApp = kibana.uiExports.apps.hidden.byId['sense-tests'];
+      const testApp = server.getHiddenUiAppById('sense-tests');
       if (testApp) {
         server.route({
           path: '/app/sense-tests',

src/core_plugins/dev_mode/index.js

@@ -1,6 +1,14 @@
 export default (kibana) => {
-  if (!kibana.config.get('env.dev')) return;
   return new kibana.Plugin({
+    id: 'dev_mode',
+
+    isEnabled(config) {
+      return (
+        config.get('env.dev') &&
+        config.get('dev_mode.enabled')
+      );
+    },
+
     uiExports: {
       spyModes: [
         'plugins/dev_mode/vis_debug_spy_panel'

src/core_plugins/kibana/server/lib/manage_uuid.js

@@ -17,7 +17,7 @@ export default async function manageUuid(server) {
       return result.toString(FILE_ENCODING);
     } catch (err) {
       if (err.code === 'ENOENT') {
-        // non-existant uuid file is ok
+        // non-existent uuid file is ok
         return false;
       }
       server.log(['error', 'read-uuid'], err);

src/core_plugins/state_session_storage_redirect/index.js

@@ -6,7 +6,7 @@ export default function (kibana) {
         title: 'Redirecting',
         id: 'stateSessionStorageRedirect',
         main: 'plugins/state_session_storage_redirect',
-        listed: false,
+        hidden: true,
       }
     }
   });

src/core_plugins/tests_bundle/index.js

@@ -1,7 +1,10 @@
 import { union } from 'lodash';
-import findSourceFiles from './find_source_files';
+
 import { fromRoot } from '../../utils';
 
+import findSourceFiles from './find_source_files';
+import { createTestEntryTemplate } from './tests_entry_template';
+
 export default (kibana) => {
   return new kibana.Plugin({
     config: (Joi) => {
@@ -13,9 +16,18 @@ export default (kibana) => {
     },
 
     uiExports: {
-      bundle: async (UiBundle, env, apps, plugins) => {
+      async __bundleProvider__(kbnServer) {
         let modules = [];
-        const config = kibana.config;
+
+        const {
+          config,
+          uiApps,
+          uiBundles,
+          plugins,
+          uiExports: {
+            uiSettingDefaults = {}
+          }
+        } = kbnServer;
 
         const testGlobs = [
           'src/ui/public/**/*.js',
@@ -26,20 +38,25 @@ export default (kibana) => {
         if (testingPluginIds) {
           testGlobs.push('!src/ui/public/**/__tests__/**/*');
           testingPluginIds.split(',').forEach((pluginId) => {
-            const plugin = plugins.byId[pluginId];
-            if (!plugin) throw new Error('Invalid testingPluginId :: unknown plugin ' + pluginId);
+            const plugin = plugins
+              .find(plugin => plugin.id === pluginId);
+
+            if (!plugin) {
+              throw new Error('Invalid testingPluginId :: unknown plugin ' + pluginId);
+            }
 
             // add the modules from all of this plugins apps
-            for (const app of plugin.apps) {
-              modules = union(modules, app.getModules());
+            for (const app of uiApps) {
+              if (app.getPluginId() === pluginId) {
+                modules = union(modules, app.getModules());
+              }
             }
 
             testGlobs.push(`${plugin.publicDir}/**/__tests__/**/*.js`);
           });
         } else {
-
           // add the modules from all of the apps
-          for (const app of apps) {
+          for (const app of uiApps) {
             modules = union(modules, app.getModules());
           }
 
@@ -52,24 +69,17 @@ export default (kibana) => {
         for (const f of testFiles) modules.push(f);
 
         if (config.get('tests_bundle.instrument')) {
-          env.addPostLoader({
+          uiBundles.addPostLoader({
             test: /\.js$/,
             exclude: /[\/\\](__tests__|node_modules|bower_components|webpackShims)[\/\\]/,
-            loader: 'istanbul-instrumenter'
+            loader: 'istanbul-instrumenter-loader'
           });
         }
 
-        env.defaultUiSettings = plugins.kbnServer.uiExports.consumers
-          // find the first uiExportsConsumer that has a getUiSettingDefaults method
-          // See src/ui/ui_settings/ui_exports_consumer.js
-          .find(consumer => typeof consumer.getUiSettingDefaults === 'function')
-          .getUiSettingDefaults();
-
-        return new UiBundle({
+        uiBundles.add({
           id: 'tests',
-          modules: modules,
-          template: require('./tests_entry_template'),
-          env: env
+          modules,
+          template: createTestEntryTemplate(uiSettingDefaults),
         });
       },
 

src/core_plugins/tests_bundle/tests_entry_template.js

@@ -1,24 +1,12 @@
 import { esTestConfig } from '../../test_utils/es';
 
-export default function ({ env, bundle }) {
-
-  const pluginSlug = env.pluginInfo.sort()
-    .map(p => ' *  - ' + p)
-    .join('\n');
-
-  const requires = bundle.modules
-    .map(m => `require(${JSON.stringify(m)});`)
-    .join('\n');
-
-  return `
+export const createTestEntryTemplate = (defaultUiSettings) => (bundle) => `
 /**
  * Test entry file
  *
  * This is programatically created and updated, do not modify
  *
- * context: ${JSON.stringify(env.context)}
- * includes code from:
-${pluginSlug}
+ * context: ${bundle.getContext()}
  *
  */
 
@@ -47,14 +35,12 @@ window.__KBN__ = {
     }
   },
   uiSettings: {
-    defaults: ${JSON.stringify(env.defaultUiSettings, null, 2).split('\n').join('\n    ')},
+    defaults: ${JSON.stringify(defaultUiSettings, null, 2).split('\n').join('\n    ')},
     user: {}
   }
 };
 
 require('ui/test_harness');
-${requires}
+${bundle.getRequires().join('\n')}
 require('ui/test_harness').bootstrap(/* go! */);
 `;
-
-}

src/core_plugins/timelion/index.js

@@ -1,15 +1,4 @@
 export default function (kibana) {
-  let mainFile = 'plugins/timelion/app';
-
-  const ownDescriptor = Object.getOwnPropertyDescriptor(kibana, 'autoload');
-  const protoDescriptor = Object.getOwnPropertyDescriptor(kibana.constructor.prototype, 'autoload');
-  const descriptor = ownDescriptor || protoDescriptor || {};
-  if (descriptor.get) {
-    // the autoload list has been replaced with a getter that complains about
-    // improper access, bypass that getter by seeing if it is defined
-    mainFile = 'plugins/timelion/app_with_autoload';
-  }
-
   return new kibana.Plugin({
     require: ['kibana', 'elasticsearch'],
     uiExports: {
@@ -18,7 +7,7 @@ export default function (kibana) {
         order: -1000,
         description: 'Time series expressions for everything',
         icon: 'plugins/timelion/icon.svg',
-        main: mainFile,
+        main: 'plugins/timelion/app',
         injectVars: function (server) {
           const config = server.config();
           return {

src/core_plugins/timelion/public/app.js

@@ -6,6 +6,7 @@ import { SavedObjectRegistryProvider } from 'ui/saved_objects/saved_object_regis
 import { notify } from 'ui/notify';
 import { timezoneProvider } from 'ui/vis/lib/timezone';
 
+require('ui/autoload/all');
 require('plugins/timelion/directives/cells/cells');
 require('plugins/timelion/directives/fixed_element');
 require('plugins/timelion/directives/fullscreen/fullscreen');

src/optimize/base_optimizer.js

@@ -25,8 +25,7 @@ const BABEL_EXCLUDE_RE = [
 
 export default class BaseOptimizer {
   constructor(opts) {
-    this.env = opts.env;
-    this.bundles = opts.bundles;
+    this.uiBundles = opts.uiBundles;
     this.profile = opts.profile || false;
 
     switch (opts.sourceMaps) {
@@ -60,7 +59,7 @@ export default class BaseOptimizer {
     this.compiler.plugin('done', stats => {
       if (!this.profile) return;
 
-      const path = resolve(this.env.workingDir, 'stats.json');
+      const path = this.uiBundles.resolvePath('stats.json');
       const content = JSON.stringify(stats.toJson());
       writeFile(path, content, function (err) {
         if (err) throw err;
@@ -71,7 +70,7 @@ export default class BaseOptimizer {
   }
 
   getConfig() {
-    const cacheDirectory = resolve(this.env.workingDir, '../.cache', this.bundles.hashBundleEntries());
+    const cacheDirectory = this.uiBundles.getCachePath();
 
     function getStyleLoaders(preProcessors = [], postProcessors = []) {
       return ExtractTextPlugin.extract({
@@ -105,13 +104,13 @@ export default class BaseOptimizer {
     const commonConfig = {
       node: { fs: 'empty' },
       context: fromRoot('.'),
-      entry: this.bundles.toWebpackEntries(),
+      entry: this.uiBundles.toWebpackEntries(),
 
       devtool: this.sourceMaps,
       profile: this.profile || false,
 
       output: {
-        path: this.env.workingDir,
+        path: this.uiBundles.getWorkingDir(),
         filename: '[name].bundle.js',
         sourceMapFilename: '[file].map',
         publicPath: PUBLIC_PATH_PLACEHOLDER,
@@ -168,7 +167,7 @@ export default class BaseOptimizer {
           },
           {
             test: /\.js$/,
-            exclude: BABEL_EXCLUDE_RE.concat(this.env.noParse),
+            exclude: BABEL_EXCLUDE_RE.concat(this.uiBundles.getWebpackNoParseRules()),
             use: [
               {
                 loader: 'cache-loader',
@@ -187,12 +186,12 @@ export default class BaseOptimizer {
               },
             ],
           },
-          ...this.env.postLoaders.map(loader => ({
+          ...this.uiBundles.getPostLoaders().map(loader => ({
             enforce: 'post',
             ...loader
           })),
         ],
-        noParse: this.env.noParse,
+        noParse: this.uiBundles.getWebpackNoParseRules(),
       },
 
       resolve: {
@@ -205,12 +204,12 @@ export default class BaseOptimizer {
           'node_modules',
           fromRoot('node_modules'),
         ],
-        alias: this.env.aliases,
+        alias: this.uiBundles.getAliases(),
         unsafeCache: this.unsafeCache,
       },
     };
 
-    if (this.env.context.env === 'development') {
+    if (this.uiBundles.isDevMode()) {
       return webpackMerge(commonConfig, {
         // In the test env we need to add react-addons (and a few other bits) for the
         // enzyme tests to work.

src/optimize/bundles_route/__tests__/bundles_route.js

@@ -219,7 +219,7 @@ describe('optimizer/bundle route', () => {
       const server = createServer();
 
       const response = await server.inject({
-        url: '/bundles/non_existant.js'
+        url: '/bundles/non_existent.js'
       });
 
       expect(response.statusCode).to.be(404);

src/optimize/index.js

@@ -17,19 +17,21 @@ export default async (kbnServer, server, config) => {
     return await kbnServer.mixin(require('./lazy/lazy'));
   }
 
-  const bundles = kbnServer.bundles;
+  const { uiBundles } = kbnServer;
   server.route(createBundlesRoute({
-    bundlesPath: bundles.env.workingDir,
+    bundlesPath: uiBundles.getWorkingDir(),
     basePublicPath: config.get('server.basePath')
   }));
 
-  await bundles.writeEntryFiles();
+  await uiBundles.writeEntryFiles();
 
   // in prod, only bundle when someing is missing or invalid
-  const invalidBundles = config.get('optimize.useBundleCache') ? await bundles.getInvalidBundles() : bundles;
+  const reuseCache = config.get('optimize.useBundleCache')
+    ? await uiBundles.areAllBundleCachesValid()
+    : false;
 
   // we might not have any work to do
-  if (!invalidBundles.getIds().length) {
+  if (reuseCache) {
     server.log(
       ['debug', 'optimize'],
       `All bundles are cached and ready to go!`
@@ -39,21 +41,20 @@ export default async (kbnServer, server, config) => {
 
   // only require the FsOptimizer when we need to
   const optimizer = new FsOptimizer({
-    env: bundles.env,
-    bundles: bundles,
+    uiBundles,
     profile: config.get('optimize.profile'),
     sourceMaps: config.get('optimize.sourceMaps'),
     unsafeCache: config.get('optimize.unsafeCache'),
   });
 
   server.log(
     ['info', 'optimize'],
-    `Optimizing and caching ${bundles.desc()}. This may take a few minutes`
+    `Optimizing and caching ${uiBundles.getDescription()}. This may take a few minutes`
   );
 
   const start = Date.now();
   await optimizer.run();
   const seconds = ((Date.now() - start) / 1000).toFixed(2);
 
-  server.log(['info', 'optimize'], `Optimization of ${bundles.desc()} complete in ${seconds} seconds`);
+  server.log(['info', 'optimize'], `Optimization of ${uiBundles.getDescription()} complete in ${seconds} seconds`);
 };