fix(NA): external plugins development flow with the new modern packag…

…e plugins in place (#153562)

This PR fixes the 3rd party external plugin development workflow by
introducing a dev step for plugins that allows for development usages of
those with Kibana.

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

docs/developer/plugin/plugin-tooling.asciidoc

@@ -43,8 +43,10 @@ It will output a`zip` archive in `kibana/plugins/my_plugin_name/build/` folder.
 See <<install-plugin, How to install a plugin>>.
 
 === Run {kib} with your plugin in dev mode
-Run `yarn start` in the {kib} root folder. Make sure {kib} found and bootstrapped your plugin:
+If your plugin isn't server only and contains `ui` in order for Kibana to pick the browser bundles you need to run `yarn dev --watch` in the plugin root folder at a dedicated terminal.
+
+Then, in a second terminal, run `yarn start` at the {kib} root folder. Make sure {kib} found and bootstrapped your plugin by:
 ["source","shell"]
 -----------
-[info][plugins-system] Setting up […] plugins: […, myPluginName, …]
+[INFO ][plugins-system.standard] Setting up […] plugins: […, myPluginName, …]
 -----------

packages/kbn-optimizer/src/common/bundle.ts

@@ -21,8 +21,8 @@ const VALID_BUNDLE_TYPES = ['plugin' as const, 'entry' as const];
 
 const DEFAULT_IMPLICIT_BUNDLE_DEPS = ['core'];
 
-const toStringArray = (input: any): string[] =>
-  Array.isArray(input) && input.every((x) => typeof x === 'string') ? input : [];
+const toStringArray = (input: any): string[] | null =>
+  Array.isArray(input) && input.every((x) => typeof x === 'string') ? input : null;
 
 export interface BundleSpec {
   readonly type: typeof VALID_BUNDLE_TYPES[0];
@@ -164,19 +164,39 @@ export class Bundle {
       );
     }
 
-    if (isObj(parsed) && isObj(parsed.plugin)) {
-      return {
-        explicit: [...toStringArray(parsed.plugin.requiredBundles)],
-        implicit: [
-          ...DEFAULT_IMPLICIT_BUNDLE_DEPS,
-          ...toStringArray(parsed.plugin.requiredPlugins),
-        ],
-      };
+    // TODO: remove once we improve the third party plugin build workflow
+    // This is only used to build legacy third party plugins in the @kbn/plugin-helpers
+
+    if (!isObj(parsed)) {
+      throw new Error(`Expected [${this.manifestPath}] to be a jsonc parseable file`);
+    }
+
+    const requiredBundles = isObj(parsed.plugin)
+      ? parsed.plugin.requiredBundles
+      : parsed.requiredBundles;
+    const requiredPlugins = isObj(parsed.plugin)
+      ? parsed.plugin.requiredPlugins
+      : parsed.requiredPlugins;
+    const requiredBundlesStringArray = toStringArray(requiredBundles);
+    const requiredPluginsStringArray = toStringArray(requiredPlugins);
+    // END-OF-TD: we just need to check for parse.plugin and not for legacy plugins manifest types
+
+    if (!requiredBundlesStringArray && requiredBundles) {
+      throw new Error(
+        `Expected "requiredBundles" in manifest file [${this.manifestPath}] to be an array of strings`
+      );
     }
 
-    throw new Error(
-      `Expected "requiredBundles" and "requiredPlugins" in manifest file [${this.manifestPath}] to be arrays of strings`
-    );
+    if (!requiredPluginsStringArray && requiredPlugins) {
+      throw new Error(
+        `Expected "requiredPlugins" in manifest file [${this.manifestPath}] to be an array of strings`
+      );
+    }
+
+    return {
+      explicit: [...(requiredBundlesStringArray || [])],
+      implicit: [...DEFAULT_IMPLICIT_BUNDLE_DEPS, ...(requiredPluginsStringArray || [])],
+    };
   }
 }
 

packages/kbn-plugin-generator/README.md

@@ -51,7 +51,7 @@ yarn kbn bootstrap
 
 Generated plugins receive a handful of scripts that can be used during development. Those scripts are detailed in the [README.md](template/README.md) file in each newly generated plugin, and expose the scripts provided by the [Kibana plugin helpers](../kbn-plugin-helpers), but here is a quick reference in case you need it:
 
-> ***NOTE:*** The following scripts should be run from the generated plugin.
+> ***NOTE:*** The following scripts should be run from the generated plugin root folder.
 
   - `yarn kbn bootstrap`
 
@@ -63,12 +63,16 @@ Generated plugins receive a handful of scripts that can be used during developme
 
     Build a distributable archive of your plugin.
 
+  - `yarn dev --watch`
+
+    Builds and starts the watch mode of your ui browser side plugin so it can be picked up by Kibana in development.
+
 
 To start kibana run the following command from Kibana root.
 
   - `yarn start`
 
-    Start kibana and it will automatically include this plugin. You can pass any arguments that you would normally send to `bin/kibana`
+    Start kibana and, if you had previously run in another terminal `yarn dev --watch` at the root of your plugin, it will automatically include this plugin. You can pass any arguments that you would normally send to `bin/kibana`
 
       ```
       yarn start --elasticsearch.hosts http://localhost:9220

packages/kbn-plugin-generator/template/.eslintrc.js.ejs

@@ -1,3 +1,5 @@
+require('@kbn/babel-register').install();
+
 module.exports = {
   root: true,
   extends: [

packages/kbn-plugin-generator/template/README.md.ejs

@@ -16,5 +16,8 @@ See the [kibana contributing guide](https://github.com/elastic/kibana/blob/main/
 
   <dt><code>yarn plugin-helpers build</code></dt>
   <dd>Execute this to create a distributable version of this plugin that can be installed in Kibana</dd>
+
+  <dt><code>yarn plugin-helpers dev --watch</code></dt>
+    <dd>Execute this to build your plugin ui browser side so Kibana could pick up when started in development</dd>
 </dl>
 <% } %>

packages/kbn-plugin-generator/template/package.json.ejs

@@ -4,6 +4,7 @@
   "private": true,
   "scripts": {
     "build": "yarn plugin-helpers build",
+    "dev": "yarn plugin-helpers dev",
     "plugin-helpers": "node ../../scripts/plugin_helpers",
     "kbn": "node ../../scripts/kbn"
   }

packages/kbn-plugin-helpers/README.md

@@ -11,6 +11,7 @@ is already the case if you use the new `node scripts/generate_plugin` script.
 {
   "scripts" : {
     "build": "yarn plugin-helpers build",
+    "dev": "yarn plugin-helpers dev",
     "plugin-helpers": "node ../../scripts/plugin_helpers",
     "kbn": "node ../../scripts/kbn"
   }
@@ -48,7 +49,15 @@ $ plugin-helpers help
 
         Options:
           --skip-archive        Don't create the zip file, just create the build/kibana directory
-          --kibana-version, -v  Kibana version that the
+          --kibana-version, -v  Kibana version this plugin will be built for
+          
+      dev
+        Builds the current plugin ui browser side so it can be picked up by Kibana
+        during development.
+        
+        Options:
+          --dist, -d  Outputs bundles in dist mode instead
+          --watch, -w  Starts the watch mode
     
 
   Global options:
@@ -92,7 +101,7 @@ Plugin code can be written in [TypeScript](http://www.typescriptlang.org/) if de
 ```js
 {
   // extend Kibana's tsconfig, or use your own settings
-  "extends": "../../kibana/tsconfig.json",
+  "extends": "../../tsconfig.json",
 
   // tell the TypeScript compiler where to find your source files
   "include": [

packages/kbn-plugin-helpers/src/cli.ts

@@ -14,7 +14,7 @@ import { createFlagError, createFailError } from '@kbn/dev-cli-errors';
 import { findPluginDir } from './find_plugin_dir';
 import { loadKibanaPlatformPlugin } from './load_kibana_platform_plugin';
 import * as Tasks from './tasks';
-import { BuildContext } from './build_context';
+import { TaskContext } from './task_context';
 import { resolveKibanaVersion } from './resolve_kibana_version';
 import { loadConfig } from './config';
 
@@ -42,7 +42,7 @@ export function runCli() {
         },
         help: `
           --skip-archive        Don't create the zip file, just create the build/kibana directory
-          --kibana-version, -v  Kibana version that the
+          --kibana-version, -v  Kibana version this plugin will be built for
         `,
       },
       async run({ log, flags }) {
@@ -56,7 +56,7 @@ export function runCli() {
           throw createFlagError('expected a single --skip-archive flag');
         }
 
-        const found = await findPluginDir();
+        const found = findPluginDir();
         if (!found) {
           throw createFailError(
             `Unable to find Kibana Platform plugin in [${process.cwd()}] or any of its parent directories. Has it been migrated properly? Does it have a kibana.json file?`
@@ -73,8 +73,10 @@ export function runCli() {
         const sourceDir = plugin.directory;
         const buildDir = Path.resolve(plugin.directory, 'build/kibana', plugin.manifest.id);
 
-        const context: BuildContext = {
+        const context: TaskContext = {
           log,
+          dev: false,
+          dist: true,
           plugin,
           config,
           sourceDir,
@@ -93,5 +95,73 @@ export function runCli() {
         }
       },
     })
+    .command({
+      name: 'dev',
+      description: `
+        Builds the current plugin ui browser side so it can be picked up by Kibana
+        during development
+
+      `,
+      flags: {
+        boolean: ['dist', 'watch'],
+        alias: {
+          d: 'dist',
+          w: 'watch',
+        },
+        help: `
+          --dist, -d  Outputs bundles in dist mode instead
+          --watch, -w  Starts the watch mode
+        `,
+      },
+      async run({ log, flags }) {
+        const dist = flags.dist;
+        if (dist !== undefined && typeof dist !== 'boolean') {
+          throw createFlagError('expected a single --dist flag');
+        }
+
+        const watch = flags.watch;
+        if (watch !== undefined && typeof watch !== 'boolean') {
+          throw createFlagError('expected a single --watch flag');
+        }
+
+        const found = findPluginDir();
+        if (!found) {
+          throw createFailError(
+            `Unable to find Kibana Platform plugin in [${process.cwd()}] or any of its parent directories. Has it been migrated properly? Does it have a kibana.json file?`
+          );
+        }
+
+        if (found.type === 'package') {
+          throw createFailError(`the plugin helpers do not currently support "package plugins"`);
+        }
+
+        const plugin = loadKibanaPlatformPlugin(found.dir);
+
+        if (!plugin.manifest.ui) {
+          log.info(
+            'Your plugin is server only and there is no need to run a dev task in order to get it ready to test. Please just run `yarn start` at the Kibana root and your plugin will be started.'
+          );
+          return;
+        }
+
+        const config = await loadConfig(log, plugin);
+        const sourceDir = plugin.directory;
+
+        const context: TaskContext = {
+          log,
+          dev: true,
+          dist,
+          watch,
+          plugin,
+          config,
+          sourceDir,
+          buildDir: '',
+          kibanaVersion: 'kibana',
+        };
+
+        await Tasks.initDev(context);
+        await Tasks.optimize(context);
+      },
+    })
     .execute();
 }

packages/kbn-plugin-helpers/src/integration_tests/build.test.ts

@@ -70,6 +70,7 @@ it('builds a generated plugin into a viable archive', async () => {
     " info deleting the build and target directories
      info running @kbn/optimizer
      │ succ browser bundle created at plugins/foo_test_plugin/build/kibana/fooTestPlugin/target/public
+     │ info stopping @kbn/optimizer
      info copying assets from \`public/assets\` to build
      info copying server source into the build and converting with babel
      info running yarn to install dependencies

packages/kbn-plugin-helpers/src/build_context.ts → packages/kbn-plugin-helpers/src/task_context.ts

@@ -11,8 +11,11 @@ import { ToolingLog } from '@kbn/tooling-log';
 import { Plugin } from './load_kibana_platform_plugin';
 import { Config } from './config';
 
-export interface BuildContext {
+export interface TaskContext {
   log: ToolingLog;
+  dev: boolean;
+  dist?: boolean;
+  watch?: boolean;
   plugin: Plugin;
   config: Config;
   sourceDir: string;

packages/kbn-plugin-helpers/src/tasks/clean.ts

@@ -11,11 +11,11 @@ import { promisify } from 'util';
 
 import del from 'del';
 
-import { BuildContext } from '../build_context';
+import { TaskContext } from '../task_context';
 
 const asyncMkdir = promisify(Fs.mkdir);
 
-export async function initTargets({ log, sourceDir, buildDir }: BuildContext) {
+export async function initTargets({ log, sourceDir, buildDir }: TaskContext) {
   log.info('deleting the build and target directories');
   await del(['build', 'target'], {
     cwd: sourceDir,
@@ -24,3 +24,10 @@ export async function initTargets({ log, sourceDir, buildDir }: BuildContext) {
   log.debug(`creating build output dir [${buildDir}]`);
   await asyncMkdir(buildDir, { recursive: true });
 }
+
+export async function initDev({ log, sourceDir }: TaskContext) {
+  log.info('deleting the target folder');
+  await del(['target'], {
+    cwd: sourceDir,
+  });
+}

packages/kbn-plugin-helpers/src/tasks/create_archive.ts

@@ -14,11 +14,11 @@ import del from 'del';
 import vfs from 'vinyl-fs';
 import zip from 'gulp-zip';
 
-import { BuildContext } from '../build_context';
+import { TaskContext } from '../task_context';
 
 const asyncPipeline = promisify(pipeline);
 
-export async function createArchive({ kibanaVersion, plugin, log }: BuildContext) {
+export async function createArchive({ kibanaVersion, plugin, log }: TaskContext) {
   const {
     manifest: { id },
     directory,