merge with master

.github/ISSUE_TEMPLATE/v8_breaking_change.md

@@ -0,0 +1,36 @@
+---
+name: 8.0 Breaking change
+about: Breaking changes from 7.x -> 8.0
+title: "[Breaking change]"
+labels: Team:Elasticsearch UI, Feature:Upgrade Assistant
+assignees: ''
+
+---
+
+## Change description
+
+**Which release will ship the breaking change?**
+
+<!-- e.g., v7.6.2 -->
+
+**Describe the change. How will it manifest to users?**
+
+**What percentage of users will be affected?**
+
+<!-- e.g., Roughly 75% will need to make changes to x. -->
+
+**What can users to do to address the change manually?**
+
+<!-- If applicable, describe the manual workaround -->
+
+**How could we make migration easier with the Upgrade Assistant?**
+
+**Are there any edge cases?**
+
+## Test Data
+
+Provide test data. We can’t build a solution without data to test it against.
+
+## Cross links
+
+Cross-link to relevant [Elasticsearch breaking changes](https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-8.0.html).

.gitignore

@@ -4,6 +4,7 @@
 /.es
 .DS_Store
 .node_binaries
+.native_modules
 node_modules
 !/src/dev/npm/integration_tests/__fixtures__/fixture1/node_modules
 !/src/dev/notice/__fixtures__/node_modules

Jenkinsfile

@@ -3,7 +3,7 @@
 library 'kibana-pipeline-library'
 kibanaLibrary.load()
 
-kibanaPipeline(timeoutMinutes: 135) {
+kibanaPipeline(timeoutMinutes: 135, checkPrChanges: true) {
   githubPr.withDefaultPrComments {
     catchError {
       retryable.enable()

docs/development/core/public/kibana-plugin-core-public.icontextprovider.md

@@ -9,7 +9,7 @@ A function that returns a context value for a specific key of given context type
 <b>Signature:</b>
 
 ```typescript
-export declare type IContextProvider<THandler extends HandlerFunction<any>, TContextName extends keyof HandlerContextType<THandler>> = (context: Partial<HandlerContextType<THandler>>, ...rest: HandlerParameters<THandler>) => Promise<HandlerContextType<THandler>[TContextName]> | HandlerContextType<THandler>[TContextName];
+export declare type IContextProvider<THandler extends HandlerFunction<any>, TContextName extends keyof HandlerContextType<THandler>> = (context: PartialExceptFor<HandlerContextType<THandler>, 'core'>, ...rest: HandlerParameters<THandler>) => Promise<HandlerContextType<THandler>[TContextName]> | HandlerContextType<THandler>[TContextName];
 ```
 
 ## Remarks

docs/development/core/server/kibana-plugin-core-server.icontextprovider.md

@@ -9,7 +9,7 @@ A function that returns a context value for a specific key of given context type
 <b>Signature:</b>
 
 ```typescript
-export declare type IContextProvider<THandler extends HandlerFunction<any>, TContextName extends keyof HandlerContextType<THandler>> = (context: Partial<HandlerContextType<THandler>>, ...rest: HandlerParameters<THandler>) => Promise<HandlerContextType<THandler>[TContextName]> | HandlerContextType<THandler>[TContextName];
+export declare type IContextProvider<THandler extends HandlerFunction<any>, TContextName extends keyof HandlerContextType<THandler>> = (context: PartialExceptFor<HandlerContextType<THandler>, 'core'>, ...rest: HandlerParameters<THandler>) => Promise<HandlerContextType<THandler>[TContextName]> | HandlerContextType<THandler>[TContextName];
 ```
 
 ## Remarks

package.json

@@ -227,6 +227,7 @@
     "pug": "^2.0.4",
     "query-string": "6.10.1",
     "raw-loader": "3.1.0",
+    "re2": "1.10.5",
     "react": "^16.12.0",
     "react-color": "^2.13.8",
     "react-dom": "^16.12.0",

src/core/public/public.api.md

@@ -735,8 +735,10 @@ export interface IContextContainer<THandler extends HandlerFunction<any>> {
     registerContext<TContextName extends keyof HandlerContextType<THandler>>(pluginOpaqueId: PluginOpaqueId, contextName: TContextName, provider: IContextProvider<THandler, TContextName>): this;
 }
 
+// Warning: (ae-forgotten-export) The symbol "PartialExceptFor" needs to be exported by the entry point index.d.ts
+//
 // @public
-export type IContextProvider<THandler extends HandlerFunction<any>, TContextName extends keyof HandlerContextType<THandler>> = (context: Partial<HandlerContextType<THandler>>, ...rest: HandlerParameters<THandler>) => Promise<HandlerContextType<THandler>[TContextName]> | HandlerContextType<THandler>[TContextName];
+export type IContextProvider<THandler extends HandlerFunction<any>, TContextName extends keyof HandlerContextType<THandler>> = (context: PartialExceptFor<HandlerContextType<THandler>, 'core'>, ...rest: HandlerParameters<THandler>) => Promise<HandlerContextType<THandler>[TContextName]> | HandlerContextType<THandler>[TContextName];
 
 // @public (undocumented)
 export interface IHttpFetchError extends Error {

src/core/server/server.api.md

@@ -882,8 +882,10 @@ export interface IContextContainer<THandler extends HandlerFunction<any>> {
     registerContext<TContextName extends keyof HandlerContextType<THandler>>(pluginOpaqueId: PluginOpaqueId, contextName: TContextName, provider: IContextProvider<THandler, TContextName>): this;
 }
 
+// Warning: (ae-forgotten-export) The symbol "PartialExceptFor" needs to be exported by the entry point index.d.ts
+//
 // @public
-export type IContextProvider<THandler extends HandlerFunction<any>, TContextName extends keyof HandlerContextType<THandler>> = (context: Partial<HandlerContextType<THandler>>, ...rest: HandlerParameters<THandler>) => Promise<HandlerContextType<THandler>[TContextName]> | HandlerContextType<THandler>[TContextName];
+export type IContextProvider<THandler extends HandlerFunction<any>, TContextName extends keyof HandlerContextType<THandler>> = (context: PartialExceptFor<HandlerContextType<THandler>, 'core'>, ...rest: HandlerParameters<THandler>) => Promise<HandlerContextType<THandler>[TContextName]> | HandlerContextType<THandler>[TContextName];
 
 // @public
 export interface ICspConfig {

src/core/utils/context.ts

@@ -22,6 +22,11 @@ import { ShallowPromise } from '@kbn/utility-types';
 import { pick } from '.';
 import { CoreId, PluginOpaqueId } from '../server';
 
+/**
+ * Make all properties in T optional, except for the properties whose keys are in the union K
+ */
+type PartialExceptFor<T, K extends keyof T> = Partial<T> & Pick<T, K>;
+
 /**
  * A function that returns a context value for a specific key of given context type.
  *
@@ -39,7 +44,8 @@ export type IContextProvider<
   THandler extends HandlerFunction<any>,
   TContextName extends keyof HandlerContextType<THandler>
 > = (
-  context: Partial<HandlerContextType<THandler>>,
+  // context.core will always be available, but plugin contexts are typed as optional
+  context: PartialExceptFor<HandlerContextType<THandler>, 'core'>,
   ...rest: HandlerParameters<THandler>
 ) =>
   | Promise<HandlerContextType<THandler>[TContextName]>
@@ -261,7 +267,7 @@ export class ContextContainer<THandler extends HandlerFunction<any>>
         // registered that provider.
         const exposedContext = pick(resolvedContext, [
           ...this.getContextNamesForSource(providerSource),
-        ]) as Partial<HandlerContextType<THandler>>;
+        ]) as PartialExceptFor<HandlerContextType<THandler>, 'core'>;
 
         return {
           ...resolvedContext,

src/dev/build/build_distributables.js

@@ -45,6 +45,7 @@ import {
   InstallDependenciesTask,
   BuildKibanaPlatformPluginsTask,
   OptimizeBuildTask,
+  PatchNativeModulesTask,
   RemovePackageJsonDepsTask,
   RemoveWorkspacesTask,
   TranspileBabelTask,
@@ -132,6 +133,7 @@ export async function buildDistributables(options) {
    * directories and perform platform-specific steps
    */
   await run(CreateArchivesSourcesTask);
+  await run(PatchNativeModulesTask);
   await run(CleanExtraBinScriptsTask);
   await run(CleanExtraBrowsersTask);
   await run(CleanNodeBuildsTask);

src/dev/build/tasks/nodejs/download.js → src/dev/build/lib/download.js

@@ -24,7 +24,7 @@ import chalk from 'chalk';
 import { createHash } from 'crypto';
 import Axios from 'axios';
 
-import { mkdirp } from '../../lib';
+import { mkdirp } from './fs';
 
 function tryUnlink(path) {
   try {

src/dev/build/lib/index.js

@@ -33,5 +33,6 @@ export {
   compress,
   isFileAccessible,
 } from './fs';
+export { download } from './download';
 export { scanDelete } from './scan_delete';
 export { scanCopy } from './scan_copy';

src/dev/build/tasks/index.js

@@ -32,6 +32,7 @@ export * from './nodejs_modules';
 export * from './notice_file_task';
 export * from './optimize_task';
 export * from './os_packages';
+export * from './patch_native_modules_task';
 export * from './transpile_babel_task';
 export * from './transpile_scss_task';
 export * from './verify_env_task';

src/dev/build/tasks/nodejs/__tests__/download_node_builds_task.js

@@ -22,7 +22,7 @@ import expect from '@kbn/expect';
 
 import * as NodeShasumsNS from '../node_shasums';
 import * as NodeDownloadInfoNS from '../node_download_info';
-import * as DownloadNS from '../download';
+import * as DownloadNS from '../../../lib/download'; // sinon can't stub '../../../lib' properly
 import { DownloadNodeBuildsTask } from '../download_node_builds_task';
 
 describe('src/dev/build/tasks/nodejs/download_node_builds_task', () => {

src/dev/build/tasks/nodejs/download_node_builds_task.js

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-import { download } from './download';
+import { download } from '../../lib';
 import { getNodeShasums } from './node_shasums';
 import { getNodeDownloadInfo } from './node_download_info';
 

src/dev/build/tasks/patch_native_modules_task.js

@@ -0,0 +1,85 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import fs from 'fs';
+import path from 'path';
+import util from 'util';
+import { deleteAll, download, untar } from '../lib';
+
+const BASE_URL = `https://storage.googleapis.com/native-modules`;
+const DOWNLOAD_DIRECTORY = '.native_modules';
+
+const packages = [
+  {
+    name: 're2',
+    version: '1.10.5',
+    destinationPath: 'node_modules/re2/build/Release/',
+    shas: {
+      darwin: '066533b592094f91e00412499e44c338ce2466d63c9eaf0dc32be8214bde2099',
+      linux: '0322cac3c2e106129b650a8eac509f598ed283791d6116984fec4c151b24e574',
+      windows: '65b5bef7de2352f4787224c2c76a619b6683a868c8d4d71e0fdd500786fc422b',
+    },
+  },
+];
+
+async function getInstalledVersion(config, packageName) {
+  const packageJSONPath = config.resolveFromRepo(
+    path.join('node_modules', packageName, 'package.json')
+  );
+  const buffer = await util.promisify(fs.readFile)(packageJSONPath);
+  const packageJSON = JSON.parse(buffer);
+  return packageJSON.version;
+}
+
+async function patchModule(config, log, build, platform, pkg) {
+  const installedVersion = await getInstalledVersion(config, pkg.name);
+  if (installedVersion !== pkg.version) {
+    throw new Error(
+      `Can't patch ${pkg.name}'s native module, we were expecting version ${pkg.version} and found ${installedVersion}`
+    );
+  }
+  const platformName = platform.getName();
+  const archiveName = `${pkg.version}-${platformName}.tar.gz`;
+  const downloadUrl = `${BASE_URL}/${pkg.name}/${archiveName}`;
+  const downloadPath = config.resolveFromRepo(DOWNLOAD_DIRECTORY, archiveName);
+  const extractedPath = build.resolvePathForPlatform(platform, pkg.destinationPath);
+  log.debug(`Patching ${pkg.name} binaries from ${downloadUrl} to ${extractedPath}`);
+
+  await deleteAll([extractedPath], log);
+  await download({
+    log,
+    url: downloadUrl,
+    destination: downloadPath,
+    sha256: pkg.shas[platformName],
+    retries: 3,
+  });
+  await untar(downloadPath, extractedPath);
+}
+
+export const PatchNativeModulesTask = {
+  description: 'Patching platform-specific native modules',
+  async run(config, log, build) {
+    for (const pkg of packages) {
+      await Promise.all(
+        config.getTargetPlatforms().map(async platform => {
+          await patchModule(config, log, build, platform, pkg);
+        })
+      );
+    }
+  },
+};