Merge branch '7.x' into backport/7.x/pr-70936

elastic · Jul 9, 2020 · 52ca84e · 52ca84e
2 parents b52cded + 40479ec
commit 52ca84e
Show file tree

Hide file tree

Showing 71 changed files with 2,282 additions and 682 deletions.
diff --git a/docs/development/core/server/kibana-plugin-core-server.md b/docs/development/core/server/kibana-plugin-core-server.md
@@ -154,7 +154,7 @@ The plugin integrates with the core system via lifecycle events: `setup`<!-- -->
 |  [SavedObjectsBulkUpdateResponse](./kibana-plugin-core-server.savedobjectsbulkupdateresponse.md) |  |
 |  [SavedObjectsClientProviderOptions](./kibana-plugin-core-server.savedobjectsclientprovideroptions.md) | Options to control the creation of the Saved Objects Client. |
 |  [SavedObjectsClientWrapperOptions](./kibana-plugin-core-server.savedobjectsclientwrapperoptions.md) | Options passed to each SavedObjectsClientWrapperFactory to aid in creating the wrapper instance. |
-|  [SavedObjectsComplexFieldMapping](./kibana-plugin-core-server.savedobjectscomplexfieldmapping.md) | See [SavedObjectsFieldMapping](./kibana-plugin-core-server.savedobjectsfieldmapping.md) for documentation.<!-- -->Note: this type intentially doesn't include a type definition for defining the <code>dynamic</code> mapping parameter. Saved Object fields should always inherit the <code>dynamic: 'strict'</code> paramater. If you are unsure of the shape of your data use <code>type: 'object', enabled: false</code> instead. |
+|  [SavedObjectsComplexFieldMapping](./kibana-plugin-core-server.savedobjectscomplexfieldmapping.md) | See [SavedObjectsFieldMapping](./kibana-plugin-core-server.savedobjectsfieldmapping.md) for documentation. |
 |  [SavedObjectsCoreFieldMapping](./kibana-plugin-core-server.savedobjectscorefieldmapping.md) | See [SavedObjectsFieldMapping](./kibana-plugin-core-server.savedobjectsfieldmapping.md) for documentation. |
 |  [SavedObjectsCreateOptions](./kibana-plugin-core-server.savedobjectscreateoptions.md) |  |
 |  [SavedObjectsDeleteByNamespaceOptions](./kibana-plugin-core-server.savedobjectsdeletebynamespaceoptions.md) |  |

diff --git a/...ore/server/kibana-plugin-core-server.savedobjectscomplexfieldmapping.dynamic.md b/...ore/server/kibana-plugin-core-server.savedobjectscomplexfieldmapping.dynamic.md
@@ -0,0 +1,15 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [SavedObjectsComplexFieldMapping](./kibana-plugin-core-server.savedobjectscomplexfieldmapping.md) &gt; [dynamic](./kibana-plugin-core-server.savedobjectscomplexfieldmapping.dynamic.md)
+
+## SavedObjectsComplexFieldMapping.dynamic property
+
+The dynamic property of the mapping, either `false` or `'strict'`<!-- -->. If unspecified `dynamic: 'strict'` will be inherited from the top-level index mappings.
+
+Note: To limit the number of mapping fields Saved Object types should \*never\* use `dynamic: true`<!-- -->.
+
+<b>Signature:</b>
+
+```typescript
+dynamic?: false | 'strict';
+```
diff --git a/...ore/server/kibana-plugin-core-server.savedobjectscomplexfieldmapping.enabled.md b/...ore/server/kibana-plugin-core-server.savedobjectscomplexfieldmapping.enabled.md
@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [SavedObjectsComplexFieldMapping](./kibana-plugin-core-server.savedobjectscomplexfieldmapping.md) &gt; [enabled](./kibana-plugin-core-server.savedobjectscomplexfieldmapping.enabled.md)
+
+## SavedObjectsComplexFieldMapping.enabled property
+
+<b>Signature:</b>
+
+```typescript
+enabled?: boolean;
+```
diff --git a/...opment/core/server/kibana-plugin-core-server.savedobjectscomplexfieldmapping.md b/...opment/core/server/kibana-plugin-core-server.savedobjectscomplexfieldmapping.md
@@ -6,8 +6,6 @@
 
 See [SavedObjectsFieldMapping](./kibana-plugin-core-server.savedobjectsfieldmapping.md) for documentation.
 
-Note: this type intentially doesn't include a type definition for defining the `dynamic` mapping parameter. Saved Object fields should always inherit the `dynamic: 'strict'` paramater. If you are unsure of the shape of your data use `type: 'object', enabled: false` instead.
-
 <b>Signature:</b>
 
 ```typescript
@@ -19,6 +17,8 @@ export interface SavedObjectsComplexFieldMapping
 |  Property | Type | Description |
 |  --- | --- | --- |
 |  [doc\_values](./kibana-plugin-core-server.savedobjectscomplexfieldmapping.doc_values.md) | <code>boolean</code> |  |
+|  [dynamic](./kibana-plugin-core-server.savedobjectscomplexfieldmapping.dynamic.md) | <code>false &#124; 'strict'</code> | The dynamic property of the mapping, either <code>false</code> or <code>'strict'</code>. If unspecified <code>dynamic: 'strict'</code> will be inherited from the top-level index mappings.<!-- -->Note: To limit the number of mapping fields Saved Object types should \*never\* use <code>dynamic: true</code>. |
+|  [enabled](./kibana-plugin-core-server.savedobjectscomplexfieldmapping.enabled.md) | <code>boolean</code> |  |
 |  [properties](./kibana-plugin-core-server.savedobjectscomplexfieldmapping.properties.md) | <code>SavedObjectsMappingProperties</code> |  |
 |  [type](./kibana-plugin-core-server.savedobjectscomplexfieldmapping.type.md) | <code>string</code> |  |
 
diff --git a/...t/core/server/kibana-plugin-core-server.savedobjectscorefieldmapping.enabled.md b/...t/core/server/kibana-plugin-core-server.savedobjectscorefieldmapping.enabled.md
diff --git a/...velopment/core/server/kibana-plugin-core-server.savedobjectscorefieldmapping.md b/...velopment/core/server/kibana-plugin-core-server.savedobjectscorefieldmapping.md
@@ -17,7 +17,6 @@ export interface SavedObjectsCoreFieldMapping
 |  Property | Type | Description |
 |  --- | --- | --- |
 |  [doc\_values](./kibana-plugin-core-server.savedobjectscorefieldmapping.doc_values.md) | <code>boolean</code> |  |
-|  [enabled](./kibana-plugin-core-server.savedobjectscorefieldmapping.enabled.md) | <code>boolean</code> |  |
 |  [fields](./kibana-plugin-core-server.savedobjectscorefieldmapping.fields.md) | <code>{</code><br/><code>        [subfield: string]: {</code><br/><code>            type: string;</code><br/><code>            ignore_above?: number;</code><br/><code>        };</code><br/><code>    }</code> |  |
 |  [index](./kibana-plugin-core-server.savedobjectscorefieldmapping.index.md) | <code>boolean</code> |  |
 |  [null\_value](./kibana-plugin-core-server.savedobjectscorefieldmapping.null_value.md) | <code>number &#124; boolean &#124; string</code> |  |

diff --git a/...e/server/kibana-plugin-core-server.savedobjectstypemappingdefinition.dynamic.md b/...e/server/kibana-plugin-core-server.savedobjectstypemappingdefinition.dynamic.md
@@ -4,7 +4,7 @@
 
 ## SavedObjectsTypeMappingDefinition.dynamic property
 
-The dynamic property of the mapping. either `false` or 'strict'. Defaults to `false`
+The dynamic property of the mapping, either `false` or `'strict'`<!-- -->. If unspecified `dynamic: 'strict'` will be inherited from the top-level index mappings.
 
 <b>Signature:</b>
 

diff --git a/...ment/core/server/kibana-plugin-core-server.savedobjectstypemappingdefinition.md b/...ment/core/server/kibana-plugin-core-server.savedobjectstypemappingdefinition.md
@@ -41,6 +41,6 @@ const typeDefinition: SavedObjectsTypeMappingDefinition = {
 
 |  Property | Type | Description |
 |  --- | --- | --- |
-|  [dynamic](./kibana-plugin-core-server.savedobjectstypemappingdefinition.dynamic.md) | <code>false &#124; 'strict'</code> | The dynamic property of the mapping. either <code>false</code> or 'strict'. Defaults to <code>false</code> |
+|  [dynamic](./kibana-plugin-core-server.savedobjectstypemappingdefinition.dynamic.md) | <code>false &#124; 'strict'</code> | The dynamic property of the mapping, either <code>false</code> or <code>'strict'</code>. If unspecified <code>dynamic: 'strict'</code> will be inherited from the top-level index mappings. |
 |  [properties](./kibana-plugin-core-server.savedobjectstypemappingdefinition.properties.md) | <code>SavedObjectsMappingProperties</code> | The underlying properties of the type mapping |
 
diff --git a/packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/reporter.js b/packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/reporter.js
@@ -37,6 +37,8 @@ export function MochaReporterProvider({ getService }) {
   let originalLogWriters;
   let reporterCaptureStartTime;
 
+  const failuresOverTime = [];
+
   return class MochaReporter extends Mocha.reporters.Base {
     constructor(runner, options) {
       super(runner, options);
@@ -155,30 +157,50 @@ export function MochaReporterProvider({ getService }) {
       //  - I started by trying to extract the Base.list() logic from mocha
       //    but it's a lot more complicated than this is horrible.
       //  - In order to fix the numbering and indentation we monkey-patch
-      //    console.log and parse the logged output.
+      //    Mocha.reporters.Base.consoleLog and parse the logged output.
       //
       let output = '';
-      const realLog = console.log;
-      console.log = (...args) => (output += `${format(...args)}\n`);
+      const realLog = Mocha.reporters.Base.consoleLog;
+      Mocha.reporters.Base.consoleLog = (...args) => (output += `${format(...args)}\n`);
       try {
         Mocha.reporters.Base.list([runnable]);
       } finally {
-        console.log = realLog;
+        Mocha.reporters.Base.consoleLog = realLog;
       }
 
+      const outputLines = output.split('\n');
+
+      const errorMarkerStart = outputLines.reduce((index, line, i) => {
+        if (index >= 0) {
+          return index;
+        }
+        return /Error:/.test(line) ? i : index;
+      }, -1);
+
+      const errorMessage = outputLines
+        // drop the first ${errorMarkerStart} lines, (empty + test title)
+        .slice(errorMarkerStart)
+        // move leading colors behind leading spaces
+        .map((line) => line.replace(/^((?:\[.+m)+)(\s+)/, '$2$1'))
+        .map((line) => ` ${line}`)
+        .join('\n');
+
       log.write(
-        `- ${colors.fail(`${symbols.err} fail: "${runnable.fullTitle()}"`)}` +
-          '\n' +
-          output
-            .split('\n')
-            // drop the first two lines, (empty + test title)
-            .slice(2)
-            // move leading colors behind leading spaces
-            .map((line) => line.replace(/^((?:\[.+m)+)(\s+)/, '$2$1'))
-            .map((line) => ` ${line}`)
-            .join('\n')
+        `- ${colors.fail(`${symbols.err} fail: ${runnable.fullTitle()}`)}` + '\n' + errorMessage
       );
 
+      // Prefer to reuse the nice Mocha nested title format for final summary
+      const nestedTitleFormat = outputLines
+        .slice(1, errorMarkerStart)
+        .join('\n')
+        // make sure to remove the list number
+        .replace(/\d+\)/, '');
+
+      failuresOverTime.push({
+        title: nestedTitleFormat,
+        error: errorMessage,
+      });
+
       // failed hooks trigger the `onFail(runnable)` callback, so we snapshot the logs for
       // them here. Tests will re-capture the snapshot in `onTestEnd()`
       snapshotLogsForRunnable(runnable);
@@ -189,7 +211,7 @@ export function MochaReporterProvider({ getService }) {
         log.setWriters(originalLogWriters);
       }
 
-      writeEpilogue(log, this.stats);
+      writeEpilogue(log, this.stats, failuresOverTime);
     };
   };
 }
diff --git a/packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/write_epilogue.js b/packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/write_epilogue.js
@@ -20,7 +20,7 @@
 import * as colors from './colors';
 import { ms } from './ms';
 
-export function writeEpilogue(log, stats) {
+export function writeEpilogue(log, stats, failuresDetail) {
   // header
   log.write('');
 
@@ -35,6 +35,12 @@ export function writeEpilogue(log, stats) {
   // failures
   if (stats.failures) {
     log.write('%d failing', stats.failures);
+    log.write('');
+    failuresDetail.forEach(({ title, error }, i) => {
+      log.write('%d) %s', i + 1, title);
+      log.write('');
+      log.write('%s', error);
+    });
   }
 
   // footer

diff --git a/src/core/server/saved_objects/mappings/types.ts b/src/core/server/saved_objects/mappings/types.ts
@@ -45,7 +45,9 @@
  * @public
  */
 export interface SavedObjectsTypeMappingDefinition {
-  /** The dynamic property of the mapping. either `false` or 'strict'. Defaults to `false` */
+  /** The dynamic property of the mapping, either `false` or `'strict'`. If
+   * unspecified `dynamic: 'strict'` will be inherited from the top-level
+   * index mappings. */
   dynamic?: false | 'strict';
   /** The underlying properties of the type mapping */
   properties: SavedObjectsMappingProperties;
@@ -134,7 +136,6 @@ export interface SavedObjectsCoreFieldMapping {
   null_value?: number | boolean | string;
   index?: boolean;
   doc_values?: boolean;
-  enabled?: boolean;
   fields?: {
     [subfield: string]: {
       type: string;
@@ -146,14 +147,19 @@ export interface SavedObjectsCoreFieldMapping {
 /**
  * See {@link SavedObjectsFieldMapping} for documentation.
  *
- * Note: this type intentially doesn't include a type definition for defining
- * the `dynamic` mapping parameter. Saved Object fields should always inherit
- * the `dynamic: 'strict'` paramater. If you are unsure of the shape of your
- * data use `type: 'object', enabled: false` instead.
- *
  * @public
  */
 export interface SavedObjectsComplexFieldMapping {
+  /**
+   * The dynamic property of the mapping, either `false` or `'strict'`. If
+   * unspecified `dynamic: 'strict'` will be inherited from the top-level
+   * index mappings.
+   *
+   * Note: To limit the number of mapping fields Saved Object types should
+   * *never* use `dynamic: true`.
+   */
+  dynamic?: false | 'strict';
+  enabled?: boolean;
   doc_values?: boolean;
   type?: string;
   properties: SavedObjectsMappingProperties;

diff --git a/src/core/server/saved_objects/migrations/core/index_migrator.test.ts b/src/core/server/saved_objects/migrations/core/index_migrator.test.ts
@@ -151,7 +151,7 @@ describe('IndexMigrator', () => {
     );
   });
 
-  test('retains mappings from the previous index', async () => {
+  test('retains unknown core field mappings from the previous index', async () => {
     const { callCluster } = testOpts;
 
     testOpts.mappingProperties = { foo: { type: 'text' } };
@@ -162,7 +162,7 @@ describe('IndexMigrator', () => {
           aliases: {},
           mappings: {
             properties: {
-              author: { type: 'text' },
+              unknown_core_field: { type: 'text' },
             },
           },
         },
@@ -187,7 +187,66 @@ describe('IndexMigrator', () => {
             },
           },
           properties: {
-            author: { type: 'text' },
+            unknown_core_field: { type: 'text' },
+            foo: { type: 'text' },
+            migrationVersion: { dynamic: 'true', type: 'object' },
+            namespace: { type: 'keyword' },
+            namespaces: { type: 'keyword' },
+            type: { type: 'keyword' },
+            updated_at: { type: 'date' },
+            references: {
+              type: 'nested',
+              properties: {
+                name: { type: 'keyword' },
+                type: { type: 'keyword' },
+                id: { type: 'keyword' },
+              },
+            },
+          },
+        },
+        settings: { number_of_shards: 1, auto_expand_replicas: '0-1' },
+      },
+      index: '.kibana_2',
+    });
+  });
+
+  test('disables complex field mappings from unknown types in the previous index', async () => {
+    const { callCluster } = testOpts;
+
+    testOpts.mappingProperties = { foo: { type: 'text' } };
+
+    withIndex(callCluster, {
+      index: {
+        '.kibana_1': {
+          aliases: {},
+          mappings: {
+            properties: {
+              unknown_complex_field: { properties: { description: { type: 'text' } } },
+            },
+          },
+        },
+      },
+    });
+
+    await new IndexMigrator(testOpts).migrate();
+
+    expect(callCluster).toHaveBeenCalledWith('indices.create', {
+      body: {
+        mappings: {
+          dynamic: 'strict',
+          _meta: {
+            migrationMappingPropertyHashes: {
+              foo: '625b32086eb1d1203564cf85062dd22e',
+              migrationVersion: '4a1746014a75ade3a714e1db5763276f',
+              namespace: '2f4316de49999235636386fe51dc06c1',
+              namespaces: '2f4316de49999235636386fe51dc06c1',
+              references: '7997cf5a56cc02bdc9c93361bde732b0',
+              type: '2f4316de49999235636386fe51dc06c1',
+              updated_at: '00da57df13e94e9d98437d13ace4bfe0',
+            },
+          },
+          properties: {
+            unknown_complex_field: { dynamic: false, properties: {} },
             foo: { type: 'text' },
             migrationVersion: { dynamic: 'true', type: 'object' },
             namespace: { type: 'keyword' },