Tag invalid objects instead of failing migrations

src/core/server/saved_objects/mappings/lib/get_root_properties_objects.ts

@@ -39,7 +39,7 @@ import { getRootProperties } from './get_root_properties';
  *  @return {EsPropertyMappings}
  */
 
-const blacklist = ['migrationVersion', 'references'];
+const blacklist = ['migrationVersion', 'references', 'valid', 'invalid_attributes'];
 
 export function getRootPropertiesObjects(mappings: IndexMapping) {
   const rootProperties = getRootProperties(mappings);

src/core/server/saved_objects/migrations/core/build_active_mappings.ts

@@ -125,7 +125,7 @@ function findChangedProp(actual: any, expected: any) {
  *
  * @returns {IndexMapping}
  */
-function defaultMapping(): IndexMapping {
+export function defaultMapping(): IndexMapping {
   return {
     dynamic: 'strict',
     properties: {
@@ -159,6 +159,14 @@ function defaultMapping(): IndexMapping {
           },
         },
       },
+      // "private" fields, not exposed through any API's
+      status: {
+        type: 'keyword',
+      },
+      unsafe_properties: {
+        enabled: false, // Don't index unsafe properties
+        type: 'object',
+      },
     },
   };
 }

src/core/server/saved_objects/migrations/core/document_migrator.test.ts

@@ -45,7 +45,6 @@ const createRegistry = (...types: Array<Partial<SavedObjectsType>>) => {
 describe('DocumentMigrator', () => {
   function testOpts() {
     return {
-      kibanaVersion: '25.2.3',
       typeRegistry: createRegistry(),
       validateDoc: _.noop,
       log: mockLogger,
@@ -54,7 +53,6 @@ describe('DocumentMigrator', () => {
 
   it('validates individual migration definitions', () => {
     const invalidDefinition = {
-      kibanaVersion: '3.2.3',
       typeRegistry: createRegistry({
         name: 'foo',
         migrations: _.noop as any,
@@ -69,7 +67,6 @@ describe('DocumentMigrator', () => {
 
   it('validates individual migration semvers', () => {
     const invalidDefinition = {
-      kibanaVersion: '3.2.3',
       typeRegistry: createRegistry({
         name: 'foo',
         migrations: {
@@ -86,7 +83,6 @@ describe('DocumentMigrator', () => {
 
   it('validates the migration function', () => {
     const invalidDefinition = {
-      kibanaVersion: '3.2.3',
       typeRegistry: createRegistry({
         name: 'foo',
         migrations: {
@@ -283,7 +279,6 @@ describe('DocumentMigrator', () => {
   it('rejects docs that belong to a newer Kibana instance', () => {
     const migrator = new DocumentMigrator({
       ...testOpts(),
-      kibanaVersion: '8.0.1',
     });
     expect(() =>
       migrator.migrate({
@@ -555,7 +550,9 @@ describe('DocumentMigrator', () => {
         name: 'dog',
         migrations: {
           '1.2.3': () => {
-            throw new Error('Dang diggity!');
+            const err = new Error('Dang diggity!');
+            err.stack = 'stack trace...';
+            throw err;
           },
         },
       }),
@@ -571,10 +568,24 @@ describe('DocumentMigrator', () => {
       migrator.migrate(_.cloneDeep(failedDoc));
       expect('Did not throw').toEqual('But it should have!');
     } catch (error) {
-      expect(error.message).toMatch(/Dang diggity!/);
-      const warning = loggingServiceMock.collect(mockLoggerFactory).warn[0][0];
-      expect(warning).toContain(JSON.stringify(failedDoc));
-      expect(warning).toContain('dog:1.2.3');
+      expect(loggingServiceMock.collect(mockLoggerFactory).warn[0]).toMatchInlineSnapshot(`
+        Array [
+          "Failed to apply the migration transform 'dog:1.2.3' to the document: dog:smelly",
+          Object {
+            "document": Object {
+              "attributes": Object {},
+              "id": "smelly",
+              "migrationVersion": Object {},
+              "type": "dog",
+            },
+            "error": Object {
+              "message": "Dang diggity!",
+              "name": "Error",
+              "stack": "stack trace...",
+            },
+          },
+        ]
+      `);
     }
   });
 

src/core/server/saved_objects/migrations/core/document_migrator.ts

@@ -76,7 +76,6 @@ export type TransformFn = (doc: SavedObjectUnsanitizedDoc) => SavedObjectUnsanit
 type ValidateDoc = (doc: SavedObjectUnsanitizedDoc) => void;
 
 interface DocumentMigratorOptions {
-  kibanaVersion: string;
   typeRegistry: ISavedObjectTypeRegistry;
   validateDoc: ValidateDoc;
   log: Logger;
@@ -118,12 +117,11 @@ export class DocumentMigrator implements VersionedTransformer {
    * @prop {Logger} log - The migration logger
    * @memberof DocumentMigrator
    */
-  constructor({ typeRegistry, kibanaVersion, log, validateDoc }: DocumentMigratorOptions) {
+  constructor({ typeRegistry, log, validateDoc }: DocumentMigratorOptions) {
     validateMigrationDefinition(typeRegistry);
 
     this.migrations = buildActiveMigrations(typeRegistry, log);
     this.transformDoc = buildDocumentTransform({
-      kibanaVersion,
       migrations: this.migrations,
       validateDoc,
     });
@@ -231,11 +229,9 @@ function buildActiveMigrations(
  * Creates a function which migrates and validates any document that is passed to it.
  */
 function buildDocumentTransform({
-  kibanaVersion,
   migrations,
   validateDoc,
 }: {
-  kibanaVersion: string;
   migrations: ActiveMigrations;
   validateDoc: ValidateDoc;
 }): TransformFn {
@@ -321,9 +317,9 @@ function wrapWithTry(
       return result;
     } catch (error) {
       const failedTransform = `${type}:${version}`;
-      const failedDoc = JSON.stringify(doc);
       log.warn(
-        `Failed to transform document ${doc}. Transform: ${failedTransform}\nDoc: ${failedDoc}`
+        `Failed to apply the migration transform '${failedTransform}' to the document: ${type}:${doc.id}`,
+        { document: doc, error: { message: error.message, name: error.name, stack: error.stack } }
       );
       throw error;
     }

src/core/server/saved_objects/migrations/core/index_migrator.test.ts

@@ -40,6 +40,7 @@ describe('IndexMigrator', () => {
         migrate: _.identity,
       },
       serializer: new SavedObjectsSerializer(new SavedObjectTypeRegistry()),
+      kibanaVersion: '7.0.0-test',
     };
   });
 
@@ -59,16 +60,19 @@ describe('IndexMigrator', () => {
           _meta: {
             migrationMappingPropertyHashes: {
               foo: '18c78c995965207ed3f6e7fc5c6e55fe',
+              unsafe_properties: '5ef305b18111b77789afefbd36b66171',
               migrationVersion: '4a1746014a75ade3a714e1db5763276f',
               namespace: '2f4316de49999235636386fe51dc06c1',
               namespaces: '2f4316de49999235636386fe51dc06c1',
               references: '7997cf5a56cc02bdc9c93361bde732b0',
               type: '2f4316de49999235636386fe51dc06c1',
               updated_at: '00da57df13e94e9d98437d13ace4bfe0',
+              status: '2f4316de49999235636386fe51dc06c1',
             },
           },
           properties: {
             foo: { type: 'long' },
+            unsafe_properties: { enabled: false, type: 'object' },
             migrationVersion: { dynamic: 'true', type: 'object' },
             namespace: { type: 'keyword' },
             namespaces: { type: 'keyword' },
@@ -82,6 +86,7 @@ describe('IndexMigrator', () => {
                 id: { type: 'keyword' },
               },
             },
+            status: { type: 'keyword' },
           },
         },
         settings: { number_of_shards: 1, auto_expand_replicas: '0-1' },
@@ -178,17 +183,20 @@ describe('IndexMigrator', () => {
           _meta: {
             migrationMappingPropertyHashes: {
               foo: '625b32086eb1d1203564cf85062dd22e',
+              unsafe_properties: '5ef305b18111b77789afefbd36b66171',
               migrationVersion: '4a1746014a75ade3a714e1db5763276f',
               namespace: '2f4316de49999235636386fe51dc06c1',
               namespaces: '2f4316de49999235636386fe51dc06c1',
               references: '7997cf5a56cc02bdc9c93361bde732b0',
               type: '2f4316de49999235636386fe51dc06c1',
               updated_at: '00da57df13e94e9d98437d13ace4bfe0',
+              status: '2f4316de49999235636386fe51dc06c1',
             },
           },
           properties: {
             author: { type: 'text' },
             foo: { type: 'text' },
+            unsafe_properties: { enabled: false, type: 'object' },
             migrationVersion: { dynamic: 'true', type: 'object' },
             namespace: { type: 'keyword' },
             namespaces: { type: 'keyword' },
@@ -202,6 +210,7 @@ describe('IndexMigrator', () => {
                 id: { type: 'keyword' },
               },
             },
+            status: { type: 'keyword' },
           },
         },
         settings: { number_of_shards: 1, auto_expand_replicas: '0-1' },
@@ -292,7 +301,7 @@ describe('IndexMigrator', () => {
       {
         body: [
           { index: { _id: 'foo:1', _index: '.kibana_2' } },
-          { foo: { name: 1 }, type: 'foo', migrationVersion: {}, references: [] },
+          { foo: { name: 1 }, type: 'foo', migrationVersion: {}, references: [], status: 'valid' },
         ],
       },
     ]);
@@ -301,7 +310,7 @@ describe('IndexMigrator', () => {
       {
         body: [
           { index: { _id: 'foo:2', _index: '.kibana_2' } },
-          { foo: { name: 2 }, type: 'foo', migrationVersion: {}, references: [] },
+          { foo: { name: 2 }, type: 'foo', migrationVersion: {}, references: [], status: 'valid' },
         ],
       },
     ]);

src/core/server/saved_objects/migrations/core/index_migrator.ts

@@ -195,7 +195,7 @@ async function migrateSourceToDest(context: Context) {
     await Index.write(
       callCluster,
       dest.indexName,
-      migrateRawDocs(serializer, documentMigrator.migrate, docs, log)
+      migrateRawDocs(serializer, documentMigrator.migrate, docs, log, context.kibanaVersion)
     );
   }
 }