[Maps] implement references for saved objects (#31745)

* [Maps] implement references for saved objects

* add source to ref name, check that source type is ES_SEARCH or ES_GEO_GRID

* extract out common find reference into a function

* add migration version to sample data objects

* joins are on layer descriptor and not source descriptor

* update one es_archive saved object to have layerListJSON stored in references to ensure injectReferences is really working in SavedGisMap

* update sample data saved objects to include applied migration

* add API test to verify migration is applied when imported saved object

x-pack/plugins/maps/common/constants.js

@@ -7,6 +7,8 @@
 export const GIS_API_PATH = 'api/maps';
 
 export const EMS_FILE = 'EMS_FILE';
+export const ES_GEO_GRID = 'ES_GEO_GRID';
+export const ES_SEARCH = 'ES_SEARCH';
 
 export const DECIMAL_DEGREES_PRECISION = 5; // meters precision
 

x-pack/plugins/maps/common/migrations/references.js

@@ -0,0 +1,104 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+// Can not use public Layer classes to extract references since this logic must run in both client and server.
+
+import _ from 'lodash';
+import { ES_GEO_GRID, ES_SEARCH } from '../constants';
+
+function doesSourceUseIndexPattern(layerDescriptor) {
+  const sourceType = _.get(layerDescriptor, 'sourceDescriptor.type');
+  return sourceType === ES_GEO_GRID || sourceType === ES_SEARCH;
+}
+
+export function extractReferences({ attributes, references = [] }) {
+  if (!attributes.layerListJSON) {
+    return { attributes, references };
+  }
+
+  const extractedReferences = [];
+
+  const layerList = JSON.parse(attributes.layerListJSON);
+  layerList.forEach((layer, layerIndex) => {
+
+    // Extract index-pattern references from source descriptor
+    if (doesSourceUseIndexPattern(layer) && _.has(layer, 'sourceDescriptor.indexPatternId')) {
+      const refName = `layer_${layerIndex}_source_index_pattern`;
+      extractedReferences.push({
+        name: refName,
+        type: 'index-pattern',
+        id: layer.sourceDescriptor.indexPatternId,
+      });
+      delete layer.sourceDescriptor.indexPatternId;
+      layer.sourceDescriptor.indexPatternRefName = refName;
+    }
+
+    // Extract index-pattern references from join
+    const joins = _.get(layer, 'joins', []);
+    joins.forEach((join, joinIndex) => {
+      if (_.has(join, 'right.indexPatternId')) {
+        const refName = `layer_${layerIndex}_join_${joinIndex}_index_pattern`;
+        extractedReferences.push({
+          name: refName,
+          type: 'index-pattern',
+          id: join.right.indexPatternId,
+        });
+        delete join.right.indexPatternId;
+        join.right.indexPatternRefName = refName;
+      }
+    });
+  });
+
+  return {
+    attributes: {
+      ...attributes,
+      layerListJSON: JSON.stringify(layerList),
+    },
+    references: references.concat(extractedReferences),
+  };
+}
+
+function findReference(targetName, references) {
+  const reference = references.find(reference => reference.name === targetName);
+  if (!reference) {
+    throw new Error(`Could not find reference "${targetName}"`);
+  }
+  return reference;
+}
+
+export function injectReferences({ attributes, references }) {
+  if (!attributes.layerListJSON) {
+    return { attributes };
+  }
+
+  const layerList = JSON.parse(attributes.layerListJSON);
+  layerList.forEach((layer) => {
+
+    // Inject index-pattern references into source descriptor
+    if (doesSourceUseIndexPattern(layer) && _.has(layer, 'sourceDescriptor.indexPatternRefName')) {
+      const reference = findReference(layer.sourceDescriptor.indexPatternRefName, references);
+      layer.sourceDescriptor.indexPatternId = reference.id;
+      delete layer.sourceDescriptor.indexPatternRefName;
+    }
+
+    // Inject index-pattern references into join
+    const joins = _.get(layer, 'joins', []);
+    joins.forEach((join) => {
+      if (_.has(join, 'right.indexPatternRefName')) {
+        const reference = findReference(join.right.indexPatternRefName, references);
+        join.right.indexPatternId = reference.id;
+        delete join.right.indexPatternRefName;
+      }
+    });
+  });
+
+  return {
+    attributes: {
+      ...attributes,
+      layerListJSON: JSON.stringify(layerList),
+    },
+  };
+}

x-pack/plugins/maps/common/migrations/references.test.js

@@ -0,0 +1,173 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint max-len: 0 */
+
+import { extractReferences, injectReferences } from './references';
+import { ES_GEO_GRID, ES_SEARCH } from '../constants';
+
+const layerListJSON = {
+  esSearchSource: {
+    withIndexPatternId: `[{\"sourceDescriptor\":{\"type\":\"${ES_SEARCH}\",\"indexPatternId\":\"c698b940-e149-11e8-a35a-370a8516603a\"}}]`,
+    withIndexPatternRef: `[{\"sourceDescriptor\":{\"type\":\"${ES_SEARCH}\",\"indexPatternRefName\":\"layer_0_source_index_pattern\"}}]`,
+  },
+  esGeoGridSource: {
+    withIndexPatternId: `[{\"sourceDescriptor\":{\"type\":\"${ES_GEO_GRID}\",\"indexPatternId\":\"c698b940-e149-11e8-a35a-370a8516603a\"}}]`,
+    withIndexPatternRef: `[{\"sourceDescriptor\":{\"type\":\"${ES_GEO_GRID}\",\"indexPatternRefName\":\"layer_0_source_index_pattern\"}}]`,
+  },
+  join: {
+    withIndexPatternId: '[{\"joins\":[{\"right\":{\"indexPatternId\":\"e20b2a30-f735-11e8-8ce0-9723965e01e3\"}}]}]',
+    withIndexPatternRef: '[{\"joins\":[{\"right\":{\"indexPatternRefName\":\"layer_0_join_0_index_pattern\"}}]}]',
+  }
+};
+
+describe('extractReferences', () => {
+
+  test('Should handle missing layerListJSON attribute', () => {
+    const attributes = {
+      title: 'my map',
+    };
+    expect(extractReferences({ attributes })).toEqual({
+      attributes: {
+        title: 'my map',
+      },
+      references: [],
+    });
+  });
+
+  test('Should extract index-pattern reference from ES search source descriptor', () => {
+    const attributes = {
+      title: 'my map',
+      layerListJSON: layerListJSON.esSearchSource.withIndexPatternId,
+    };
+    expect(extractReferences({ attributes })).toEqual({
+      attributes: {
+        title: 'my map',
+        layerListJSON: layerListJSON.esSearchSource.withIndexPatternRef,
+      },
+      references: [
+        {
+          id: 'c698b940-e149-11e8-a35a-370a8516603a',
+          name: 'layer_0_source_index_pattern',
+          type: 'index-pattern',
+        }
+      ],
+    });
+  });
+
+  test('Should extract index-pattern reference from ES geo grid source descriptor', () => {
+    const attributes = {
+      title: 'my map',
+      layerListJSON: layerListJSON.esGeoGridSource.withIndexPatternId,
+    };
+    expect(extractReferences({ attributes })).toEqual({
+      attributes: {
+        title: 'my map',
+        layerListJSON: layerListJSON.esGeoGridSource.withIndexPatternRef,
+      },
+      references: [
+        {
+          id: 'c698b940-e149-11e8-a35a-370a8516603a',
+          name: 'layer_0_source_index_pattern',
+          type: 'index-pattern',
+        }
+      ],
+    });
+  });
+
+  test('Should extract index-pattern reference from joins', () => {
+    const attributes = {
+      title: 'my map',
+      layerListJSON: layerListJSON.join.withIndexPatternId,
+    };
+    expect(extractReferences({ attributes })).toEqual({
+      attributes: {
+        title: 'my map',
+        layerListJSON: layerListJSON.join.withIndexPatternRef,
+      },
+      references: [
+        {
+          id: 'e20b2a30-f735-11e8-8ce0-9723965e01e3',
+          name: 'layer_0_join_0_index_pattern',
+          type: 'index-pattern',
+        }
+      ],
+    });
+  });
+});
+
+describe('injectReferences', () => {
+  test('Should handle missing layerListJSON attribute', () => {
+    const attributes = {
+      title: 'my map',
+    };
+    expect(injectReferences({ attributes })).toEqual({
+      attributes: {
+        title: 'my map',
+      }
+    });
+  });
+
+  test('Should inject index-pattern reference into ES search source descriptor', () => {
+    const attributes = {
+      title: 'my map',
+      layerListJSON: layerListJSON.esSearchSource.withIndexPatternRef,
+    };
+    const references = [
+      {
+        id: 'c698b940-e149-11e8-a35a-370a8516603a',
+        name: 'layer_0_source_index_pattern',
+        type: 'index-pattern',
+      }
+    ];
+    expect(injectReferences({ attributes, references })).toEqual({
+      attributes: {
+        title: 'my map',
+        layerListJSON: layerListJSON.esSearchSource.withIndexPatternId,
+      }
+    });
+  });
+
+  test('Should inject index-pattern reference into ES geo grid source descriptor', () => {
+    const attributes = {
+      title: 'my map',
+      layerListJSON: layerListJSON.esGeoGridSource.withIndexPatternRef,
+    };
+    const references = [
+      {
+        id: 'c698b940-e149-11e8-a35a-370a8516603a',
+        name: 'layer_0_source_index_pattern',
+        type: 'index-pattern',
+      }
+    ];
+    expect(injectReferences({ attributes, references })).toEqual({
+      attributes: {
+        title: 'my map',
+        layerListJSON: layerListJSON.esGeoGridSource.withIndexPatternId,
+      }
+    });
+  });
+
+  test('Should inject index-pattern reference into joins', () => {
+    const attributes = {
+      title: 'my map',
+      layerListJSON: layerListJSON.join.withIndexPatternRef,
+    };
+    const references = [
+      {
+        id: 'e20b2a30-f735-11e8-8ce0-9723965e01e3',
+        name: 'layer_0_join_0_index_pattern',
+        type: 'index-pattern',
+      }
+    ];
+    expect(injectReferences({ attributes, references })).toEqual({
+      attributes: {
+        title: 'my map',
+        layerListJSON: layerListJSON.join.withIndexPatternId,
+      }
+    });
+  });
+});

x-pack/plugins/maps/index.js

@@ -11,6 +11,7 @@ import fligthsSavedObjects from './server/sample_data/flights_saved_objects.json
 import webLogsSavedObjects from './server/sample_data/web_logs_saved_objects.json';
 import mappings from './mappings.json';
 import { checkLicense } from './check_license';
+import { migrations } from './migrations';
 import { watchStatusAndLicenseToInitialize } from
   '../../server/lib/watch_status_and_license_to_initialize';
 import { initTelemetryCollection } from './server/maps_telemetry';
@@ -48,7 +49,8 @@ export function maps(kibana) {
           isNamespaceAgnostic: true
         }
       },
-      mappings
+      mappings,
+      migrations,
     },
     config(Joi) {
       return Joi.object({

x-pack/plugins/maps/migrations.js

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { extractReferences } from './common/migrations/references';
+
+export const migrations = {
+  'map': {
+    '7.1.0': (doc) => {
+      const { attributes, references } = extractReferences(doc);
+
+      return {
+        ...doc,
+        attributes,
+        references,
+      };
+    },
+  },
+};

x-pack/plugins/maps/public/angular/services/saved_gis_map.js

@@ -19,6 +19,7 @@ import {
 } from '../../selectors/map_selectors';
 import { convertMapExtentToPolygon } from '../../elasticsearch_geo_utils';
 import { copyPersistentState } from '../../store/util';
+import { extractReferences, injectReferences } from '../../../common/migrations/references';
 
 const module = uiModules.get('app/maps');
 
@@ -30,6 +31,15 @@ module.factory('SavedGisMap', function (Private) {
       type: SavedGisMap.type,
       mapping: SavedGisMap.mapping,
       searchSource: SavedGisMap.searchsource,
+      extractReferences,
+      injectReferences: (savedObject, references) => {
+        const { attributes } = injectReferences({
+          attributes: { layerListJSON: savedObject.layerListJSON },
+          references
+        });
+
+        savedObject.layerListJSON = attributes.layerListJSON;
+      },
 
       // if this is null/undefined then the SavedObject will be assigned the defaults
       id: id,

x-pack/plugins/maps/public/shared/layers/sources/es_geo_grid_source/es_geo_grid_source.js

@@ -19,6 +19,7 @@ import { RENDER_AS } from './render_as';
 import { CreateSourceEditor } from './create_source_editor';
 import { UpdateSourceEditor } from './update_source_editor';
 import { GRID_RESOLUTION } from '../../grid_resolution';
+import { ES_GEO_GRID } from '../../../../../common/constants';
 import { filterPropertiesForTooltip } from '../../util';
 
 const COUNT_PROP_LABEL = 'count';
@@ -49,7 +50,7 @@ const aggSchemas = new Schemas([
 
 export class ESGeoGridSource extends AbstractESSource {
 
-  static type = 'ES_GEO_GRID';
+  static type = ES_GEO_GRID;
   static title = 'Grid aggregation';
   static description = 'Geospatial data grouped in grids with metrics for each gridded cell';
 

x-pack/plugins/maps/public/shared/layers/sources/es_search_source/es_search_source.js

@@ -12,12 +12,13 @@ import { AbstractESSource } from '../es_source';
 import { hitsToGeoJson } from '../../../../elasticsearch_geo_utils';
 import { CreateSourceEditor } from './create_source_editor';
 import { UpdateSourceEditor } from './update_source_editor';
+import { ES_SEARCH } from '../../../../../common/constants';
 
 const DEFAULT_LIMIT = 2048;
 
 export class ESSearchSource extends AbstractESSource {
 
-  static type = 'ES_SEARCH';
+  static type = ES_SEARCH;
   static title = 'Documents';
   static description = 'Geospatial data from a Kibana index pattern';