From b6f7fb9460f93a5fd20120ef8f2445f6b3e393f9 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Wed, 16 Oct 2019 14:03:38 -0600 Subject: [PATCH 1/3] [Maps] blended ES document source that shows clusters and documents --- .../legacy/plugins/maps/common/constants.js | 2 +- .../es_geo_grid_source/es_geo_grid_source.js | 3 + .../sources/es_geo_grid_source/index.js | 2 + .../es_search_source/es_search_source.js | 121 +++++++++++++++++- 4 files changed, 122 insertions(+), 6 deletions(-) diff --git a/x-pack/legacy/plugins/maps/common/constants.js b/x-pack/legacy/plugins/maps/common/constants.js index 1fd1f4b43bbdad..24bb623063a311 100644 --- a/x-pack/legacy/plugins/maps/common/constants.js +++ b/x-pack/legacy/plugins/maps/common/constants.js @@ -61,7 +61,7 @@ export const GEOJSON_FILE = 'GEOJSON_FILE'; export const DECIMAL_DEGREES_PRECISION = 5; // meters precision export const ZOOM_PRECISION = 2; -export const ES_SIZE_LIMIT = 10000; +export const ES_SIZE_LIMIT = 1000; export const FEATURE_ID_PROPERTY_NAME = '__kbn__feature_id__'; export const FEATURE_VISIBLE_PROPERTY_NAME = '__kbn__isvisible__'; diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js index 8416ef5709e309..1a3b6404bcde9b 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js @@ -186,6 +186,7 @@ export class ESGeoGridSource extends AbstractESSource { async getGeoJsonWithMeta(layerName, searchFilters, registerCancelCallback) { const indexPattern = await this._getIndexPattern(); const searchSource = await this._makeSearchSource(searchFilters, 0); + console.log('precision', searchFilters.geogridPrecision); const aggConfigs = new AggConfigs(indexPattern, this._makeAggConfigs(searchFilters.geogridPrecision), aggSchemas.all); searchSource.setField('aggs', aggConfigs.toDsl()); const esResponse = await this._runEsQuery( @@ -202,6 +203,8 @@ export class ESGeoGridSource extends AbstractESSource { renderAs: this._descriptor.requestType, }); + console.log('grid feature collection', featureCollection); + return { data: featureCollection, meta: { diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/index.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/index.js index 58d74c04c55529..f86522ac518929 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/index.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/index.js @@ -5,3 +5,5 @@ */ export { ESGeoGridSource } from './es_geo_grid_source'; + +export { getTileBoundingBox } from './geo_tile_utils'; diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js index e5fbde681b3083..45549834d179b3 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js @@ -10,12 +10,14 @@ import uuid from 'uuid/v4'; import { VECTOR_SHAPE_TYPES } from '../vector_feature_types'; import { AbstractESSource } from '../es_source'; +import { ESGeoGridSource, getTileBoundingBox } from '../es_geo_grid_source'; import { SearchSource } from '../../../kibana_services'; import { hitsToGeoJson } from '../../../elasticsearch_geo_utils'; import { CreateSourceEditor } from './create_source_editor'; import { UpdateSourceEditor } from './update_source_editor'; import { ES_SEARCH, + ES_GEO_GRID, ES_GEO_FIELD_TYPE, ES_SIZE_LIMIT, FEATURE_ID_PROPERTY_NAME, @@ -28,6 +30,8 @@ import { getSourceFields } from '../../../index_pattern_util'; import { DEFAULT_FILTER_BY_MAP_BOUNDS } from './constants'; +const CLUSTER_ID_PREFIX = 'cluster:'; + export class ESSearchSource extends AbstractESSource { static type = ES_SEARCH; @@ -273,7 +277,7 @@ export class ESSearchSource extends AbstractESSource { // searchFilters.fieldNames contains geo field and any fields needed for styling features // Performs Elasticsearch search request being careful to pull back only required fields to minimize response size - async _getSearchHits(layerName, searchFilters, registerCancelCallback) { + async _getSearchHits(layerName, searchFilters, registerCancelCallback, limit) { const initialSearchContext = { docvalue_fields: await this._getDateDocvalueFields(searchFilters.fieldNames) }; @@ -285,12 +289,12 @@ export class ESSearchSource extends AbstractESSource { // 1) Returns geo_point in a consistent format regardless of how geo_point is stored in source // 2) Setting _source to false so we avoid pulling back unneeded fields. initialSearchContext.docvalue_fields.push(...(await this._excludeDateFields(searchFilters.fieldNames))); - searchSource = await this._makeSearchSource(searchFilters, ES_SIZE_LIMIT, initialSearchContext); + searchSource = await this._makeSearchSource(searchFilters, limit, initialSearchContext); searchSource.setField('source', false); // do not need anything from _source searchSource.setField('fields', searchFilters.fieldNames); // Setting "fields" filters out unused scripted fields } else { // geo_shape fields do not support docvalue_fields yet, so still have to be pulled from _source - searchSource = await this._makeSearchSource(searchFilters, ES_SIZE_LIMIT, initialSearchContext); + searchSource = await this._makeSearchSource(searchFilters, limit, initialSearchContext); // Setting "fields" instead of "source: { includes: []}" // because SearchSource automatically adds the following by default // 1) all scripted fields @@ -323,10 +327,10 @@ export class ESSearchSource extends AbstractESSource { return !!sortField && !!sortOrder; } - async getGeoJsonWithMeta(layerName, searchFilters, registerCancelCallback) { + async _getGeoJsonFromHits(layerName, searchFilters, registerCancelCallback, limit) { const { hits, meta } = this._isTopHits() ? await this._getTopHits(layerName, searchFilters, registerCancelCallback) - : await this._getSearchHits(layerName, searchFilters, registerCancelCallback); + : await this._getSearchHits(layerName, searchFilters, registerCancelCallback, limit); const indexPattern = await this._getIndexPattern(); const unusedMetaFields = indexPattern.metaFields.filter(metaField => { @@ -360,6 +364,95 @@ export class ESSearchSource extends AbstractESSource { }; } + async _getGeoJsonFromBlended(layerName, searchFilters, registerCancelCallback) { + // Use es_grid_source to get clustered view + const gridSourceDescriptor = { + type: ES_GEO_GRID, + resolution: 'COARSE', + id: this.getId(), + geoField: this._descriptor.geoField, + requestType: 'grid', + metrics: [ + { + type: 'count', + } + ], + indexPatternId: this._descriptor.indexPatternId + }; + const gridSource = new ESGeoGridSource(gridSourceDescriptor, this._inspectorAdapters); + const gridSearchFilters = { + ...searchFilters, + geogridPrecision: gridSource.getGeoGridPrecision(searchFilters.zoom), + }; + const { data: gridGeoJson } = await gridSource.getGeoJsonWithMeta(layerName, gridSearchFilters, registerCancelCallback); + + const totalCount = gridGeoJson.features.reduce((total, feature) => { + return total + feature.properties.doc_count; + }, 0); + + if (totalCount <= ES_SIZE_LIMIT) { + return this._getGeoJsonFromHits(layerName, searchFilters, registerCancelCallback, ES_SIZE_LIMIT); + } + + const clusterLimit = ES_SIZE_LIMIT / 10; + const fetchPromises = gridGeoJson.features + .filter(feature => { + return feature.properties.doc_count <= clusterLimit; + }) + .map(feature => { + const { top, bottom, right, left } = getTileBoundingBox(feature.properties[FEATURE_ID_PROPERTY_NAME]); + const hitsSearchFilters = { + ...searchFilters, + buffer: { + maxLat: top, + maxLon: right, + minLat: bottom, + minLon: left, + }, + }; + return this._getGeoJsonFromHits(layerName, hitsSearchFilters, registerCancelCallback, clusterLimit); + }); + + const results = await Promise.all(fetchPromises); + + const featureCollection = { + type: 'FeatureCollection', + features: [] + }; + + // put clusters into feature collection + gridGeoJson.features + .filter(feature => { + return feature.properties.doc_count > clusterLimit; + }) + .forEach(feature => { + feature.properties[FEATURE_ID_PROPERTY_NAME] = `${CLUSTER_ID_PREFIX}${feature.properties[FEATURE_ID_PROPERTY_NAME]}`; + featureCollection.features.push(feature); + }); + + // put documents into feature collection + results.forEach(({ data }) => { + featureCollection.features.push(...data.features); + }); + + return { + data: featureCollection, + meta: { + areResultsTrimmed: true + } + }; + } + + async getGeoJsonWithMeta(layerName, searchFilters, registerCancelCallback) { + const geoField = await this._getGeoField(); + + if (this._isTopHits() || geoField.type !== ES_GEO_FIELD_TYPE.GEO_POINT) { + return this._getGeoJsonFromHits(layerName, searchFilters, registerCancelCallback, ES_SIZE_LIMIT); + } else { + return this._getGeoJsonFromBlended(layerName, searchFilters, registerCancelCallback); + } + } + canFormatFeatureProperties() { return this._descriptor.tooltipProperties.length > 0; } @@ -401,6 +494,24 @@ export class ESSearchSource extends AbstractESSource { } async filterAndFormatPropertiesToHtml(properties) { + if (properties[FEATURE_ID_PROPERTY_NAME].startsWith(CLUSTER_ID_PREFIX)) { + const gridSourceDescriptor = { + type: ES_GEO_GRID, + resolution: 'COARSE', + id: this.getId(), + geoField: this._descriptor.geoField, + requestType: 'grid', + metrics: [ + { + type: 'count', + } + ], + indexPatternId: this._descriptor.indexPatternId + }; + const gridSource = new ESGeoGridSource(gridSourceDescriptor, this._inspectorAdapters); + return gridSource.filterAndFormatPropertiesToHtml(properties); + } + const indexPattern = await this._getIndexPattern(); const propertyValues = await this._loadTooltipProperties(properties[FEATURE_ID_PROPERTY_NAME], indexPattern); From faa3fa5a60e6c53fd92c1c202d29297d2e77b5c1 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Wed, 16 Oct 2019 14:06:46 -0600 Subject: [PATCH 2/3] fix ES_SIZE_LIMIT --- x-pack/legacy/plugins/maps/common/constants.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/maps/common/constants.js b/x-pack/legacy/plugins/maps/common/constants.js index 24bb623063a311..1fd1f4b43bbdad 100644 --- a/x-pack/legacy/plugins/maps/common/constants.js +++ b/x-pack/legacy/plugins/maps/common/constants.js @@ -61,7 +61,7 @@ export const GEOJSON_FILE = 'GEOJSON_FILE'; export const DECIMAL_DEGREES_PRECISION = 5; // meters precision export const ZOOM_PRECISION = 2; -export const ES_SIZE_LIMIT = 1000; +export const ES_SIZE_LIMIT = 10000; export const FEATURE_ID_PROPERTY_NAME = '__kbn__feature_id__'; export const FEATURE_VISIBLE_PROPERTY_NAME = '__kbn__isvisible__'; From d1d7b4daa46255d5d64c061d9e708f03147fabf3 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Wed, 16 Oct 2019 14:07:54 -0600 Subject: [PATCH 3/3] remove console statements --- .../layers/sources/es_geo_grid_source/es_geo_grid_source.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js index 1a3b6404bcde9b..8416ef5709e309 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js @@ -186,7 +186,6 @@ export class ESGeoGridSource extends AbstractESSource { async getGeoJsonWithMeta(layerName, searchFilters, registerCancelCallback) { const indexPattern = await this._getIndexPattern(); const searchSource = await this._makeSearchSource(searchFilters, 0); - console.log('precision', searchFilters.geogridPrecision); const aggConfigs = new AggConfigs(indexPattern, this._makeAggConfigs(searchFilters.geogridPrecision), aggSchemas.all); searchSource.setField('aggs', aggConfigs.toDsl()); const esResponse = await this._runEsQuery( @@ -203,8 +202,6 @@ export class ESGeoGridSource extends AbstractESSource { renderAs: this._descriptor.requestType, }); - console.log('grid feature collection', featureCollection); - return { data: featureCollection, meta: {