From 76f6dc3d13337b7e520fbda6bb931e4272f31b49 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Wed, 22 Nov 2023 12:37:52 -0700 Subject: [PATCH] [maps] ignore indices without geometry field in vector tile requests (#171472) Closes https://github.com/elastic/kibana/issues/170656 PR adds exists filter to ensure geo field exists ### Test instructions 1. In console, run: ``` PUT geo1 {} PUT geo1/_mapping { "properties": { "location": { "type": "geo_point" } } } PUT geo1/_doc/1 { "location": "25,25" } PUT geo2 {} PUT geo2/_doc/1 {} ``` 2. Create `geo*` data view 3. create new map 4. add documents layer from `geo*` data view. 5. Add heatmap layer from `geo*` data view. 6. Verify geo1 data is displayed and warning is not displayed fro geo2 shard failures --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../es_geo_grid_source/es_geo_grid_source.test.ts | 9 ++++++++- .../sources/es_geo_grid_source/es_geo_grid_source.tsx | 6 ++++++ .../sources/es_search_source/es_search_source.test.ts | 9 ++++++++- .../sources/es_search_source/es_search_source.tsx | 8 +++++++- .../test/functional/apps/maps/group4/mvt_geotile_grid.js | 2 +- x-pack/test/functional/apps/maps/group4/mvt_scaling.js | 2 +- 6 files changed, 31 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.test.ts b/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.test.ts index 7e23353460df6e..105b17dcd5a21a 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.test.ts +++ b/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.test.ts @@ -27,6 +27,13 @@ import { LICENSED_FEATURES } from '../../../licensed_features'; jest.mock('../../../kibana_services'); export class MockSearchSource { + getField(fieldName: string) { + if (fieldName === 'filter') { + return []; + } + + throw new Error(`Unsupported search source field: ${fieldName}`); + } setField = jest.fn(); setParent() {} getSearchRequestBody() { @@ -324,7 +331,7 @@ describe('ESGeoGridSource', () => { index: 'foo-*', renderAs: 'heatmap', requestBody: - "(fields:('0':('0':index,'1':(fields:())),'1':('0':size,'1':0),'2':('0':filter,'1':!()),'3':('0':query),'4':('0':index,'1':(fields:())),'5':('0':query,'1':(language:KQL,query:'')),'6':('0':aggs,'1':())))", + "(fields:('0':('0':index,'1':(fields:())),'1':('0':size,'1':0),'2':('0':filter,'1':!()),'3':('0':query),'4':('0':index,'1':(fields:())),'5':('0':query,'1':(language:KQL,query:'')),'6':('0':aggs,'1':()),'7':('0':filter,'1':!((meta:(),query:(exists:(field:bar)))))))", token: '1234', }); }); diff --git a/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.tsx b/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.tsx index 4acb400807ad03..703ab4f45b9629 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.tsx +++ b/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.tsx @@ -21,6 +21,7 @@ import { DataView } from '@kbn/data-plugin/common'; import { Adapters } from '@kbn/inspector-plugin/common/adapters'; import { ACTION_GLOBAL_APPLY_FILTER } from '@kbn/unified-search-plugin/public'; import { getTileUrlParams } from '@kbn/maps-vector-tile-utils'; +import { type Filter, buildExistsFilter } from '@kbn/es-query'; import { makeESBbox } from '../../../../common/elasticsearch_util'; import { convertCompositeRespToGeoJson, convertRegularRespToGeoJson } from './convert_to_geojson'; import { UpdateSourceEditor } from './update_source_editor'; @@ -553,6 +554,11 @@ export class ESGeoGridSource extends AbstractESAggSource implements IMvtVectorSo const dataView = await this.getIndexPattern(); const searchSource = await this.makeSearchSource(requestMeta, 0); searchSource.setField('aggs', this.getValueAggsDsl(dataView)); + // Filter out documents without geo fields for broad index-pattern support + searchSource.setField('filter', [ + ...(searchSource.getField('filter') as Filter[]), + buildExistsFilter({ name: this._descriptor.geoField, type: 'geo_point' }, dataView), + ]); const mvtUrlServicePath = getHttp().basePath.prepend( `${MVT_GETGRIDTILE_API_PATH}/{z}/{x}/{y}.pbf` diff --git a/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.test.ts b/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.test.ts index 7f7fde393de91d..3585150459b890 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.test.ts +++ b/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.test.ts @@ -53,6 +53,13 @@ describe('ESSearchSource', () => { beforeEach(async () => { const mockSearchSource = { + getField: (fieldName: string) => { + if (fieldName === 'filter') { + return []; + } + + throw new Error(`Unsupported search source field: ${fieldName}`); + }, setField: jest.fn(), getSearchRequestBody() { return { @@ -131,7 +138,7 @@ describe('ESSearchSource', () => { hasLabels: 'false', index: 'foobar-title-*', requestBody: - "(fields:('0':('0':index,'1':(fields:(),title:'foobar-title-*')),'1':('0':size,'1':1000),'2':('0':filter,'1':!()),'3':('0':query),'4':('0':index,'1':(fields:(),title:'foobar-title-*')),'5':('0':query,'1':(language:KQL,query:'tooltipField: foobar')),'6':('0':fieldsFromSource,'1':!(_id)),'7':('0':source,'1':!f),'8':('0':fields,'1':!(tooltipField,styleField))))", + "(fields:('0':('0':index,'1':(fields:(),title:'foobar-title-*')),'1':('0':size,'1':1000),'2':('0':filter,'1':!()),'3':('0':query),'4':('0':index,'1':(fields:(),title:'foobar-title-*')),'5':('0':query,'1':(language:KQL,query:'tooltipField: foobar')),'6':('0':fieldsFromSource,'1':!(_id)),'7':('0':source,'1':!f),'8':('0':fields,'1':!(tooltipField,styleField)),'9':('0':filter,'1':!((meta:(),query:(exists:(field:bar)))))))", token: '1234', }); }); diff --git a/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.tsx b/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.tsx index 77f4b684caf1f5..5d0f6aa59c55d2 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.tsx +++ b/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.tsx @@ -12,7 +12,7 @@ import { i18n } from '@kbn/i18n'; import type { SearchResponseWarning } from '@kbn/search-response-warnings'; import { GeoJsonProperties, Geometry, Position } from 'geojson'; import type { KibanaExecutionContext } from '@kbn/core/public'; -import { type Filter, buildPhraseFilter, type TimeRange } from '@kbn/es-query'; +import { type Filter, buildExistsFilter, buildPhraseFilter, type TimeRange } from '@kbn/es-query'; import type { DataViewField, DataView } from '@kbn/data-plugin/common'; import { lastValueFrom } from 'rxjs'; import { Adapters } from '@kbn/inspector-plugin/common/adapters'; @@ -923,6 +923,12 @@ export class ESSearchSource extends AbstractESSource implements IMvtVectorSource }) ); + // Filter out documents without geo fields to avoid shard failures for indices without geo fields + searchSource.setField('filter', [ + ...(searchSource.getField('filter') as Filter[]), + buildExistsFilter({ name: this._descriptor.geoField, type: 'geo_point' }, dataView), + ]); + const mvtUrlServicePath = getHttp().basePath.prepend(`${MVT_GETTILE_API_PATH}/{z}/{x}/{y}.pbf`); const tileUrlParams = getTileUrlParams({ diff --git a/x-pack/test/functional/apps/maps/group4/mvt_geotile_grid.js b/x-pack/test/functional/apps/maps/group4/mvt_geotile_grid.js index 081a1a41a93df8..55e504e207d4ad 100644 --- a/x-pack/test/functional/apps/maps/group4/mvt_geotile_grid.js +++ b/x-pack/test/functional/apps/maps/group4/mvt_geotile_grid.js @@ -53,7 +53,7 @@ export default function ({ getPageObjects, getService }) { index: 'logstash-*', gridPrecision: '8', renderAs: 'grid', - requestBody: `(aggs:(max_of_bytes:(max:(field:bytes))),fields:!((field:'@timestamp',format:date_time),(field:'relatedContent.article:modified_time',format:date_time),(field:'relatedContent.article:published_time',format:date_time),(field:utc_time,format:date_time)),query:(bool:(filter:!((range:('@timestamp':(format:strict_date_optional_time,gte:'2015-09-20T00:00:00.000Z',lte:'2015-09-20T01:00:00.000Z')))),must:!(),must_not:!(),should:!())),runtime_mappings:())`, + requestBody: `(aggs:(max_of_bytes:(max:(field:bytes))),fields:!((field:'@timestamp',format:date_time),(field:'relatedContent.article:modified_time',format:date_time),(field:'relatedContent.article:published_time',format:date_time),(field:utc_time,format:date_time)),query:(bool:(filter:!((range:('@timestamp':(format:strict_date_optional_time,gte:'2015-09-20T00:00:00.000Z',lte:'2015-09-20T01:00:00.000Z'))),(exists:(field:geo.coordinates))),must:!(),must_not:!(),should:!())),runtime_mappings:())`, }); //Should correctly load meta for style-rule (sigma is set to 1, opacity to 1) diff --git a/x-pack/test/functional/apps/maps/group4/mvt_scaling.js b/x-pack/test/functional/apps/maps/group4/mvt_scaling.js index cd779392ba821e..e624ab39dee122 100644 --- a/x-pack/test/functional/apps/maps/group4/mvt_scaling.js +++ b/x-pack/test/functional/apps/maps/group4/mvt_scaling.js @@ -55,7 +55,7 @@ export default function ({ getPageObjects, getService }) { hasLabels: 'false', index: 'geo_shapes*', requestBody: - '(fields:!(prop1),query:(bool:(filter:!(),must:!(),must_not:!(),should:!())),runtime_mappings:(),size:10001)', + '(fields:!(prop1),query:(bool:(filter:!((exists:(field:geometry))),must:!(),must_not:!(),should:!())),runtime_mappings:(),size:10001)', }); });