From 30b03a9b95e76452f17d4c6693b1525d50ef3f91 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 18 Feb 2020 05:48:59 -0700 Subject: [PATCH 01/29] [Maps] add Top term aggregation --- .../legacy/plugins/maps/common/constants.ts | 5 +- .../maps/public/components/metric_editor.js | 13 ++-- .../maps/public/components/metric_select.js | 20 +++-- .../maps/public/components/metrics_editor.js | 4 +- .../resources/metrics_expression.js | 8 +- .../plugins/maps/public/kibana_services.js | 2 + .../maps/public/layers/fields/es_agg_field.js | 26 +++++-- .../public/layers/fields/es_agg_field.test.js | 14 ++-- .../public/layers/sources/es_agg_source.js | 35 ++++++--- .../es_geo_grid_source/convert_to_geojson.js | 61 +++++---------- .../convert_to_geojson.test.js | 78 +++++++++++++++++++ .../es_geo_grid_source/es_geo_grid_source.js | 63 +++++---------- .../maps/public/layers/sources/es_source.js | 4 +- .../public/layers/sources/es_term_source.js | 10 +-- .../tooltips/es_aggmetric_tooltip_property.js | 6 +- .../public/layers/util/is_metric_countable.js | 4 +- 16 files changed, 207 insertions(+), 146 deletions(-) create mode 100644 x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.test.js diff --git a/x-pack/legacy/plugins/maps/common/constants.ts b/x-pack/legacy/plugins/maps/common/constants.ts index ab9a696fa3a177..f8c87212aeea50 100644 --- a/x-pack/legacy/plugins/maps/common/constants.ts +++ b/x-pack/legacy/plugins/maps/common/constants.ts @@ -117,16 +117,17 @@ export const DRAW_TYPE = { POLYGON: 'POLYGON', }; -export const METRIC_TYPE = { +export const AGG_TYPE = { AVG: 'avg', COUNT: 'count', MAX: 'max', MIN: 'min', SUM: 'sum', + TERMS: 'terms', UNIQUE_COUNT: 'cardinality', }; -export const COUNT_AGG_TYPE = METRIC_TYPE.COUNT; +export const COUNT_AGG_TYPE = AGG_TYPE.COUNT; export const COUNT_PROP_LABEL = i18n.translate('xpack.maps.aggs.defaultCountLabel', { defaultMessage: 'count', }); diff --git a/x-pack/legacy/plugins/maps/public/components/metric_editor.js b/x-pack/legacy/plugins/maps/public/components/metric_editor.js index e60c2ac0dd7ab5..530f402592b2ba 100644 --- a/x-pack/legacy/plugins/maps/public/components/metric_editor.js +++ b/x-pack/legacy/plugins/maps/public/components/metric_editor.js @@ -12,17 +12,16 @@ import { EuiFieldText, EuiFormRow } from '@elastic/eui'; import { MetricSelect, METRIC_AGGREGATION_VALUES } from './metric_select'; import { SingleFieldSelect } from './single_field_select'; -import { METRIC_TYPE } from '../../common/constants'; +import { AGG_TYPE } from '../../common/constants'; +import { getTermsFields } from '../index_pattern_util'; function filterFieldsForAgg(fields, aggType) { if (!fields) { return []; } - if (aggType === METRIC_TYPE.UNIQUE_COUNT) { - return fields.filter(field => { - return field.aggregatable; - }); + if (aggType === AGG_TYPE.UNIQUE_COUNT || aggType === AGG_TYPE.TERMS) { + return getTermsFields(fields); } return fields.filter(field => { @@ -38,7 +37,7 @@ export function MetricEditor({ fields, metricsFilter, metric, onChange, removeBu }; // unset field when new agg type does not support currently selected field. - if (metric.field && metricAggregationType !== METRIC_TYPE.COUNT) { + if (metric.field && metricAggregationType !== AGG_TYPE.COUNT) { const fieldsForNewAggType = filterFieldsForAgg(fields, metricAggregationType); const found = fieldsForNewAggType.find(field => { return field.name === metric.field; @@ -64,7 +63,7 @@ export function MetricEditor({ fields, metricsFilter, metric, onChange, removeBu }; let fieldSelect; - if (metric.type && metric.type !== METRIC_TYPE.COUNT) { + if (metric.type && metric.type !== AGG_TYPE.COUNT) { fieldSelect = ( { - if (type === METRIC_TYPE.COUNT) { + if (type === AGG_TYPE.COUNT) { return true; } @@ -70,7 +70,7 @@ export class MetricsExpression extends Component { }) .map(({ type, field }) => { // do not use metric label so field and aggregation are not obscured. - if (type === METRIC_TYPE.COUNT) { + if (type === AGG_TYPE.COUNT) { return 'count'; } @@ -130,5 +130,5 @@ MetricsExpression.propTypes = { }; MetricsExpression.defaultProps = { - metrics: [{ type: METRIC_TYPE.COUNT }], + metrics: [{ type: AGG_TYPE.COUNT }], }; diff --git a/x-pack/legacy/plugins/maps/public/kibana_services.js b/x-pack/legacy/plugins/maps/public/kibana_services.js index a1b1c9ec1518e6..05d333bda336d3 100644 --- a/x-pack/legacy/plugins/maps/public/kibana_services.js +++ b/x-pack/legacy/plugins/maps/public/kibana_services.js @@ -13,6 +13,8 @@ import { npStart } from 'ui/new_platform'; export const SPATIAL_FILTER_TYPE = esFilters.FILTERS.SPATIAL_FILTER; export { SearchSource } from '../../../../../src/plugins/data/public'; +//export { KBN_FIELD_TYPES } from '../../../../../src/plugins/data/common'; +export const KBN_FIELD_TYPES = {}; export const indexPatternService = npStart.plugins.data.indexPatterns; export const autocompleteService = npStart.plugins.data.autocomplete; diff --git a/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.js b/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.js index 65109cb99809fc..8cf947a296d33f 100644 --- a/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.js +++ b/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.js @@ -5,7 +5,7 @@ */ import { AbstractField } from './field'; -import { COUNT_AGG_TYPE } from '../../../common/constants'; +import { AGG_TYPE } from '../../../common/constants'; import { isMetricCountable } from '../util/is_metric_countable'; import { ESAggMetricTooltipProperty } from '../tooltips/es_aggmetric_tooltip_property'; @@ -34,12 +34,11 @@ export class ESAggMetricField extends AbstractField { } isValid() { - return this.getAggType() === COUNT_AGG_TYPE ? true : !!this._esDocField; + return this.getAggType() === AGG_TYPE.COUNT ? true : !!this._esDocField; } async getDataType() { - // aggregations only provide numerical data - return 'number'; + return this.getAggType() === AGG_TYPE.TERMS ? 'string' : 'number'; } getESDocFieldName() { @@ -47,9 +46,9 @@ export class ESAggMetricField extends AbstractField { } getRequestDescription() { - return this.getAggType() !== COUNT_AGG_TYPE + return this.getAggType() !== AGG_TYPE.COUNT ? `${this.getAggType()} ${this.getESDocFieldName()}` - : COUNT_AGG_TYPE; + : AGG_TYPE.COUNT; } async createTooltipProperty(value) { @@ -71,12 +70,25 @@ export class ESAggMetricField extends AbstractField { schema: 'metric', params: {}, }; - if (this.getAggType() !== COUNT_AGG_TYPE) { + if (this.getAggType() !== AGG_TYPE.COUNT) { metricAggConfig.params = { field: this.getESDocFieldName() }; } return metricAggConfig; } + getValueAggDsl() { + const aggType = this.getAggType(); + const aggBody = { + field: this.getESDocFieldName(), + }; + if (aggType === AGG_TYPE.TERMS) { + aggBody.size = 1; + } + return { + [aggType]: aggBody, + }; + } + supportsFieldMeta() { // count and sum aggregations are not within field bounds so they do not support field meta. return !isMetricCountable(this.getAggType()); diff --git a/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.test.js b/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.test.js index 2f18987513d925..aeeffd63607eeb 100644 --- a/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.test.js +++ b/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.test.js @@ -5,24 +5,24 @@ */ import { ESAggMetricField } from './es_agg_field'; -import { METRIC_TYPE } from '../../../common/constants'; +import { AGG_TYPE } from '../../../common/constants'; describe('supportsFieldMeta', () => { test('Non-counting aggregations should support field meta', () => { - const avgMetric = new ESAggMetricField({ aggType: METRIC_TYPE.AVG }); + const avgMetric = new ESAggMetricField({ aggType: AGG_TYPE.AVG }); expect(avgMetric.supportsFieldMeta()).toBe(true); - const maxMetric = new ESAggMetricField({ aggType: METRIC_TYPE.MAX }); + const maxMetric = new ESAggMetricField({ aggType: AGG_TYPE.MAX }); expect(maxMetric.supportsFieldMeta()).toBe(true); - const minMetric = new ESAggMetricField({ aggType: METRIC_TYPE.MIN }); + const minMetric = new ESAggMetricField({ aggType: AGG_TYPE.MIN }); expect(minMetric.supportsFieldMeta()).toBe(true); }); test('Counting aggregations should not support field meta', () => { - const countMetric = new ESAggMetricField({ aggType: METRIC_TYPE.COUNT }); + const countMetric = new ESAggMetricField({ aggType: AGG_TYPE.COUNT }); expect(countMetric.supportsFieldMeta()).toBe(false); - const sumMetric = new ESAggMetricField({ aggType: METRIC_TYPE.SUM }); + const sumMetric = new ESAggMetricField({ aggType: AGG_TYPE.SUM }); expect(sumMetric.supportsFieldMeta()).toBe(false); - const uniqueCountMetric = new ESAggMetricField({ aggType: METRIC_TYPE.UNIQUE_COUNT }); + const uniqueCountMetric = new ESAggMetricField({ aggType: AGG_TYPE.UNIQUE_COUNT }); expect(uniqueCountMetric.supportsFieldMeta()).toBe(false); }); }); diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_agg_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_agg_source.js index 967a3c41aec269..492e3ef41eaee3 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_agg_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_agg_source.js @@ -8,8 +8,7 @@ import { AbstractESSource } from './es_source'; import { ESAggMetricField } from '../fields/es_agg_field'; import { ESDocField } from '../fields/es_doc_field'; import { - METRIC_TYPE, - COUNT_AGG_TYPE, + AGG_TYPE, COUNT_PROP_LABEL, COUNT_PROP_NAME, FIELD_ORIGIN, @@ -25,14 +24,14 @@ export class AbstractESAggSource extends AbstractESSource { min: 1, max: Infinity, aggFilter: [ - METRIC_TYPE.AVG, - METRIC_TYPE.COUNT, - METRIC_TYPE.MAX, - METRIC_TYPE.MIN, - METRIC_TYPE.SUM, - METRIC_TYPE.UNIQUE_COUNT, + AGG_TYPE.AVG, + AGG_TYPE.COUNT, + AGG_TYPE.MAX, + AGG_TYPE.MIN, + AGG_TYPE.SUM, + AGG_TYPE.UNIQUE_COUNT, ], - defaults: [{ schema: 'metric', type: METRIC_TYPE.COUNT }], + defaults: [{ schema: 'metric', type: AGG_TYPE.COUNT }], }; constructor(descriptor, inspectorAdapters) { @@ -81,7 +80,7 @@ export class AbstractESAggSource extends AbstractESSource { if (metrics.length === 0) { metrics.push( new ESAggMetricField({ - aggType: COUNT_AGG_TYPE, + aggType: AGG_TYPE.COUNT, source: this, origin: this.getOriginForField(), }) @@ -91,17 +90,29 @@ export class AbstractESAggSource extends AbstractESSource { } formatMetricKey(aggType, fieldName) { - return aggType !== COUNT_AGG_TYPE ? `${aggType}${AGG_DELIMITER}${fieldName}` : COUNT_PROP_NAME; + return aggType !== AGG_TYPE.COUNT ? `${aggType}${AGG_DELIMITER}${fieldName}` : COUNT_PROP_NAME; } formatMetricLabel(aggType, fieldName) { - return aggType !== COUNT_AGG_TYPE ? `${aggType} of ${fieldName}` : COUNT_PROP_LABEL; + return aggType !== AGG_TYPE.COUNT ? `${aggType} of ${fieldName}` : COUNT_PROP_LABEL; } createMetricAggConfigs() { return this.getMetricFields().map(esAggMetric => esAggMetric.makeMetricAggConfig()); } + getValueAggsDsl() { + const valueAggsDsl = {}; + this.getMetricFields() + .filter(esAggMetric => { + return esAggMetric.getAggType() !== AGG_TYPE.COUNT; + }) + .forEach(esAggMetric => { + valueAggsDsl[esAggMetric.getName()] = esAggMetric.getValueAggDsl(); + }); + return valueAggsDsl; + } + async getNumberFields() { return this.getMetricFields(); } diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js index 4e15d1c927c364..aaa7f426ab5af7 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js @@ -4,58 +4,37 @@ * you may not use this file except in compliance with the Elastic License. */ +import _ from 'lodash'; import { RENDER_AS } from './render_as'; import { getTileBoundingBox } from './geo_tile_utils'; -import { EMPTY_FEATURE_COLLECTION } from '../../../../common/constants'; - -export function convertToGeoJson({ table, renderAs }) { - if (!table || !table.rows) { - return EMPTY_FEATURE_COLLECTION; - } - - const geoGridColumn = table.columns.find( - column => column.aggConfig.type.dslName === 'geotile_grid' - ); - if (!geoGridColumn) { - return EMPTY_FEATURE_COLLECTION; - } - - const metricColumns = table.columns.filter(column => { - return ( - column.aggConfig.type.type === 'metrics' && column.aggConfig.type.dslName !== 'geo_centroid' - ); - }); - const geocentroidColumn = table.columns.find( - column => column.aggConfig.type.dslName === 'geo_centroid' - ); - if (!geocentroidColumn) { - return EMPTY_FEATURE_COLLECTION; - } +export function convertToGeoJson(esResponse, renderAs) { const features = []; - table.rows.forEach(row => { - const gridKey = row[geoGridColumn.id]; - if (!gridKey) { - return; - } - const properties = {}; - metricColumns.forEach(metricColumn => { - properties[metricColumn.aggConfig.id] = row[metricColumn.id]; + const gridBuckets = _.get(esResponse, 'aggregations.gridSplit.buckets', []); + for (let i = 0; i < gridBuckets.length; i++) { + const gridBucket = gridBuckets[i]; + const { key, gridCentroid, doc_count, ...rest } = gridBucket; // eslint-disable-line camelcase + const properties = { doc_count }; // eslint-disable-line camelcase + Object.keys(rest).forEach(key => { + if (_.has(rest[key], 'value')) { + properties[key] = rest[key].value; + } else if (_.has(rest[key], 'buckets')) { + properties[key] = _.get(rest[key], 'buckets[0].key'); + } }); features.push({ type: 'Feature', geometry: rowToGeometry({ - row, - gridKey, - geocentroidColumn, + gridKey: key, + gridCentroid, renderAs, }), - id: gridKey, + id: key, properties, }); - }); + } return { featureCollection: { @@ -65,7 +44,7 @@ export function convertToGeoJson({ table, renderAs }) { }; } -function rowToGeometry({ row, gridKey, geocentroidColumn, renderAs }) { +function rowToGeometry({ gridKey, gridCentroid, renderAs }) { const { top, bottom, right, left } = getTileBoundingBox(gridKey); if (renderAs === RENDER_AS.GRID) { @@ -85,8 +64,8 @@ function rowToGeometry({ row, gridKey, geocentroidColumn, renderAs }) { // see https://github.com/elastic/elasticsearch/issues/24694 for why clampGrid is used const pointCoordinates = [ - clampGrid(row[geocentroidColumn.id].lon, left, right), - clampGrid(row[geocentroidColumn.id].lat, bottom, top), + clampGrid(gridCentroid.location.lon, left, right), + clampGrid(gridCentroid.location.lat, bottom, top), ]; return { diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.test.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.test.js new file mode 100644 index 00000000000000..77fb1ae349b077 --- /dev/null +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.test.js @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { convertToGeoJson } from './convert_to_geojson'; +import { RENDER_AS } from './render_as'; + +const esResponse = { + aggregations: { + gridSplit: { + buckets: [ + { + key: '4/4/6', + doc_count: 65, + avg_of_bytes: { value: 5359.2307692307695 }, + 'terms_of_machine.os.keyword': { + buckets: [ + { + key: 'win xp', + doc_count: 16, + }, + ], + }, + gridCentroid: { + location: { lat: 36.62813963153614, lon: -81.94552666092149 }, + count: 65, + }, + }, + ], + }, + }, +}; + +it('Should convert elasticsearch aggregation response into feature collection of points', () => { + const geoJson = convertToGeoJson(esResponse, RENDER_AS.POINT); + expect(geoJson.featureCollection.features.length).toBe(1); + expect(geoJson.featureCollection.features[0]).toEqual({ + geometry: { + coordinates: [-81.94552666092149, 36.62813963153614], + type: 'Point', + }, + id: '4/4/6', + properties: { + avg_of_bytes: 5359.2307692307695, + doc_count: 65, + 'terms_of_machine.os.keyword': 'win xp', + }, + type: 'Feature', + }); +}); + +it('Should convert elasticsearch aggregation response into feature collection of Polygons', () => { + const geoJson = convertToGeoJson(esResponse, RENDER_AS.GRID); + expect(geoJson.featureCollection.features.length).toBe(1); + expect(geoJson.featureCollection.features[0]).toEqual({ + geometry: { + coordinates: [ + [ + [-67.5, 40.9799], + [-90, 40.9799], + [-90, 21.94305], + [-67.5, 21.94305], + [-67.5, 40.9799], + ], + ], + type: 'Polygon', + }, + id: '4/4/6', + properties: { + avg_of_bytes: 5359.2307692307695, + doc_count: 65, + 'terms_of_machine.os.keyword': 'win xp', + }, + type: 'Feature', + }); +}); diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js index cb8b43a6c312b2..bcc96b8dffffb7 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js @@ -10,8 +10,6 @@ import uuid from 'uuid/v4'; import { VECTOR_SHAPE_TYPES } from '../vector_feature_types'; import { HeatmapLayer } from '../../heatmap_layer'; import { VectorLayer } from '../../vector_layer'; -import { AggConfigs, Schemas } from 'ui/agg_types'; -import { tabifyAggResponse } from 'ui/agg_response/tabify'; import { convertToGeoJson } from './convert_to_geojson'; import { VectorStyle } from '../../styles/vector/vector_style'; import { @@ -37,18 +35,6 @@ import { StaticStyleProperty } from '../../styles/vector/properties/static_style const MAX_GEOTILE_LEVEL = 29; -const aggSchemas = new Schemas([ - AbstractESAggSource.METRIC_SCHEMA_CONFIG, - { - group: 'buckets', - name: 'segment', - title: 'Geo Grid', - aggFilter: 'geotile_grid', - min: 1, - max: 1, - }, -]); - export class ESGeoGridSource extends AbstractESAggSource { static type = ES_GEO_GRID; static title = i18n.translate('xpack.maps.source.esGridTitle', { @@ -176,14 +162,24 @@ export class ESGeoGridSource extends AbstractESAggSource { } async getGeoJsonWithMeta(layerName, searchFilters, registerCancelCallback) { - const indexPattern = await this.getIndexPattern(); const searchSource = await this._makeSearchSource(searchFilters, 0); - const aggConfigs = new AggConfigs( - indexPattern, - this._makeAggConfigs(searchFilters.geogridPrecision), - aggSchemas.all - ); - searchSource.setField('aggs', aggConfigs.toDsl()); + searchSource.setField('aggs', { + gridSplit: { + geotile_grid: { + field: this._descriptor.geoField, + precision: searchFilters.geogridPrecision, + }, + aggs: { + gridCentroid: { + geo_centroid: { + field: this._descriptor.geoField, + }, + }, + ...this.getValueAggsDsl(), + }, + }, + }); + const esResponse = await this._runEsQuery({ requestId: this.getId(), requestName: layerName, @@ -194,12 +190,7 @@ export class ESGeoGridSource extends AbstractESAggSource { }), }); - const tabifiedResp = tabifyAggResponse(aggConfigs, esResponse); - const { featureCollection } = convertToGeoJson({ - table: tabifiedResp, - renderAs: this._descriptor.requestType, - }); - + const { featureCollection } = convertToGeoJson(esResponse, this._descriptor.requestType); return { data: featureCollection, meta: { @@ -212,24 +203,6 @@ export class ESGeoGridSource extends AbstractESAggSource { return true; } - _makeAggConfigs(precision) { - const metricAggConfigs = this.createMetricAggConfigs(); - return [ - ...metricAggConfigs, - { - id: 'grid', - enabled: true, - type: 'geotile_grid', - schema: 'segment', - params: { - field: this._descriptor.geoField, - useGeocentroid: true, - precision: precision, - }, - }, - ]; - } - _createHeatmapLayerDescriptor(options) { return HeatmapLayer.createDescriptor({ sourceDescriptor: this._descriptor, diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_source.js index d78d3038f870d9..f3cdc8cb290b2a 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_source.js @@ -18,7 +18,7 @@ import { AggConfigs } from 'ui/agg_types'; import { i18n } from '@kbn/i18n'; import uuid from 'uuid/v4'; import { copyPersistentState } from '../../reducers/util'; -import { ES_GEO_FIELD_TYPE, METRIC_TYPE } from '../../../common/constants'; +import { ES_GEO_FIELD_TYPE, AGG_TYPE } from '../../../common/constants'; import { DataRequestAbortError } from '../util/data_request'; import { expandToTileBoundaries } from './es_geo_grid_source/geo_tile_utils'; @@ -270,7 +270,7 @@ export class AbstractESSource extends AbstractVectorSource { // Do not use field formatters for counting metrics if ( metricField && - (metricField.type === METRIC_TYPE.COUNT || metricField.type === METRIC_TYPE.UNIQUE_COUNT) + (metricField.type === AGG_TYPE.COUNT || metricField.type === AGG_TYPE.UNIQUE_COUNT) ) { return null; } diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.js index 7d7a2e159d1283..6bcc6dc01bbbf5 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.js @@ -12,7 +12,7 @@ import { COUNT_PROP_LABEL, DEFAULT_MAX_BUCKETS_LIMIT, FIELD_ORIGIN, - METRIC_TYPE, + AGG_TYPE, } from '../../../common/constants'; import { ESDocField } from '../fields/es_doc_field'; import { AbstractESAggSource, AGG_DELIMITER } from './es_agg_source'; @@ -90,14 +90,14 @@ export class ESTermSource extends AbstractESAggSource { formatMetricKey(aggType, fieldName) { const metricKey = - aggType !== METRIC_TYPE.COUNT ? `${aggType}${AGG_DELIMITER}${fieldName}` : aggType; + aggType !== AGG_TYPE.COUNT ? `${aggType}${AGG_DELIMITER}${fieldName}` : aggType; return `${FIELD_NAME_PREFIX}${metricKey}${GROUP_BY_DELIMITER}${ this._descriptor.indexPatternTitle }.${this._termField.getName()}`; } formatMetricLabel(type, fieldName) { - const metricLabel = type !== METRIC_TYPE.COUNT ? `${type} ${fieldName}` : COUNT_PROP_LABEL; + const metricLabel = type !== AGG_TYPE.COUNT ? `${type} ${fieldName}` : COUNT_PROP_LABEL; return `${metricLabel} of ${this._descriptor.indexPatternTitle}:${this._termField.getName()}`; } @@ -122,13 +122,13 @@ export class ESTermSource extends AbstractESAggSource { const metricPropertyNames = configStates .filter(configState => { - return configState.schema === 'metric' && configState.type !== METRIC_TYPE.COUNT; + return configState.schema === 'metric' && configState.type !== AGG_TYPE.COUNT; }) .map(configState => { return configState.id; }); const countConfigState = configStates.find(configState => { - return configState.type === METRIC_TYPE.COUNT; + return configState.type === AGG_TYPE.COUNT; }); const countPropertyName = _.get(countConfigState, 'id'); return { diff --git a/x-pack/legacy/plugins/maps/public/layers/tooltips/es_aggmetric_tooltip_property.js b/x-pack/legacy/plugins/maps/public/layers/tooltips/es_aggmetric_tooltip_property.js index 7cfb60910c1557..229c84fe234bd9 100644 --- a/x-pack/legacy/plugins/maps/public/layers/tooltips/es_aggmetric_tooltip_property.js +++ b/x-pack/legacy/plugins/maps/public/layers/tooltips/es_aggmetric_tooltip_property.js @@ -5,7 +5,7 @@ */ import { ESTooltipProperty } from './es_tooltip_property'; -import { METRIC_TYPE } from '../../../common/constants'; +import { AGG_TYPE } from '../../../common/constants'; export class ESAggMetricTooltipProperty extends ESTooltipProperty { constructor(propertyKey, propertyName, rawValue, indexPattern, metricField) { @@ -22,8 +22,8 @@ export class ESAggMetricTooltipProperty extends ESTooltipProperty { return '-'; } if ( - this._metricField.getAggType() === METRIC_TYPE.COUNT || - this._metricField.getAggType() === METRIC_TYPE.UNIQUE_COUNT + this._metricField.getAggType() === AGG_TYPE.COUNT || + this._metricField.getAggType() === AGG_TYPE.UNIQUE_COUNT ) { return this._rawValue; } diff --git a/x-pack/legacy/plugins/maps/public/layers/util/is_metric_countable.js b/x-pack/legacy/plugins/maps/public/layers/util/is_metric_countable.js index 54d8794b1e3cf0..69ccb8890d10cc 100644 --- a/x-pack/legacy/plugins/maps/public/layers/util/is_metric_countable.js +++ b/x-pack/legacy/plugins/maps/public/layers/util/is_metric_countable.js @@ -4,8 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ -import { METRIC_TYPE } from '../../../common/constants'; +import { AGG_TYPE } from '../../../common/constants'; export function isMetricCountable(aggType) { - return [METRIC_TYPE.COUNT, METRIC_TYPE.SUM, METRIC_TYPE.UNIQUE_COUNT].includes(aggType); + return [AGG_TYPE.COUNT, AGG_TYPE.SUM, AGG_TYPE.UNIQUE_COUNT].includes(aggType); } From 127b381c643f329d2d962e946592944412aef060 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 18 Feb 2020 06:33:50 -0700 Subject: [PATCH 02/29] update pew-pew source to handle terms agg --- .../maps/public/layers/fields/es_agg_field.js | 13 ++-- .../public/layers/sources/es_agg_source.js | 4 +- .../es_geo_grid_source/es_geo_grid_source.js | 3 +- .../es_pew_pew_source/convert_to_lines.js | 2 + .../convert_to_lines.test.js | 67 +++++++++++++++++++ .../es_pew_pew_source/es_pew_pew_source.js | 8 +-- .../es_search_source/es_search_source.js | 26 +------ .../maps/public/layers/util/es_agg_utils.js | 31 +++++++++ 8 files changed, 111 insertions(+), 43 deletions(-) create mode 100644 x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.test.js create mode 100644 x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.js diff --git a/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.js b/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.js index 8cf947a296d33f..33e1afe0b3e865 100644 --- a/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.js +++ b/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.js @@ -8,6 +8,7 @@ import { AbstractField } from './field'; import { AGG_TYPE } from '../../../common/constants'; import { isMetricCountable } from '../util/is_metric_countable'; import { ESAggMetricTooltipProperty } from '../tooltips/es_aggmetric_tooltip_property'; +import { getField, addFieldToDSL } from '../util/es_agg_utils'; export class ESAggMetricField extends AbstractField { static type = 'ES_AGG'; @@ -76,16 +77,12 @@ export class ESAggMetricField extends AbstractField { return metricAggConfig; } - getValueAggDsl() { + getValueAggDsl(indexPattern) { + const field = getField(indexPattern, this.getESDocFieldName()); const aggType = this.getAggType(); - const aggBody = { - field: this.getESDocFieldName(), - }; - if (aggType === AGG_TYPE.TERMS) { - aggBody.size = 1; - } + const aggBody = aggType === AGG_TYPE.TERMS ? { size: 1 } : {}; return { - [aggType]: aggBody, + [aggType]: addFieldToDSL(aggBody, field), }; } diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_agg_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_agg_source.js index 492e3ef41eaee3..fb27cb22bcad72 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_agg_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_agg_source.js @@ -101,14 +101,14 @@ export class AbstractESAggSource extends AbstractESSource { return this.getMetricFields().map(esAggMetric => esAggMetric.makeMetricAggConfig()); } - getValueAggsDsl() { + getValueAggsDsl(indexPattern) { const valueAggsDsl = {}; this.getMetricFields() .filter(esAggMetric => { return esAggMetric.getAggType() !== AGG_TYPE.COUNT; }) .forEach(esAggMetric => { - valueAggsDsl[esAggMetric.getName()] = esAggMetric.getValueAggDsl(); + valueAggsDsl[esAggMetric.getName()] = esAggMetric.getValueAggDsl(indexPattern); }); return valueAggsDsl; } diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js index bcc96b8dffffb7..ca008a820d4637 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js @@ -162,6 +162,7 @@ export class ESGeoGridSource extends AbstractESAggSource { } async getGeoJsonWithMeta(layerName, searchFilters, registerCancelCallback) { + const indexPattern = await this.getIndexPattern(); const searchSource = await this._makeSearchSource(searchFilters, 0); searchSource.setField('aggs', { gridSplit: { @@ -175,7 +176,7 @@ export class ESGeoGridSource extends AbstractESAggSource { field: this._descriptor.geoField, }, }, - ...this.getValueAggsDsl(), + ...this.getValueAggsDsl(indexPattern), }, }, }); diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.js index 2057949c30c882..abf2960983a3c6 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.js @@ -31,6 +31,8 @@ export function convertToLines(esResponse) { Object.keys(rest).forEach(key => { if (_.has(rest[key], 'value')) { rest[key] = rest[key].value; + } else if (_.has(rest[key], 'buckets')) { + rest[key] = _.get(rest[key], 'buckets[0].key'); } }); diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.test.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.test.js new file mode 100644 index 00000000000000..54f7c9d0bd84df --- /dev/null +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.test.js @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { convertToLines } from './convert_to_lines'; + +const esResponse = { + aggregations: { + destSplit: { + buckets: [ + { + key: '43.68389896117151, 10.39269994944334', + doc_count: 2, + sourceGrid: { + buckets: [ + { + key: '4/9/3', + doc_count: 1, + terms_of_Carrier: { + buckets: [ + { + key: 'ES-Air', + doc_count: 1, + }, + ], + }, + sourceCentroid: { + location: { + lat: 68.15180202014744, + lon: 33.46390150487423, + }, + count: 1, + }, + avg_of_FlightDelayMin: { + value: 3, + }, + }, + ], + }, + }, + ], + }, + }, +}; + +it('Should convert elasticsearch aggregation response into feature collection of lines', () => { + const geoJson = convertToLines(esResponse); + expect(geoJson.featureCollection.features.length).toBe(1); + expect(geoJson.featureCollection.features[0]).toEqual({ + geometry: { + coordinates: [ + [33.46390150487423, 68.15180202014744], + [10.39269994944334, 43.68389896117151], + ], + type: 'LineString', + }, + id: '10.39269994944334,43.68389896117151,4/9/3', + properties: { + avg_of_FlightDelayMin: 3, + doc_count: 1, + terms_of_Carrier: 'ES-Air', + }, + type: 'Feature', + }); +}); diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/es_pew_pew_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/es_pew_pew_source.js index 176ab62baf98cd..53536b11aaca66 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/es_pew_pew_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/es_pew_pew_source.js @@ -20,7 +20,6 @@ import { i18n } from '@kbn/i18n'; import { SOURCE_DATA_ID_ORIGIN, ES_PEW_PEW, COUNT_PROP_NAME } from '../../../../common/constants'; import { getDataSourceLabel } from '../../../../common/i18n_getters'; import { convertToLines } from './convert_to_lines'; -import { AggConfigs, Schemas } from 'ui/agg_types'; import { AbstractESAggSource } from '../es_agg_source'; import { DynamicStyleProperty } from '../../styles/vector/properties/dynamic_style_property'; import { COLOR_GRADIENTS } from '../../styles/color_utils'; @@ -28,8 +27,6 @@ import { indexPatterns } from '../../../../../../../../src/plugins/data/public'; const MAX_GEOTILE_LEVEL = 29; -const aggSchemas = new Schemas([AbstractESAggSource.METRIC_SCHEMA_CONFIG]); - export class ESPewPewSource extends AbstractESAggSource { static type = ES_PEW_PEW; static title = i18n.translate('xpack.maps.source.pewPewTitle', { @@ -170,9 +167,6 @@ export class ESPewPewSource extends AbstractESAggSource { async getGeoJsonWithMeta(layerName, searchFilters, registerCancelCallback) { const indexPattern = await this.getIndexPattern(); - const metricAggConfigs = this.createMetricAggConfigs(); - const aggConfigs = new AggConfigs(indexPattern, metricAggConfigs, aggSchemas.all); - const searchSource = await this._makeSearchSource(searchFilters, 0); searchSource.setField('aggs', { destSplit: { @@ -199,7 +193,7 @@ export class ESPewPewSource extends AbstractESAggSource { field: this._descriptor.sourceGeoField, }, }, - ...aggConfigs.toDsl(), + ...this.getValueAggsDsl(indexPattern), }, }, }, diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js index 93ef40162a584e..524225be4b8d81 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js @@ -28,31 +28,7 @@ import { loadIndexSettings } from './load_index_settings'; import { DEFAULT_FILTER_BY_MAP_BOUNDS } from './constants'; import { ESDocField } from '../../fields/es_doc_field'; - -function getField(indexPattern, fieldName) { - const field = indexPattern.fields.getByName(fieldName); - if (!field) { - throw new Error( - i18n.translate('xpack.maps.source.esSearch.fieldNotFoundMsg', { - defaultMessage: `Unable to find '{fieldName}' in index-pattern '{indexPatternTitle}'.`, - values: { fieldName, indexPatternTitle: indexPattern.title }, - }) - ); - } - return field; -} - -function addFieldToDSL(dsl, field) { - return !field.scripted - ? { ...dsl, field: field.name } - : { - ...dsl, - script: { - source: field.script, - lang: field.lang, - }, - }; -} +import { getField, addFieldToDSL } from '../../util/es_agg_utils'; export class ESSearchSource extends AbstractESSource { static type = ES_SEARCH; diff --git a/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.js b/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.js new file mode 100644 index 00000000000000..094cf13bcc4c38 --- /dev/null +++ b/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.js @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { i18n } from '@kbn/i18n'; + +export function getField(indexPattern, fieldName) { + const field = indexPattern.fields.getByName(fieldName); + if (!field) { + throw new Error( + i18n.translate('xpack.maps.source.esSearch.fieldNotFoundMsg', { + defaultMessage: `Unable to find '{fieldName}' in index-pattern '{indexPatternTitle}'.`, + values: { fieldName, indexPatternTitle: indexPattern.title }, + }) + ); + } + return field; +} + +export function addFieldToDSL(dsl, field) { + return !field.scripted + ? { ...dsl, field: field.name } + : { + ...dsl, + script: { + source: field.script, + lang: field.lang, + }, + }; +} From 9e313339f4f0c1092f41aa8847b33528f93bcc69 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 18 Feb 2020 06:43:20 -0700 Subject: [PATCH 03/29] make helper function for pulling values from bucket --- .../es_geo_grid_source/convert_to_geojson.js | 19 +++++------------- .../es_pew_pew_source/convert_to_lines.js | 20 +++++-------------- .../maps/public/layers/util/es_agg_utils.js | 19 ++++++++++++++++++ 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js index aaa7f426ab5af7..b5930832647fb4 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js @@ -7,6 +7,7 @@ import _ from 'lodash'; import { RENDER_AS } from './render_as'; import { getTileBoundingBox } from './geo_tile_utils'; +import { extractPropertiesFromBucket } from '../../util/es_agg_utils'; export function convertToGeoJson(esResponse, renderAs) { const features = []; @@ -14,25 +15,15 @@ export function convertToGeoJson(esResponse, renderAs) { const gridBuckets = _.get(esResponse, 'aggregations.gridSplit.buckets', []); for (let i = 0; i < gridBuckets.length; i++) { const gridBucket = gridBuckets[i]; - const { key, gridCentroid, doc_count, ...rest } = gridBucket; // eslint-disable-line camelcase - const properties = { doc_count }; // eslint-disable-line camelcase - Object.keys(rest).forEach(key => { - if (_.has(rest[key], 'value')) { - properties[key] = rest[key].value; - } else if (_.has(rest[key], 'buckets')) { - properties[key] = _.get(rest[key], 'buckets[0].key'); - } - }); - features.push({ type: 'Feature', geometry: rowToGeometry({ - gridKey: key, - gridCentroid, + gridKey: gridBucket.key, + gridCentroid: gridBucket.gridCentroid, renderAs, }), - id: key, - properties, + id: gridBucket.key, + properties: extractPropertiesFromBucket(gridBucket, ['key', 'gridCentroid']), }); } diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.js index abf2960983a3c6..8395f7d2b34c3b 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.js @@ -5,6 +5,7 @@ */ import _ from 'lodash'; +import { extractPropertiesFromBucket } from '../../util/es_agg_utils'; const LAT_INDEX = 0; const LON_INDEX = 1; @@ -25,27 +26,16 @@ export function convertToLines(esResponse) { const dest = parsePointFromKey(destBucket.key); const sourceBuckets = _.get(destBucket, 'sourceGrid.buckets', []); for (let j = 0; j < sourceBuckets.length; j++) { - const { key, sourceCentroid, ...rest } = sourceBuckets[j]; - - // flatten metrics - Object.keys(rest).forEach(key => { - if (_.has(rest[key], 'value')) { - rest[key] = rest[key].value; - } else if (_.has(rest[key], 'buckets')) { - rest[key] = _.get(rest[key], 'buckets[0].key'); - } - }); - + const sourceBucket = sourceBuckets[j]; + const sourceCentroid = sourceBucket.sourceCentroid; lineFeatures.push({ type: 'Feature', geometry: { type: 'LineString', coordinates: [[sourceCentroid.location.lon, sourceCentroid.location.lat], dest], }, - id: `${dest.join()},${key}`, - properties: { - ...rest, - }, + id: `${dest.join()},${sourceBucket.key}`, + properties: extractPropertiesFromBucket(sourceBucket, ['key', 'sourceCentroid']), }); } } diff --git a/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.js b/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.js index 094cf13bcc4c38..95af8e7465103f 100644 --- a/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.js +++ b/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.js @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ import { i18n } from '@kbn/i18n'; +import _ from 'lodash'; export function getField(indexPattern, fieldName) { const field = indexPattern.fields.getByName(fieldName); @@ -29,3 +30,21 @@ export function addFieldToDSL(dsl, field) { }, }; } + +export function extractPropertiesFromBucket(bucket, ignoreKeys) { + const properties = {}; + Object.keys(bucket).forEach(key => { + if (ignoreKeys.includes(key)) { + return; + } + + if (_.has(bucket[key], 'value')) { + properties[key] = bucket[key].value; + } else if (_.has(bucket[key], 'buckets')) { + properties[key] = _.get(bucket[key], 'buckets[0].key'); + } else { + properties[key] = bucket[key]; + } + }); + return properties; +} From 402358e09dd4f9b19ead4279c7072fa744a56de3 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 18 Feb 2020 07:47:02 -0700 Subject: [PATCH 04/29] update terms source --- .../maps/public/layers/fields/es_agg_field.js | 14 ---- .../public/layers/sources/es_agg_source.js | 4 - .../public/layers/sources/es_term_source.js | 65 +++----------- .../layers/sources/es_term_source.test.js | 84 ++----------------- 4 files changed, 19 insertions(+), 148 deletions(-) diff --git a/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.js b/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.js index 33e1afe0b3e865..ef234ceaa8daf7 100644 --- a/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.js +++ b/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.js @@ -63,20 +63,6 @@ export class ESAggMetricField extends AbstractField { ); } - makeMetricAggConfig() { - const metricAggConfig = { - id: this.getName(), - enabled: true, - type: this.getAggType(), - schema: 'metric', - params: {}, - }; - if (this.getAggType() !== AGG_TYPE.COUNT) { - metricAggConfig.params = { field: this.getESDocFieldName() }; - } - return metricAggConfig; - } - getValueAggDsl(indexPattern) { const field = getField(indexPattern, this.getESDocFieldName()); const aggType = this.getAggType(); diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_agg_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_agg_source.js index fb27cb22bcad72..f24c3dda3bef1d 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_agg_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_agg_source.js @@ -97,10 +97,6 @@ export class AbstractESAggSource extends AbstractESSource { return aggType !== AGG_TYPE.COUNT ? `${aggType} of ${fieldName}` : COUNT_PROP_LABEL; } - createMetricAggConfigs() { - return this.getMetricFields().map(esAggMetric => esAggMetric.makeMetricAggConfig()); - } - getValueAggsDsl(indexPattern) { const valueAggsDsl = {}; this.getMetricFields() diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.js index 6bcc6dc01bbbf5..39d335c74819fe 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.js @@ -6,7 +6,6 @@ import _ from 'lodash'; -import { AggConfigs, Schemas } from 'ui/agg_types'; import { i18n } from '@kbn/i18n'; import { COUNT_PROP_LABEL, @@ -16,36 +15,20 @@ import { } from '../../../common/constants'; import { ESDocField } from '../fields/es_doc_field'; import { AbstractESAggSource, AGG_DELIMITER } from './es_agg_source'; +import { getField, addFieldToDSL, extractPropertiesFromBucket } from '../util/es_agg_utils'; const TERMS_AGG_NAME = 'join'; const FIELD_NAME_PREFIX = '__kbnjoin__'; const GROUP_BY_DELIMITER = '_groupby_'; -const aggSchemas = new Schemas([ - AbstractESAggSource.METRIC_SCHEMA_CONFIG, - { - group: 'buckets', - name: 'segment', - title: 'Terms', - aggFilter: 'terms', - min: 1, - max: 1, - }, -]); - -export function extractPropertiesMap(rawEsData, propertyNames, countPropertyName) { +export function extractPropertiesMap(rawEsData, countPropertyName) { const propertiesMap = new Map(); _.get(rawEsData, ['aggregations', TERMS_AGG_NAME, 'buckets'], []).forEach(termBucket => { - const properties = {}; + const properties = extractPropertiesFromBucket(termBucket, ['key', 'doc_count']); if (countPropertyName) { properties[countPropertyName] = termBucket.doc_count; } - propertyNames.forEach(propertyName => { - if (_.has(termBucket, [propertyName, 'value'])) { - properties[propertyName] = _.get(termBucket, [propertyName, 'value']); - } - }); propertiesMap.set(termBucket.key.toString(), properties); }); return propertiesMap; @@ -108,9 +91,14 @@ export class ESTermSource extends AbstractESAggSource { const indexPattern = await this.getIndexPattern(); const searchSource = await this._makeSearchSource(searchFilters, 0); - const configStates = this._makeAggConfigs(); - const aggConfigs = new AggConfigs(indexPattern, configStates, aggSchemas.all); - searchSource.setField('aggs', aggConfigs.toDsl()); + const termsField = getField(indexPattern, this._termField.getName()); + const termsAgg = { size: DEFAULT_MAX_BUCKETS_LIMIT }; + searchSource.setField('aggs', { + [TERMS_AGG_NAME]: { + terms: addFieldToDSL(termsAgg, termsField), + aggs: { ...this.getValueAggsDsl(indexPattern) }, + }, + }); const rawEsData = await this._runEsQuery({ requestId: this.getId(), @@ -120,19 +108,9 @@ export class ESTermSource extends AbstractESAggSource { requestDescription: this._getRequestDescription(leftSourceName, leftFieldName), }); - const metricPropertyNames = configStates - .filter(configState => { - return configState.schema === 'metric' && configState.type !== AGG_TYPE.COUNT; - }) - .map(configState => { - return configState.id; - }); - const countConfigState = configStates.find(configState => { - return configState.type === AGG_TYPE.COUNT; - }); - const countPropertyName = _.get(countConfigState, 'id'); + const countPropertyName = this.formatMetricKey(AGG_TYPE.COUNT); return { - propertiesMap: extractPropertiesMap(rawEsData, metricPropertyNames, countPropertyName), + propertiesMap: extractPropertiesMap(rawEsData, countPropertyName), }; } @@ -164,23 +142,6 @@ export class ESTermSource extends AbstractESAggSource { }); } - _makeAggConfigs() { - const metricAggConfigs = this.createMetricAggConfigs(); - return [ - ...metricAggConfigs, - { - id: TERMS_AGG_NAME, - enabled: true, - type: 'terms', - schema: 'segment', - params: { - field: this._termField.getName(), - size: DEFAULT_MAX_BUCKETS_LIMIT, - }, - }, - ]; - } - async getDisplayName() { //no need to localize. this is never rendered. return `es_table ${this._descriptor.indexPatternId}`; diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.test.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.test.js index ffaaf2d705b5c7..3247ce24d8a59d 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.test.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.test.js @@ -69,80 +69,9 @@ describe('getMetricFields', () => { }); }); -describe('_makeAggConfigs', () => { - describe('no metrics', () => { - let aggConfigs; - beforeAll(() => { - const source = new ESTermSource({ - indexPatternTitle: indexPatternTitle, - term: termFieldName, - }); - aggConfigs = source._makeAggConfigs(); - }); - - it('should make default "count" metric agg config', () => { - expect(aggConfigs.length).toBe(2); - expect(aggConfigs[0]).toEqual({ - id: '__kbnjoin__count_groupby_myIndex.myTermField', - enabled: true, - type: 'count', - schema: 'metric', - params: {}, - }); - }); - - it('should make "terms" buckets agg config', () => { - expect(aggConfigs.length).toBe(2); - expect(aggConfigs[1]).toEqual({ - id: 'join', - enabled: true, - type: 'terms', - schema: 'segment', - params: { - field: termFieldName, - size: 10000, - }, - }); - }); - }); - - describe('metrics', () => { - let aggConfigs; - beforeAll(() => { - const source = new ESTermSource({ - indexPatternTitle: indexPatternTitle, - term: 'myTermField', - metrics: metricExamples, - }); - aggConfigs = source._makeAggConfigs(); - }); - - it('should ignore invalid metrics configs', () => { - expect(aggConfigs.length).toBe(3); - }); - - it('should make agg config for each valid metric', () => { - expect(aggConfigs[0]).toEqual({ - id: '__kbnjoin__sum_of_myFieldGettingSummed_groupby_myIndex.myTermField', - enabled: true, - type: 'sum', - schema: 'metric', - params: { - field: sumFieldName, - }, - }); - expect(aggConfigs[1]).toEqual({ - id: '__kbnjoin__count_groupby_myIndex.myTermField', - enabled: true, - type: 'count', - schema: 'metric', - params: {}, - }); - }); - }); -}); - describe('extractPropertiesMap', () => { + const minPropName = + '__kbnjoin__min_of_avlAirTemp_groupby_kibana_sample_data_ky_avl.kytcCountyNmbr'; const responseWithNumberTypes = { aggregations: { join: { @@ -150,14 +79,14 @@ describe('extractPropertiesMap', () => { { key: 109, doc_count: 1130, - '__kbnjoin__min_of_avlAirTemp_groupby_kibana_sample_data_ky_avl.kytcCountyNmbr': { + [minPropName]: { value: 36, }, }, { key: 62, doc_count: 448, - '__kbnjoin__min_of_avlAirTemp_groupby_kibana_sample_data_ky_avl.kytcCountyNmbr': { + [minPropName]: { value: 0, }, }, @@ -166,11 +95,10 @@ describe('extractPropertiesMap', () => { }, }; const countPropName = '__kbnjoin__count_groupby_kibana_sample_data_ky_avl.kytcCountyNmbr'; - const minPropName = - '__kbnjoin__min_of_avlAirTemp_groupby_kibana_sample_data_ky_avl.kytcCountyNmbr'; + let propertiesMap; beforeAll(() => { - propertiesMap = extractPropertiesMap(responseWithNumberTypes, [minPropName], countPropName); + propertiesMap = extractPropertiesMap(responseWithNumberTypes, countPropName); }); it('should create key for each join term', () => { From 66967ab1d301fbf61e39293787d2a47eb1eb7a78 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 18 Feb 2020 09:17:26 -0700 Subject: [PATCH 05/29] better join labels --- .../public/layers/sources/es_term_source.js | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.js index 39d335c74819fe..f5a55fe7a3d9df 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.js @@ -7,12 +7,7 @@ import _ from 'lodash'; import { i18n } from '@kbn/i18n'; -import { - COUNT_PROP_LABEL, - DEFAULT_MAX_BUCKETS_LIMIT, - FIELD_ORIGIN, - AGG_TYPE, -} from '../../../common/constants'; +import { DEFAULT_MAX_BUCKETS_LIMIT, FIELD_ORIGIN, AGG_TYPE } from '../../../common/constants'; import { ESDocField } from '../fields/es_doc_field'; import { AbstractESAggSource, AGG_DELIMITER } from './es_agg_source'; import { getField, addFieldToDSL, extractPropertiesFromBucket } from '../util/es_agg_utils'; @@ -80,8 +75,20 @@ export class ESTermSource extends AbstractESAggSource { } formatMetricLabel(type, fieldName) { - const metricLabel = type !== AGG_TYPE.COUNT ? `${type} ${fieldName}` : COUNT_PROP_LABEL; - return `${metricLabel} of ${this._descriptor.indexPatternTitle}:${this._termField.getName()}`; + switch (type) { + case AGG_TYPE.COUNT: + return i18n.translate('xpack.maps.source.esJoin.countLabel', { + defaultMessage: `Count of {indexPatternTitle}`, + values: { indexPatternTitle: this._descriptor.indexPatternTitle }, + }); + case AGG_TYPE.TERMS: + return i18n.translate('xpack.maps.source.esJoin.topTermLabel', { + defaultMessage: `Top {fieldName}`, + values: { fieldName }, + }); + default: + return `${type} ${fieldName}`; + } } async getPropertiesMap(searchFilters, leftSourceName, leftFieldName, registerCancelCallback) { From 1dff40cdc0a5b60d450914cc8ff6e444875b1aa9 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 18 Feb 2020 09:23:59 -0700 Subject: [PATCH 06/29] categoricla meta --- .../legacy/plugins/maps/public/layers/fields/es_agg_field.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.js b/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.js index ef234ceaa8daf7..ee9be2d99d043a 100644 --- a/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.js +++ b/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.js @@ -80,4 +80,8 @@ export class ESAggMetricField extends AbstractField { async getOrdinalFieldMetaRequest(config) { return this._esDocField.getOrdinalFieldMetaRequest(config); } + + async getCategoricalFieldMetaRequest() { + return this._esDocField.getCategoricalFieldMetaRequest(); + } } From 19d372027e98c3664f6ccb2658a089ab27ed9ae0 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 18 Feb 2020 09:29:08 -0700 Subject: [PATCH 07/29] remove unused constant --- x-pack/legacy/plugins/maps/common/constants.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/legacy/plugins/maps/common/constants.ts b/x-pack/legacy/plugins/maps/common/constants.ts index f8c87212aeea50..542abfb004d0d6 100644 --- a/x-pack/legacy/plugins/maps/common/constants.ts +++ b/x-pack/legacy/plugins/maps/common/constants.ts @@ -127,7 +127,6 @@ export const AGG_TYPE = { UNIQUE_COUNT: 'cardinality', }; -export const COUNT_AGG_TYPE = AGG_TYPE.COUNT; export const COUNT_PROP_LABEL = i18n.translate('xpack.maps.aggs.defaultCountLabel', { defaultMessage: 'count', }); From b99138a92823d57b346d77e1b5c33ff39c4ff750 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 18 Feb 2020 09:30:43 -0700 Subject: [PATCH 08/29] remove unused changes --- x-pack/legacy/plugins/maps/public/kibana_services.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/legacy/plugins/maps/public/kibana_services.js b/x-pack/legacy/plugins/maps/public/kibana_services.js index 05d333bda336d3..a1b1c9ec1518e6 100644 --- a/x-pack/legacy/plugins/maps/public/kibana_services.js +++ b/x-pack/legacy/plugins/maps/public/kibana_services.js @@ -13,8 +13,6 @@ import { npStart } from 'ui/new_platform'; export const SPATIAL_FILTER_TYPE = esFilters.FILTERS.SPATIAL_FILTER; export { SearchSource } from '../../../../../src/plugins/data/public'; -//export { KBN_FIELD_TYPES } from '../../../../../src/plugins/data/common'; -export const KBN_FIELD_TYPES = {}; export const indexPatternService = npStart.plugins.data.indexPatterns; export const autocompleteService = npStart.plugins.data.autocomplete; From 8aceadb766621d24f3c748ea0e27aea84b916c08 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 18 Feb 2020 09:33:36 -0700 Subject: [PATCH 09/29] remove unused constant METRIC_SCHEMA_CONFIG --- .../maps/public/layers/sources/es_agg_source.js | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_agg_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_agg_source.js index f24c3dda3bef1d..bee35216f59dab 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_agg_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_agg_source.js @@ -17,23 +17,6 @@ import { export const AGG_DELIMITER = '_of_'; export class AbstractESAggSource extends AbstractESSource { - static METRIC_SCHEMA_CONFIG = { - group: 'metrics', - name: 'metric', - title: 'Value', - min: 1, - max: Infinity, - aggFilter: [ - AGG_TYPE.AVG, - AGG_TYPE.COUNT, - AGG_TYPE.MAX, - AGG_TYPE.MIN, - AGG_TYPE.SUM, - AGG_TYPE.UNIQUE_COUNT, - ], - defaults: [{ schema: 'metric', type: AGG_TYPE.COUNT }], - }; - constructor(descriptor, inspectorAdapters) { super(descriptor, inspectorAdapters); this._metricFields = this._descriptor.metrics From 6011af3e9c1d739450ad76b8fc79c9a691bc655d Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 18 Feb 2020 11:23:15 -0700 Subject: [PATCH 10/29] update jest expect --- .../plugins/maps/public/layers/sources/es_term_source.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.test.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.test.js index 3247ce24d8a59d..e14b2973af5b92 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.test.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.test.js @@ -44,7 +44,7 @@ describe('getMetricFields', () => { expect(metrics[0].getAggType()).toEqual('count'); expect(metrics[0].getName()).toEqual('__kbnjoin__count_groupby_myIndex.myTermField'); - expect(await metrics[0].getLabel()).toEqual('count of myIndex:myTermField'); + expect(await metrics[0].getLabel()).toEqual('Count of myIndex'); }); it('should remove incomplete metric configurations', async () => { @@ -65,7 +65,7 @@ describe('getMetricFields', () => { expect(metrics[1].getAggType()).toEqual('count'); expect(metrics[1].getName()).toEqual('__kbnjoin__count_groupby_myIndex.myTermField'); - expect(await metrics[1].getLabel()).toEqual('count of myIndex:myTermField'); + expect(await metrics[1].getLabel()).toEqual('Count of myIndex'); }); }); From 90aeae29dc1e3543f7697894a8f3c9b1d7a135e2 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 18 Feb 2020 11:52:56 -0700 Subject: [PATCH 11/29] fix auto complete suggestions for top term --- .../legacy/plugins/maps/public/layers/sources/es_source.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_source.js index f3cdc8cb290b2a..782f2845ceeffc 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_source.js @@ -347,13 +347,16 @@ export class AbstractESSource extends AbstractVectorSource { } getValueSuggestions = async (fieldName, query) => { - if (!fieldName) { + // fieldName could be an aggregation so it needs to be unpacked to expose raw field. + const metricField = this.getMetricFields().find(field => field.getName() === fieldName); + const realFieldName = metricField ? metricField.getESDocFieldName() : fieldName; + if (!realFieldName) { return []; } try { const indexPattern = await this.getIndexPattern(); - const field = indexPattern.fields.getByName(fieldName); + const field = indexPattern.fields.getByName(realFieldName); return await autocompleteService.getValueSuggestions({ indexPattern, field, From 28a854fc57b54728a2cd6f13dc4b641eaf2c0bc4 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 18 Feb 2020 12:25:33 -0700 Subject: [PATCH 12/29] get category autocomplete working with style props from joins --- .../vector/properties/dynamic_size_property.js | 12 ++---------- .../vector/properties/dynamic_style_property.js | 10 +++++++--- .../maps/public/layers/styles/vector/vector_style.js | 10 +++------- 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/x-pack/legacy/plugins/maps/public/layers/styles/vector/properties/dynamic_size_property.js b/x-pack/legacy/plugins/maps/public/layers/styles/vector/properties/dynamic_size_property.js index e137e15730827c..dfc5c530cc90fa 100644 --- a/x-pack/legacy/plugins/maps/public/layers/styles/vector/properties/dynamic_size_property.js +++ b/x-pack/legacy/plugins/maps/public/layers/styles/vector/properties/dynamic_size_property.js @@ -43,16 +43,8 @@ function getSymbolSizeIcons() { } export class DynamicSizeProperty extends DynamicStyleProperty { - constructor( - options, - styleName, - field, - getFieldMeta, - getFieldFormatter, - getValueSuggestions, - isSymbolizedAsIcon - ) { - super(options, styleName, field, getFieldMeta, getFieldFormatter, getValueSuggestions); + constructor(options, styleName, field, getFieldMeta, getFieldFormatter, isSymbolizedAsIcon) { + super(options, styleName, field, getFieldMeta, getFieldFormatter); this._isSymbolizedAsIcon = isSymbolizedAsIcon; } diff --git a/x-pack/legacy/plugins/maps/public/layers/styles/vector/properties/dynamic_style_property.js b/x-pack/legacy/plugins/maps/public/layers/styles/vector/properties/dynamic_style_property.js index ef19e9b23b10de..56f8daf9f17c58 100644 --- a/x-pack/legacy/plugins/maps/public/layers/styles/vector/properties/dynamic_style_property.js +++ b/x-pack/legacy/plugins/maps/public/layers/styles/vector/properties/dynamic_style_property.js @@ -17,17 +17,17 @@ import { OrdinalFieldMetaOptionsPopover } from '../components/ordinal_field_meta export class DynamicStyleProperty extends AbstractStyleProperty { static type = STYLE_TYPE.DYNAMIC; - constructor(options, styleName, field, getFieldMeta, getFieldFormatter, source) { + constructor(options, styleName, field, getFieldMeta, getFieldFormatter) { super(options, styleName); this._field = field; this._getFieldMeta = getFieldMeta; this._getFieldFormatter = getFieldFormatter; - this._source = source; } getValueSuggestions = query => { const fieldName = this.getFieldName(); - return this._source && fieldName ? this._source.getValueSuggestions(fieldName, query) : []; + const fieldSource = this.getFieldSource(); + return fieldSource && fieldName ? fieldSource.getValueSuggestions(fieldName, query) : []; }; getFieldMeta() { @@ -38,6 +38,10 @@ export class DynamicStyleProperty extends AbstractStyleProperty { return this._field; } + getFieldSource() { + return this._field ? this._field.getSource() : null; + } + getFieldName() { return this._field ? this._field.getName() : ''; } diff --git a/x-pack/legacy/plugins/maps/public/layers/styles/vector/vector_style.js b/x-pack/legacy/plugins/maps/public/layers/styles/vector/vector_style.js index 62651fdd702d60..053aa114d94aea 100644 --- a/x-pack/legacy/plugins/maps/public/layers/styles/vector/vector_style.js +++ b/x-pack/legacy/plugins/maps/public/layers/styles/vector/vector_style.js @@ -625,7 +625,6 @@ export class VectorStyle extends AbstractStyle { field, this._getFieldMeta, this._getFieldFormatter, - this._source, isSymbolizedAsIcon ); } else { @@ -645,8 +644,7 @@ export class VectorStyle extends AbstractStyle { styleName, field, this._getFieldMeta, - this._getFieldFormatter, - this._source + this._getFieldFormatter ); } else { throw new Error(`${descriptor} not implemented`); @@ -678,8 +676,7 @@ export class VectorStyle extends AbstractStyle { VECTOR_STYLES.LABEL_TEXT, field, this._getFieldMeta, - this._getFieldFormatter, - this._source + this._getFieldFormatter ); } else { throw new Error(`${descriptor} not implemented`); @@ -698,8 +695,7 @@ export class VectorStyle extends AbstractStyle { VECTOR_STYLES.ICON, field, this._getFieldMeta, - this._getFieldFormatter, - this._source + this._getFieldFormatter ); } else { throw new Error(`${descriptor} not implemented`); From 46af66736dbf899d28f89f327dc18d650493d04e Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 18 Feb 2020 13:22:51 -0700 Subject: [PATCH 13/29] pluck categorical style meta with real field name --- .../styles/vector/properties/dynamic_style_property.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/x-pack/legacy/plugins/maps/public/layers/styles/vector/properties/dynamic_style_property.js b/x-pack/legacy/plugins/maps/public/layers/styles/vector/properties/dynamic_style_property.js index 56f8daf9f17c58..679a039d5f4d4c 100644 --- a/x-pack/legacy/plugins/maps/public/layers/styles/vector/properties/dynamic_style_property.js +++ b/x-pack/legacy/plugins/maps/public/layers/styles/vector/properties/dynamic_style_property.js @@ -207,12 +207,14 @@ export class DynamicStyleProperty extends AbstractStyleProperty { } _pluckCategoricalStyleMetaFromFieldMetaData(fieldMetaData) { - const name = this.getField().getName(); - if (!fieldMetaData[name] || !fieldMetaData[name].buckets) { + const realFieldName = this._field.getESDocFieldName + ? this._field.getESDocFieldName() + : this._field.getName(); + if (!fieldMetaData[realFieldName] || !fieldMetaData[realFieldName].buckets) { return null; } - const ordered = fieldMetaData[name].buckets.map(bucket => { + const ordered = fieldMetaData[realFieldName].buckets.map(bucket => { return { key: bucket.key, count: bucket.doc_count, From 66c5f2b16c3c5e12442e95d730e0913d0ea6a65a Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 18 Feb 2020 15:16:37 -0700 Subject: [PATCH 14/29] mock MetricsEditor to fix jest test --- .../join_editor/resources/metrics_expression.test.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/x-pack/legacy/plugins/maps/public/connected_components/layer_panel/join_editor/resources/metrics_expression.test.js b/x-pack/legacy/plugins/maps/public/connected_components/layer_panel/join_editor/resources/metrics_expression.test.js index e0e1556ecde068..e4e3776c8e92ca 100644 --- a/x-pack/legacy/plugins/maps/public/connected_components/layer_panel/join_editor/resources/metrics_expression.test.js +++ b/x-pack/legacy/plugins/maps/public/connected_components/layer_panel/join_editor/resources/metrics_expression.test.js @@ -4,6 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ +jest.mock('../../../../components/metrics_editor', () => ({ + MetricsEditor: () => { + return
mockMetricsEditor
; + }, +})); + import React from 'react'; import { shallow } from 'enzyme'; import { MetricsExpression } from './metrics_expression'; From ecc64b0e62b7e6366ede4f4c7e16c94aefc78fd2 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Wed, 19 Feb 2020 15:10:39 -0700 Subject: [PATCH 15/29] review feedback --- .../es_geo_grid_source/convert_to_geojson.js | 4 +- .../es_pew_pew_source/convert_to_lines.js | 3 +- .../public/layers/sources/es_term_source.js | 3 +- .../layers/sources/es_term_source.test.js | 3 -- .../properties/dynamic_style_property.js | 15 +++++--- .../maps/public/layers/util/es_agg_utils.js | 8 ++-- .../public/layers/util/es_agg_utils.test.js | 37 +++++++++++++++++++ 7 files changed, 57 insertions(+), 16 deletions(-) create mode 100644 x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.test.js diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js index b5930832647fb4..669dd2aff737e3 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js @@ -9,6 +9,8 @@ import { RENDER_AS } from './render_as'; import { getTileBoundingBox } from './geo_tile_utils'; import { extractPropertiesFromBucket } from '../../util/es_agg_utils'; +const GRID_BUCKET_KEYS_TO_IGNORE = ['key', 'gridCentroid']; + export function convertToGeoJson(esResponse, renderAs) { const features = []; @@ -23,7 +25,7 @@ export function convertToGeoJson(esResponse, renderAs) { renderAs, }), id: gridBucket.key, - properties: extractPropertiesFromBucket(gridBucket, ['key', 'gridCentroid']), + properties: extractPropertiesFromBucket(gridBucket, GRID_BUCKET_KEYS_TO_IGNORE), }); } diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.js index 8395f7d2b34c3b..96a7f50cdf523a 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.js @@ -9,6 +9,7 @@ import { extractPropertiesFromBucket } from '../../util/es_agg_utils'; const LAT_INDEX = 0; const LON_INDEX = 1; +const PEW_PEW_BUCKET_KEYS_TO_IGNORE = ['key', 'sourceCentroid']; function parsePointFromKey(key) { const split = key.split(','); @@ -35,7 +36,7 @@ export function convertToLines(esResponse) { coordinates: [[sourceCentroid.location.lon, sourceCentroid.location.lat], dest], }, id: `${dest.join()},${sourceBucket.key}`, - properties: extractPropertiesFromBucket(sourceBucket, ['key', 'sourceCentroid']), + properties: extractPropertiesFromBucket(sourceBucket, PEW_PEW_BUCKET_KEYS_TO_IGNORE), }); } } diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.js index f5a55fe7a3d9df..9cc2919404a940 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.js @@ -16,11 +16,12 @@ const TERMS_AGG_NAME = 'join'; const FIELD_NAME_PREFIX = '__kbnjoin__'; const GROUP_BY_DELIMITER = '_groupby_'; +const TERMS_BUCKET_KEYS_TO_IGNORE = ['key', 'doc_count']; export function extractPropertiesMap(rawEsData, countPropertyName) { const propertiesMap = new Map(); _.get(rawEsData, ['aggregations', TERMS_AGG_NAME, 'buckets'], []).forEach(termBucket => { - const properties = extractPropertiesFromBucket(termBucket, ['key', 'doc_count']); + const properties = extractPropertiesFromBucket(termBucket, TERMS_BUCKET_KEYS_TO_IGNORE); if (countPropertyName) { properties[countPropertyName] = termBucket.doc_count; } diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.test.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.test.js index e14b2973af5b92..39cc301d458cbe 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.test.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.test.js @@ -8,9 +8,6 @@ import { ESTermSource, extractPropertiesMap } from './es_term_source'; jest.mock('ui/new_platform'); jest.mock('../vector_layer', () => {}); -jest.mock('ui/agg_types', () => ({ - Schemas: function() {}, -})); jest.mock('ui/timefilter', () => {}); const indexPatternTitle = 'myIndex'; diff --git a/x-pack/legacy/plugins/maps/public/layers/styles/vector/properties/dynamic_style_property.js b/x-pack/legacy/plugins/maps/public/layers/styles/vector/properties/dynamic_style_property.js index 679a039d5f4d4c..af78c4c0e461e8 100644 --- a/x-pack/legacy/plugins/maps/public/layers/styles/vector/properties/dynamic_style_property.js +++ b/x-pack/legacy/plugins/maps/public/layers/styles/vector/properties/dynamic_style_property.js @@ -13,6 +13,7 @@ import React from 'react'; import { OrdinalLegend } from './components/ordinal_legend'; import { CategoricalLegend } from './components/categorical_legend'; import { OrdinalFieldMetaOptionsPopover } from '../components/ordinal_field_meta_options_popover'; +import { ESAggMetricField } from '../../../fields/es_agg_field'; export class DynamicStyleProperty extends AbstractStyleProperty { static type = STYLE_TYPE.DYNAMIC; @@ -184,9 +185,10 @@ export class DynamicStyleProperty extends AbstractStyleProperty { } _pluckOrdinalStyleMetaFromFieldMetaData(fieldMetaData) { - const realFieldName = this._field.getESDocFieldName - ? this._field.getESDocFieldName() - : this._field.getName(); + const realFieldName = + this._field instanceof ESAggMetricField + ? this._field.getESDocFieldName() + : this._field.getName(); const stats = fieldMetaData[realFieldName]; if (!stats) { return null; @@ -207,9 +209,10 @@ export class DynamicStyleProperty extends AbstractStyleProperty { } _pluckCategoricalStyleMetaFromFieldMetaData(fieldMetaData) { - const realFieldName = this._field.getESDocFieldName - ? this._field.getESDocFieldName() - : this._field.getName(); + const realFieldName = + this._field instanceof ESAggMetricField + ? this._field.getESDocFieldName() + : this._field.getName(); if (!fieldMetaData[realFieldName] || !fieldMetaData[realFieldName].buckets) { return null; } diff --git a/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.js b/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.js index 95af8e7465103f..40f4752a2b9940 100644 --- a/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.js +++ b/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.js @@ -31,11 +31,11 @@ export function addFieldToDSL(dsl, field) { }; } -export function extractPropertiesFromBucket(bucket, ignoreKeys) { +export function extractPropertiesFromBucket(bucket, ignoreKeys = []) { const properties = {}; - Object.keys(bucket).forEach(key => { + for (const key in bucket) { if (ignoreKeys.includes(key)) { - return; + continue; } if (_.has(bucket[key], 'value')) { @@ -45,6 +45,6 @@ export function extractPropertiesFromBucket(bucket, ignoreKeys) { } else { properties[key] = bucket[key]; } - }); + } return properties; } diff --git a/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.test.js b/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.test.js new file mode 100644 index 00000000000000..201d6907981a2b --- /dev/null +++ b/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.test.js @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { extractPropertiesFromBucket } from './es_agg_utils'; + +describe('extractPropertiesFromBucket', () => { + test('Should ignore specified keys', () => { + const properties = extractPropertiesFromBucket({ key: '4/4/6' }, ['key']); + expect(properties).toEqual({}); + }); + + test('Should extract metric aggregation values', () => { + const properties = extractPropertiesFromBucket({ avg_of_bytes: { value: 5359 } }); + expect(properties).toEqual({ + avg_of_bytes: 5359, + }); + }); + + test('Should extract bucket aggregation values', () => { + const properties = extractPropertiesFromBucket({ + 'terms_of_machine.os.keyword': { + buckets: [ + { + key: 'win xp', + doc_count: 16, + }, + ], + }, + }); + expect(properties).toEqual({ + 'terms_of_machine.os.keyword': 'win xp', + }); + }); +}); From dd6f7a892e4f1dfa7b4524cd40f3f7e479da6e61 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Wed, 19 Feb 2020 15:36:34 -0700 Subject: [PATCH 16/29] es_agg_utils.js to es_agg_utils.ts --- .../layers/util/{es_agg_utils.js => es_agg_utils.ts} | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) rename x-pack/legacy/plugins/maps/public/layers/util/{es_agg_utils.js => es_agg_utils.ts} (78%) diff --git a/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.js b/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.ts similarity index 78% rename from x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.js rename to x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.ts index 40f4752a2b9940..46bd1ebbc9f2cc 100644 --- a/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.js +++ b/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.ts @@ -5,8 +5,9 @@ */ import { i18n } from '@kbn/i18n'; import _ from 'lodash'; +import { IndexPattern, IFieldType } from '../../../../../../../src/plugins/data/public'; -export function getField(indexPattern, fieldName) { +export function getField(indexPattern: IndexPattern, fieldName: string) { const field = indexPattern.fields.getByName(fieldName); if (!field) { throw new Error( @@ -19,7 +20,7 @@ export function getField(indexPattern, fieldName) { return field; } -export function addFieldToDSL(dsl, field) { +export function addFieldToDSL(dsl: any, field: IFieldType) { return !field.scripted ? { ...dsl, field: field.name } : { @@ -31,8 +32,8 @@ export function addFieldToDSL(dsl, field) { }; } -export function extractPropertiesFromBucket(bucket, ignoreKeys = []) { - const properties = {}; +export function extractPropertiesFromBucket(bucket: any, ignoreKeys: string[] = []) { + const properties: any = {}; for (const key in bucket) { if (ignoreKeys.includes(key)) { continue; From 71d93c198db981eede93a56e256e753525421615 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Thu, 20 Feb 2020 06:23:06 -0700 Subject: [PATCH 17/29] typing updates --- x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.ts b/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.ts index 46bd1ebbc9f2cc..933b4d6b613383 100644 --- a/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.ts +++ b/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.ts @@ -20,7 +20,7 @@ export function getField(indexPattern: IndexPattern, fieldName: string) { return field; } -export function addFieldToDSL(dsl: any, field: IFieldType) { +export function addFieldToDSL(dsl: object, field: IFieldType) { return !field.scripted ? { ...dsl, field: field.name } : { @@ -33,7 +33,7 @@ export function addFieldToDSL(dsl: any, field: IFieldType) { } export function extractPropertiesFromBucket(bucket: any, ignoreKeys: string[] = []) { - const properties: any = {}; + const properties: Record = {}; for (const key in bucket) { if (ignoreKeys.includes(key)) { continue; From adddefeea66c0b4f0c1dd85b8aed442d738fce6b Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Fri, 21 Feb 2020 07:51:39 -0700 Subject: [PATCH 18/29] use composit agg to avoid search.buckets limit --- .../maps/public/layers/fields/es_agg_field.js | 2 +- .../es_geo_grid_source/convert_to_geojson.js | 14 ++- .../convert_to_geojson.test.js | 19 ++-- .../es_geo_grid_source/es_geo_grid_source.js | 87 +++++++++++++++---- .../es_geo_grid_source/geo_tile_utils.js | 22 ++++- 5 files changed, 108 insertions(+), 36 deletions(-) diff --git a/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.js b/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.js index ee9be2d99d043a..28c199b64d3ef4 100644 --- a/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.js +++ b/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.js @@ -66,7 +66,7 @@ export class ESAggMetricField extends AbstractField { getValueAggDsl(indexPattern) { const field = getField(indexPattern, this.getESDocFieldName()); const aggType = this.getAggType(); - const aggBody = aggType === AGG_TYPE.TERMS ? { size: 1 } : {}; + const aggBody = aggType === AGG_TYPE.TERMS ? { size: 1, shard_size: 1 } : {}; return { [aggType]: addFieldToDSL(aggBody, field), }; diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js index 669dd2aff737e3..4ff6ca69192615 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js @@ -14,27 +14,23 @@ const GRID_BUCKET_KEYS_TO_IGNORE = ['key', 'gridCentroid']; export function convertToGeoJson(esResponse, renderAs) { const features = []; - const gridBuckets = _.get(esResponse, 'aggregations.gridSplit.buckets', []); + const gridBuckets = _.get(esResponse, 'aggregations.compositeSplit.buckets', []); for (let i = 0; i < gridBuckets.length; i++) { const gridBucket = gridBuckets[i]; + const gridKey = gridBucket.key.gridSplit; features.push({ type: 'Feature', geometry: rowToGeometry({ - gridKey: gridBucket.key, + gridKey, gridCentroid: gridBucket.gridCentroid, renderAs, }), - id: gridBucket.key, + id: gridKey, properties: extractPropertiesFromBucket(gridBucket, GRID_BUCKET_KEYS_TO_IGNORE), }); } - return { - featureCollection: { - type: 'FeatureCollection', - features: features, - }, - }; + return features; } function rowToGeometry({ gridKey, gridCentroid, renderAs }) { diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.test.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.test.js index 77fb1ae349b077..bc0e524aba7be9 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.test.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.test.js @@ -9,10 +9,13 @@ import { RENDER_AS } from './render_as'; const esResponse = { aggregations: { - gridSplit: { + compositeSplit: { + after_key: { + gridSplit: '10/327/460', + }, buckets: [ { - key: '4/4/6', + key: { gridSplit: '4/4/6' }, doc_count: 65, avg_of_bytes: { value: 5359.2307692307695 }, 'terms_of_machine.os.keyword': { @@ -34,9 +37,9 @@ const esResponse = { }; it('Should convert elasticsearch aggregation response into feature collection of points', () => { - const geoJson = convertToGeoJson(esResponse, RENDER_AS.POINT); - expect(geoJson.featureCollection.features.length).toBe(1); - expect(geoJson.featureCollection.features[0]).toEqual({ + const features = convertToGeoJson(esResponse, RENDER_AS.POINT); + expect(features.length).toBe(1); + expect(features[0]).toEqual({ geometry: { coordinates: [-81.94552666092149, 36.62813963153614], type: 'Point', @@ -52,9 +55,9 @@ it('Should convert elasticsearch aggregation response into feature collection of }); it('Should convert elasticsearch aggregation response into feature collection of Polygons', () => { - const geoJson = convertToGeoJson(esResponse, RENDER_AS.GRID); - expect(geoJson.featureCollection.features.length).toBe(1); - expect(geoJson.featureCollection.features[0]).toEqual({ + const features = convertToGeoJson(esResponse, RENDER_AS.GRID); + expect(features.length).toBe(1); + expect(features[0]).toEqual({ geometry: { coordinates: [ [ diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js index ca008a820d4637..8a4de35d14b084 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js @@ -22,6 +22,8 @@ import { CreateSourceEditor } from './create_source_editor'; import { UpdateSourceEditor } from './update_source_editor'; import { GRID_RESOLUTION } from '../../grid_resolution'; import { + AGG_TYPE, + DEFAULT_MAX_BUCKETS_LIMIT, SOURCE_DATA_ID_ORIGIN, ES_GEO_GRID, COUNT_PROP_NAME, @@ -164,11 +166,29 @@ export class ESGeoGridSource extends AbstractESAggSource { async getGeoJsonWithMeta(layerName, searchFilters, registerCancelCallback) { const indexPattern = await this.getIndexPattern(); const searchSource = await this._makeSearchSource(searchFilters, 0); - searchSource.setField('aggs', { - gridSplit: { - geotile_grid: { - field: this._descriptor.geoField, - precision: searchFilters.geogridPrecision, + + let bucketsPerGrid = 1; + this.getMetricFields().forEach(metricField => { + if (metricField.getAggType() === AGG_TYPE.TERMS) { + // each terms aggregation increases the overall number of buckets per grid + bucketsPerGrid++; + } + }); + const gridsPerRequest = DEFAULT_MAX_BUCKETS_LIMIT / bucketsPerGrid; + const aggs = { + compositeSplit: { + composite: { + size: gridsPerRequest, + sources: [ + { + gridSplit: { + geotile_grid: { + field: this._descriptor.geoField, + precision: searchFilters.geogridPrecision, + }, + }, + }, + ], }, aggs: { gridCentroid: { @@ -179,21 +199,54 @@ export class ESGeoGridSource extends AbstractESAggSource { ...this.getValueAggsDsl(indexPattern), }, }, - }); + }; - const esResponse = await this._runEsQuery({ - requestId: this.getId(), - requestName: layerName, - searchSource, - registerCancelCallback, - requestDescription: i18n.translate('xpack.maps.source.esGrid.inspectorDescription', { - defaultMessage: 'Elasticsearch geo grid aggregation request', - }), - }); + const features = []; + let requestCount = 0; + let afterKey = null; + while (true) { + requestCount++; + + // circuit breaker to ensure reasonable number of requests + if (requestCount > 5) { + throw new Error( + i18n.translate('xpack.maps.source.esGrid.compositePaginationErrorMessage', { + defaultMessage: `{layerName} is causing too many requests. Reduce "Grid resolution" and/or reduce the number of top term "Metrics".`, + values: { layerName }, + }) + ); + } + + if (afterKey) { + aggs.compositeSplit.composite.after = afterKey; + } + searchSource.setField('aggs', aggs); + const requestId = afterKey ? `${this.getId()} afterKey ${afterKey.geoSplit}` : this.getId(); + const esResponse = await this._runEsQuery({ + requestId, + requestName: `${layerName} (${requestCount})`, + searchSource, + registerCancelCallback, + requestDescription: i18n.translate('xpack.maps.source.esGrid.inspectorDescription', { + defaultMessage: 'Elasticsearch geo grid aggregation request: {requestId}', + values: { requestId }, + }), + }); + + features.push(...convertToGeoJson(esResponse, this._descriptor.requestType)); + + afterKey = esResponse.aggregations.compositeSplit.after_key; + if (esResponse.aggregations.compositeSplit.buckets.length < gridsPerRequest) { + // Finished because request did not get full resultset back + break; + } + } - const { featureCollection } = convertToGeoJson(esResponse, this._descriptor.requestType); return { - data: featureCollection, + data: { + type: 'FeatureCollection', + features: features, + }, meta: { areResultsTrimmed: false, }, diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/geo_tile_utils.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/geo_tile_utils.js index da0bc1685f2237..e1d998ea8d506f 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/geo_tile_utils.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/geo_tile_utils.js @@ -87,7 +87,13 @@ function sec(value) { } function latitudeToTile(lat, tileCount) { - const radians = (lat * Math.PI) / 180; + let boundedLat = lat; + if (lat >= 90) { + boundedLat = 89.9; + } else if (lat <= -90) { + boundedLat = -89.9; + } + const radians = (boundedLat * Math.PI) / 180; const y = ((1 - Math.log(Math.tan(radians) + sec(radians)) / Math.PI) / 2) * tileCount; return Math.floor(y); } @@ -112,3 +118,17 @@ export function expandToTileBoundaries(extent, zoom) { maxLat: tileToLatitude(upperLeftY, tileCount), }; } + +export function getTileCountInExtent(extent, zoom) { + const tileCount = getTileCount(zoom); + + const upperLeftX = longitudeToTile(extent.minLon, tileCount); + const upperLeftY = latitudeToTile(Math.min(extent.maxLat, 90), tileCount); + const lowerRightX = longitudeToTile(extent.maxLon, tileCount); + const lowerRightY = latitudeToTile(Math.max(extent.minLat, -90), tileCount); + + const tilesY = Math.abs(upperLeftY - lowerRightY); + const tilesX = Math.abs(upperLeftX - lowerRightX); + + return tilesX * tilesY; +} From 17bf8050c3aa5c7cab1629fdb8c04032ea257be6 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Fri, 21 Feb 2020 10:12:58 -0700 Subject: [PATCH 19/29] i18n update and functional test fix --- x-pack/plugins/translations/translations/ja-JP.json | 1 - x-pack/plugins/translations/translations/zh-CN.json | 1 - x-pack/test/functional/apps/maps/embeddable/dashboard.js | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 4b06645cdfe043..c0360fec680242 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -7277,7 +7277,6 @@ "xpack.maps.source.esGrid.finestDropdownOption": "最も細かい", "xpack.maps.source.esGrid.geospatialFieldLabel": "地理空間フィールド", "xpack.maps.source.esGrid.indexPatternLabel": "インデックスパターン", - "xpack.maps.source.esGrid.inspectorDescription": "Elasticsearch ジオグリッド集約リクエスト", "xpack.maps.source.esGrid.metricsLabel": "メトリック", "xpack.maps.source.esGrid.noIndexPatternErrorMessage": "インデックスパターン {id} が見つかりません", "xpack.maps.source.esGrid.resolutionParamErrorMessage": "グリッド解像度パラメーターが認識されません: {resolution}", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index ecf4dfbb33be65..0f52a00a8f2aa1 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -7276,7 +7276,6 @@ "xpack.maps.source.esGrid.finestDropdownOption": "最精致化", "xpack.maps.source.esGrid.geospatialFieldLabel": "地理空间字段", "xpack.maps.source.esGrid.indexPatternLabel": "索引模式", - "xpack.maps.source.esGrid.inspectorDescription": "Elasticsearch 地理网格聚合请求", "xpack.maps.source.esGrid.metricsLabel": "指标", "xpack.maps.source.esGrid.noIndexPatternErrorMessage": "找不到索引模式 {id}", "xpack.maps.source.esGrid.resolutionParamErrorMessage": "无法识别网格分辨率参数：{resolution}", diff --git a/x-pack/test/functional/apps/maps/embeddable/dashboard.js b/x-pack/test/functional/apps/maps/embeddable/dashboard.js index 43d5cccb209058..2baf7994ce9ffb 100644 --- a/x-pack/test/functional/apps/maps/embeddable/dashboard.js +++ b/x-pack/test/functional/apps/maps/embeddable/dashboard.js @@ -55,7 +55,7 @@ export default function({ getPageObjects, getService }) { await dashboardPanelActions.openInspectorByTitle('geo grid vector grid example'); const gridExampleRequestNames = await inspector.getRequestNames(); await inspector.close(); - expect(gridExampleRequestNames).to.equal('logstash-*'); + expect(gridExampleRequestNames).to.equal('logstash-* (1)'); }); it('should apply container state (time, query, filters) to embeddable when loaded', async () => { From c8762eb04e7fb565a8e7079aaea1e80cc95c20d4 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Fri, 21 Feb 2020 12:05:27 -0700 Subject: [PATCH 20/29] stop paging through results when request is aborted --- .../maps/public/actions/map_actions.js | 22 +++++++++++++------ .../layers/obsolete_data_request_error.js | 11 ++++++++++ .../es_geo_grid_source/es_geo_grid_source.js | 10 +++++++-- .../maps/public/layers/vector_layer.js | 15 ++++++++----- .../maps/public/selectors/map_selectors.js | 15 +++++++++++++ 5 files changed, 59 insertions(+), 14 deletions(-) create mode 100644 x-pack/legacy/plugins/maps/public/layers/obsolete_data_request_error.js diff --git a/x-pack/legacy/plugins/maps/public/actions/map_actions.js b/x-pack/legacy/plugins/maps/public/actions/map_actions.js index 2c6c60db9a0124..607185a30c77f0 100644 --- a/x-pack/legacy/plugins/maps/public/actions/map_actions.js +++ b/x-pack/legacy/plugins/maps/public/actions/map_actions.js @@ -18,6 +18,7 @@ import { getTransientLayerId, getOpenTooltips, getQuery, + getDataRequest, } from '../selectors/map_selectors'; import { FLYOUT_STATE } from '../reducers/ui'; import { @@ -76,7 +77,7 @@ export const HIDE_LAYER_CONTROL = 'HIDE_LAYER_CONTROL'; export const HIDE_VIEW_CONTROL = 'HIDE_VIEW_CONTROL'; export const SET_WAITING_FOR_READY_HIDDEN_LAYERS = 'SET_WAITING_FOR_READY_HIDDEN_LAYERS'; -function getLayerLoadingCallbacks(dispatch, layerId) { +function getLayerLoadingCallbacks(dispatch, getState, layerId) { return { startLoading: (dataId, requestToken, meta) => dispatch(startDataLoad(layerId, dataId, requestToken, meta)), @@ -87,6 +88,13 @@ function getLayerLoadingCallbacks(dispatch, layerId) { updateSourceData: newData => { dispatch(updateSourceDataRequest(layerId, newData)); }, + isRequestStillActive: (dataId, requestToken) => { + const dataRequest = getDataRequest(getState(), layerId, dataId); + if (!dataRequest) { + return false; + } + return dataRequest.dataRequestToken === requestToken; + }, registerCancelCallback: (requestToken, callback) => dispatch(registerCancelCallback(requestToken, callback)), }; @@ -98,11 +106,11 @@ function getLayerById(layerId, state) { }); } -async function syncDataForAllLayers(getState, dispatch, dataFilters) { +async function syncDataForAllLayers(dispatch, getState, dataFilters) { const state = getState(); const layerList = getLayerList(state); const syncs = layerList.map(layer => { - const loadingFunctions = getLayerLoadingCallbacks(dispatch, layer.getId()); + const loadingFunctions = getLayerLoadingCallbacks(dispatch, getState, layer.getId()); return layer.syncData({ ...loadingFunctions, dataFilters }); }); await Promise.all(syncs); @@ -412,7 +420,7 @@ export function mapExtentChanged(newMapConstants) { }, }); const newDataFilters = { ...dataFilters, ...newMapConstants }; - await syncDataForAllLayers(getState, dispatch, newDataFilters); + await syncDataForAllLayers(dispatch, getState, newDataFilters); }; } @@ -653,7 +661,7 @@ export function syncDataForLayer(layerId) { const targetLayer = getLayerById(layerId, getState()); if (targetLayer) { const dataFilters = getDataFilters(getState()); - const loadingFunctions = getLayerLoadingCallbacks(dispatch, layerId); + const loadingFunctions = getLayerLoadingCallbacks(dispatch, getState, layerId); await targetLayer.syncData({ ...loadingFunctions, dataFilters, @@ -773,7 +781,7 @@ export function setQuery({ query, timeFilters, filters = [], refresh = false }) }); const dataFilters = getDataFilters(getState()); - await syncDataForAllLayers(getState, dispatch, dataFilters); + await syncDataForAllLayers(dispatch, getState, dataFilters); }; } @@ -792,7 +800,7 @@ export function triggerRefreshTimer() { }); const dataFilters = getDataFilters(getState()); - await syncDataForAllLayers(getState, dispatch, dataFilters); + await syncDataForAllLayers(dispatch, getState, dataFilters); }; } diff --git a/x-pack/legacy/plugins/maps/public/layers/obsolete_data_request_error.js b/x-pack/legacy/plugins/maps/public/layers/obsolete_data_request_error.js new file mode 100644 index 00000000000000..d66105d1747a9a --- /dev/null +++ b/x-pack/legacy/plugins/maps/public/layers/obsolete_data_request_error.js @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export class ObsoleteDataRequestError extends Error { + constructor() { + super(); + } +} diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js index 8a4de35d14b084..55ab4f33f75b37 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js @@ -34,6 +34,7 @@ import { getDataSourceLabel } from '../../../../common/i18n_getters'; import { AbstractESAggSource } from '../es_agg_source'; import { DynamicStyleProperty } from '../../styles/vector/properties/dynamic_style_property'; import { StaticStyleProperty } from '../../styles/vector/properties/static_style_property'; +import { DataRequestAbortError } from '../../util/data_request'; const MAX_GEOTILE_LEVEL = 29; @@ -163,7 +164,7 @@ export class ESGeoGridSource extends AbstractESAggSource { ); } - async getGeoJsonWithMeta(layerName, searchFilters, registerCancelCallback) { + async getGeoJsonWithMeta(layerName, searchFilters, registerCancelCallback, isRequestStillActive) { const indexPattern = await this.getIndexPattern(); const searchSource = await this._makeSearchSource(searchFilters, 0); @@ -174,7 +175,7 @@ export class ESGeoGridSource extends AbstractESAggSource { bucketsPerGrid++; } }); - const gridsPerRequest = DEFAULT_MAX_BUCKETS_LIMIT / bucketsPerGrid; + const gridsPerRequest = Math.floor(DEFAULT_MAX_BUCKETS_LIMIT / bucketsPerGrid); const aggs = { compositeSplit: { composite: { @@ -205,6 +206,11 @@ export class ESGeoGridSource extends AbstractESAggSource { let requestCount = 0; let afterKey = null; while (true) { + if (!isRequestStillActive()) { + // Stop paging through results if request is obsolete + throw new DataRequestAbortError(); + } + requestCount++; // circuit breaker to ensure reasonable number of requests diff --git a/x-pack/legacy/plugins/maps/public/layers/vector_layer.js b/x-pack/legacy/plugins/maps/public/layers/vector_layer.js index 1698d52ea4406d..e1a30c8aef1d37 100644 --- a/x-pack/legacy/plugins/maps/public/layers/vector_layer.js +++ b/x-pack/legacy/plugins/maps/public/layers/vector_layer.js @@ -365,8 +365,10 @@ export class VectorLayer extends AbstractLayer { onLoadError, registerCancelCallback, dataFilters, + isRequestStillActive, }) { - const requestToken = Symbol(`layer-${this.getId()}-${SOURCE_DATA_ID_ORIGIN}`); + const dataRequestId = SOURCE_DATA_ID_ORIGIN; + const requestToken = Symbol(`layer-${this.getId()}-${dataRequestId}`); const searchFilters = this._getSearchFilters(dataFilters); const prevDataRequest = this.getSourceDataRequest(); const canSkipFetch = await canSkipSourceUpdate({ @@ -382,22 +384,25 @@ export class VectorLayer extends AbstractLayer { } try { - startLoading(SOURCE_DATA_ID_ORIGIN, requestToken, searchFilters); + startLoading(dataRequestId, requestToken, searchFilters); const layerName = await this.getDisplayName(); const { data: sourceFeatureCollection, meta } = await this._source.getGeoJsonWithMeta( layerName, searchFilters, - registerCancelCallback.bind(null, requestToken) + registerCancelCallback.bind(null, requestToken), + () => { + return isRequestStillActive(dataRequestId, requestToken); + } ); const layerFeatureCollection = assignFeatureIds(sourceFeatureCollection); - stopLoading(SOURCE_DATA_ID_ORIGIN, requestToken, layerFeatureCollection, meta); + stopLoading(dataRequestId, requestToken, layerFeatureCollection, meta); return { refreshed: true, featureCollection: layerFeatureCollection, }; } catch (error) { if (!(error instanceof DataRequestAbortError)) { - onLoadError(SOURCE_DATA_ID_ORIGIN, requestToken, error.message); + onLoadError(dataRequestId, requestToken, error.message); } return { refreshed: false, diff --git a/x-pack/legacy/plugins/maps/public/selectors/map_selectors.js b/x-pack/legacy/plugins/maps/public/selectors/map_selectors.js index d1048a759beca0..b554b459f51b4d 100644 --- a/x-pack/legacy/plugins/maps/public/selectors/map_selectors.js +++ b/x-pack/legacy/plugins/maps/public/selectors/map_selectors.js @@ -125,6 +125,21 @@ export const getRefreshConfig = ({ map }) => { export const getRefreshTimerLastTriggeredAt = ({ map }) => map.mapState.refreshTimerLastTriggeredAt; +export function getLayerDescriptor(state = {}, layerId) { + const layerListRaw = getLayerListRaw(state); + return layerListRaw.find(layer => layer.id === layerId); +} + +export function getDataRequest(state = {}, layerId, dataId) { + const layerDescriptor = getLayerDescriptor(state, layerId); + if (!layerDescriptor || !layerDescriptor.__dataRequests) { + return; + } + return _.get(layerDescriptor, '__dataRequests', []).find(dataRequest => { + return dataRequest.dataId === dataId; + }); +} + export const getDataFilters = createSelector( getMapExtent, getMapBuffer, From 9be819ef808edeafc031d988b344e83f9abe50db Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Fri, 21 Feb 2020 12:21:41 -0700 Subject: [PATCH 21/29] remove unused file --- .../maps/public/layers/obsolete_data_request_error.js | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 x-pack/legacy/plugins/maps/public/layers/obsolete_data_request_error.js diff --git a/x-pack/legacy/plugins/maps/public/layers/obsolete_data_request_error.js b/x-pack/legacy/plugins/maps/public/layers/obsolete_data_request_error.js deleted file mode 100644 index d66105d1747a9a..00000000000000 --- a/x-pack/legacy/plugins/maps/public/layers/obsolete_data_request_error.js +++ /dev/null @@ -1,11 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -export class ObsoleteDataRequestError extends Error { - constructor() { - super(); - } -} From 572a97fb7e4caa321de688e60a87513402c85b88 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Mon, 24 Feb 2020 13:11:46 -0700 Subject: [PATCH 22/29] do not use composite agg when no terms sub-aggregations --- .../es_geo_grid_source/convert_to_geojson.js | 32 ++- .../convert_to_geojson.test.js | 200 ++++++++++++------ .../es_geo_grid_source/es_geo_grid_source.js | 152 +++++++------ 3 files changed, 259 insertions(+), 125 deletions(-) diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js index 4ff6ca69192615..aaf7a66f07d0d0 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js @@ -11,13 +11,39 @@ import { extractPropertiesFromBucket } from '../../util/es_agg_utils'; const GRID_BUCKET_KEYS_TO_IGNORE = ['key', 'gridCentroid']; -export function convertToGeoJson(esResponse, renderAs) { +export function convertCompositeRespToGeoJson(esResponse, renderAs) { + return convertToGeoJson( + esResponse, + renderAs, + esResponse => { + return _.get(esResponse, 'aggregations.compositeSplit.buckets', []); + }, + gridBucket => { + return gridBucket.key.gridSplit; + } + ); +} + +export function convertRegularRespToGeoJson(esResponse, renderAs) { + return convertToGeoJson( + esResponse, + renderAs, + esResponse => { + return _.get(esResponse, 'aggregations.gridSplit.buckets', []); + }, + gridBucket => { + return gridBucket.key; + } + ); +} + +function convertToGeoJson(esResponse, renderAs, pluckGridBuckets, pluckGridKey) { const features = []; - const gridBuckets = _.get(esResponse, 'aggregations.compositeSplit.buckets', []); + const gridBuckets = pluckGridBuckets(esResponse); for (let i = 0; i < gridBuckets.length; i++) { const gridBucket = gridBuckets[i]; - const gridKey = gridBucket.key.gridSplit; + const gridKey = pluckGridKey(gridBucket); features.push({ type: 'Feature', geometry: rowToGeometry({ diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.test.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.test.js index bc0e524aba7be9..09781bf335b759 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.test.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.test.js @@ -4,78 +4,152 @@ * you may not use this file except in compliance with the Elastic License. */ -import { convertToGeoJson } from './convert_to_geojson'; +import { convertCompositeRespToGeoJson, convertRegularRespToGeoJson } from './convert_to_geojson'; import { RENDER_AS } from './render_as'; -const esResponse = { - aggregations: { - compositeSplit: { - after_key: { - gridSplit: '10/327/460', - }, - buckets: [ - { - key: { gridSplit: '4/4/6' }, - doc_count: 65, - avg_of_bytes: { value: 5359.2307692307695 }, - 'terms_of_machine.os.keyword': { - buckets: [ - { - key: 'win xp', - doc_count: 16, - }, - ], - }, - gridCentroid: { - location: { lat: 36.62813963153614, lon: -81.94552666092149 }, - count: 65, - }, +describe('convertCompositeRespToGeoJson', () => { + const esResponse = { + aggregations: { + compositeSplit: { + after_key: { + gridSplit: '10/327/460', }, - ], + buckets: [ + { + key: { gridSplit: '4/4/6' }, + doc_count: 65, + avg_of_bytes: { value: 5359.2307692307695 }, + 'terms_of_machine.os.keyword': { + buckets: [ + { + key: 'win xp', + doc_count: 16, + }, + ], + }, + gridCentroid: { + location: { lat: 36.62813963153614, lon: -81.94552666092149 }, + count: 65, + }, + }, + ], + }, }, - }, -}; + }; -it('Should convert elasticsearch aggregation response into feature collection of points', () => { - const features = convertToGeoJson(esResponse, RENDER_AS.POINT); - expect(features.length).toBe(1); - expect(features[0]).toEqual({ - geometry: { - coordinates: [-81.94552666092149, 36.62813963153614], - type: 'Point', - }, - id: '4/4/6', - properties: { - avg_of_bytes: 5359.2307692307695, - doc_count: 65, - 'terms_of_machine.os.keyword': 'win xp', - }, - type: 'Feature', + it('Should convert elasticsearch aggregation response into feature collection of points', () => { + const features = convertCompositeRespToGeoJson(esResponse, RENDER_AS.POINT); + expect(features.length).toBe(1); + expect(features[0]).toEqual({ + geometry: { + coordinates: [-81.94552666092149, 36.62813963153614], + type: 'Point', + }, + id: '4/4/6', + properties: { + avg_of_bytes: 5359.2307692307695, + doc_count: 65, + 'terms_of_machine.os.keyword': 'win xp', + }, + type: 'Feature', + }); + }); + + it('Should convert elasticsearch aggregation response into feature collection of Polygons', () => { + const features = convertCompositeRespToGeoJson(esResponse, RENDER_AS.GRID); + expect(features.length).toBe(1); + expect(features[0]).toEqual({ + geometry: { + coordinates: [ + [ + [-67.5, 40.9799], + [-90, 40.9799], + [-90, 21.94305], + [-67.5, 21.94305], + [-67.5, 40.9799], + ], + ], + type: 'Polygon', + }, + id: '4/4/6', + properties: { + avg_of_bytes: 5359.2307692307695, + doc_count: 65, + 'terms_of_machine.os.keyword': 'win xp', + }, + type: 'Feature', + }); }); }); -it('Should convert elasticsearch aggregation response into feature collection of Polygons', () => { - const features = convertToGeoJson(esResponse, RENDER_AS.GRID); - expect(features.length).toBe(1); - expect(features[0]).toEqual({ - geometry: { - coordinates: [ - [ - [-67.5, 40.9799], - [-90, 40.9799], - [-90, 21.94305], - [-67.5, 21.94305], - [-67.5, 40.9799], +describe('convertRegularRespToGeoJson', () => { + const esResponse = { + aggregations: { + gridSplit: { + buckets: [ + { + key: '4/4/6', + doc_count: 65, + avg_of_bytes: { value: 5359.2307692307695 }, + 'terms_of_machine.os.keyword': { + buckets: [ + { + key: 'win xp', + doc_count: 16, + }, + ], + }, + gridCentroid: { + location: { lat: 36.62813963153614, lon: -81.94552666092149 }, + count: 65, + }, + }, ], - ], - type: 'Polygon', - }, - id: '4/4/6', - properties: { - avg_of_bytes: 5359.2307692307695, - doc_count: 65, - 'terms_of_machine.os.keyword': 'win xp', + }, }, - type: 'Feature', + }; + + it('Should convert elasticsearch aggregation response into feature collection of points', () => { + const features = convertRegularRespToGeoJson(esResponse, RENDER_AS.POINT); + expect(features.length).toBe(1); + expect(features[0]).toEqual({ + geometry: { + coordinates: [-81.94552666092149, 36.62813963153614], + type: 'Point', + }, + id: '4/4/6', + properties: { + avg_of_bytes: 5359.2307692307695, + doc_count: 65, + 'terms_of_machine.os.keyword': 'win xp', + }, + type: 'Feature', + }); + }); + + it('Should convert elasticsearch aggregation response into feature collection of Polygons', () => { + const features = convertRegularRespToGeoJson(esResponse, RENDER_AS.GRID); + expect(features.length).toBe(1); + expect(features[0]).toEqual({ + geometry: { + coordinates: [ + [ + [-67.5, 40.9799], + [-90, 40.9799], + [-90, 21.94305], + [-67.5, 21.94305], + [-67.5, 40.9799], + ], + ], + type: 'Polygon', + }, + id: '4/4/6', + properties: { + avg_of_bytes: 5359.2307692307695, + doc_count: 65, + 'terms_of_machine.os.keyword': 'win xp', + }, + type: 'Feature', + }); }); }); diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js index 55ab4f33f75b37..238092442a38d2 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js @@ -10,7 +10,7 @@ import uuid from 'uuid/v4'; import { VECTOR_SHAPE_TYPES } from '../vector_feature_types'; import { HeatmapLayer } from '../../heatmap_layer'; import { VectorLayer } from '../../vector_layer'; -import { convertToGeoJson } from './convert_to_geojson'; +import { convertCompositeRespToGeoJson, convertRegularRespToGeoJson } from './convert_to_geojson'; import { VectorStyle } from '../../styles/vector/vector_style'; import { getDefaultDynamicProperties, @@ -175,76 +175,110 @@ export class ESGeoGridSource extends AbstractESAggSource { bucketsPerGrid++; } }); - const gridsPerRequest = Math.floor(DEFAULT_MAX_BUCKETS_LIMIT / bucketsPerGrid); - const aggs = { - compositeSplit: { - composite: { - size: gridsPerRequest, - sources: [ - { - gridSplit: { - geotile_grid: { - field: this._descriptor.geoField, - precision: searchFilters.geogridPrecision, - }, + + const features = []; + // Do not use composite aggregation when there are no terms sub-aggregations + // see https://github.com/elastic/kibana/pull/57875#issuecomment-590515482 for explanation on using separate code paths + if (bucketsPerGrid === 1) { + searchSource.setField('aggs', { + gridSplit: { + geotile_grid: { + field: this._descriptor.geoField, + precision: searchFilters.geogridPrecision, + }, + aggs: { + gridCentroid: { + geo_centroid: { + field: this._descriptor.geoField, }, }, - ], - }, - aggs: { - gridCentroid: { - geo_centroid: { - field: this._descriptor.geoField, - }, + ...this.getValueAggsDsl(), }, - ...this.getValueAggsDsl(indexPattern), }, - }, - }; - - const features = []; - let requestCount = 0; - let afterKey = null; - while (true) { - if (!isRequestStillActive()) { - // Stop paging through results if request is obsolete - throw new DataRequestAbortError(); - } - - requestCount++; - - // circuit breaker to ensure reasonable number of requests - if (requestCount > 5) { - throw new Error( - i18n.translate('xpack.maps.source.esGrid.compositePaginationErrorMessage', { - defaultMessage: `{layerName} is causing too many requests. Reduce "Grid resolution" and/or reduce the number of top term "Metrics".`, - values: { layerName }, - }) - ); - } + }); - if (afterKey) { - aggs.compositeSplit.composite.after = afterKey; - } - searchSource.setField('aggs', aggs); - const requestId = afterKey ? `${this.getId()} afterKey ${afterKey.geoSplit}` : this.getId(); const esResponse = await this._runEsQuery({ - requestId, - requestName: `${layerName} (${requestCount})`, + requestId: this.getId(), + requestName: layerName, searchSource, registerCancelCallback, requestDescription: i18n.translate('xpack.maps.source.esGrid.inspectorDescription', { - defaultMessage: 'Elasticsearch geo grid aggregation request: {requestId}', - values: { requestId }, + defaultMessage: 'Elasticsearch geo grid aggregation request', }), }); - features.push(...convertToGeoJson(esResponse, this._descriptor.requestType)); - - afterKey = esResponse.aggregations.compositeSplit.after_key; - if (esResponse.aggregations.compositeSplit.buckets.length < gridsPerRequest) { - // Finished because request did not get full resultset back - break; + features.push(...convertRegularRespToGeoJson(esResponse, this._descriptor.requestType)); + } else { + const gridsPerRequest = Math.floor(DEFAULT_MAX_BUCKETS_LIMIT / bucketsPerGrid); + const aggs = { + compositeSplit: { + composite: { + size: gridsPerRequest, + sources: [ + { + gridSplit: { + geotile_grid: { + field: this._descriptor.geoField, + precision: searchFilters.geogridPrecision, + }, + }, + }, + ], + }, + aggs: { + gridCentroid: { + geo_centroid: { + field: this._descriptor.geoField, + }, + }, + ...this.getValueAggsDsl(indexPattern), + }, + }, + }; + + let requestCount = 0; + let afterKey = null; + while (true) { + if (!isRequestStillActive()) { + // Stop paging through results if request is obsolete + throw new DataRequestAbortError(); + } + + requestCount++; + + // circuit breaker to ensure reasonable number of requests + if (requestCount > 5) { + throw new Error( + i18n.translate('xpack.maps.source.esGrid.compositePaginationErrorMessage', { + defaultMessage: `{layerName} is causing too many requests. Reduce "Grid resolution" and/or reduce the number of top term "Metrics".`, + values: { layerName }, + }) + ); + } + + if (afterKey) { + aggs.compositeSplit.composite.after = afterKey; + } + searchSource.setField('aggs', aggs); + const requestId = afterKey ? `${this.getId()} afterKey ${afterKey.geoSplit}` : this.getId(); + const esResponse = await this._runEsQuery({ + requestId, + requestName: `${layerName} (${requestCount})`, + searchSource, + registerCancelCallback, + requestDescription: i18n.translate('xpack.maps.source.esGrid.inspectorDescription', { + defaultMessage: 'Elasticsearch geo grid aggregation request: {requestId}', + values: { requestId }, + }), + }); + + features.push(...convertCompositeRespToGeoJson(esResponse, this._descriptor.requestType)); + + afterKey = esResponse.aggregations.compositeSplit.after_key; + if (esResponse.aggregations.compositeSplit.buckets.length < gridsPerRequest) { + // Finished because request did not get full resultset back + break; + } } } From 1aae2ea15a6163bea850c4cc1009fd359e65630a Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Mon, 24 Feb 2020 16:14:03 -0700 Subject: [PATCH 23/29] clean up --- .../sources/es_geo_grid_source/es_geo_grid_source.js | 11 +++++++---- .../test/functional/apps/maps/embeddable/dashboard.js | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js index 238092442a38d2..5b5b2b21e491b1 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js @@ -266,10 +266,13 @@ export class ESGeoGridSource extends AbstractESAggSource { requestName: `${layerName} (${requestCount})`, searchSource, registerCancelCallback, - requestDescription: i18n.translate('xpack.maps.source.esGrid.inspectorDescription', { - defaultMessage: 'Elasticsearch geo grid aggregation request: {requestId}', - values: { requestId }, - }), + requestDescription: i18n.translate( + 'xpack.maps.source.esGrid.compositeInspectorDescription', + { + defaultMessage: 'Elasticsearch geo grid aggregation request: {requestId}', + values: { requestId }, + } + ), }); features.push(...convertCompositeRespToGeoJson(esResponse, this._descriptor.requestType)); diff --git a/x-pack/test/functional/apps/maps/embeddable/dashboard.js b/x-pack/test/functional/apps/maps/embeddable/dashboard.js index 2baf7994ce9ffb..43d5cccb209058 100644 --- a/x-pack/test/functional/apps/maps/embeddable/dashboard.js +++ b/x-pack/test/functional/apps/maps/embeddable/dashboard.js @@ -55,7 +55,7 @@ export default function({ getPageObjects, getService }) { await dashboardPanelActions.openInspectorByTitle('geo grid vector grid example'); const gridExampleRequestNames = await inspector.getRequestNames(); await inspector.close(); - expect(gridExampleRequestNames).to.equal('logstash-* (1)'); + expect(gridExampleRequestNames).to.equal('logstash-*'); }); it('should apply container state (time, query, filters) to embeddable when loaded', async () => { From a126388ea34d3d7841c554256a0c742a87a54c07 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Mon, 24 Feb 2020 19:10:29 -0700 Subject: [PATCH 24/29] pass indexPattern to getValueAggsDsl --- .../layers/sources/es_geo_grid_source/es_geo_grid_source.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js index 5b5b2b21e491b1..ae1b1ebaa45575 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js @@ -192,7 +192,7 @@ export class ESGeoGridSource extends AbstractESAggSource { field: this._descriptor.geoField, }, }, - ...this.getValueAggsDsl(), + ...this.getValueAggsDsl(indexPattern), }, }, }); From 7a0ffa8459cc2304976911dce8cf17a5e3da5808 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 25 Feb 2020 11:11:48 -0700 Subject: [PATCH 25/29] review feedback --- ...son.test.js => convert_to_geojson.test.ts} | 0 .../es_geo_grid_source/es_geo_grid_source.js | 249 ++++++++++-------- .../es_geo_grid_source/geo_tile_utils.js | 14 - ...lines.test.js => convert_to_lines.test.ts} | 0 4 files changed, 143 insertions(+), 120 deletions(-) rename x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/{convert_to_geojson.test.js => convert_to_geojson.test.ts} (100%) rename x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/{convert_to_lines.test.js => convert_to_lines.test.ts} (100%) diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.test.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.test.ts similarity index 100% rename from x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.test.js rename to x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.test.ts diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js index ae1b1ebaa45575..9906c10ede001b 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js @@ -164,126 +164,163 @@ export class ESGeoGridSource extends AbstractESAggSource { ); } - async getGeoJsonWithMeta(layerName, searchFilters, registerCancelCallback, isRequestStillActive) { - const indexPattern = await this.getIndexPattern(); - const searchSource = await this._makeSearchSource(searchFilters, 0); - - let bucketsPerGrid = 1; - this.getMetricFields().forEach(metricField => { - if (metricField.getAggType() === AGG_TYPE.TERMS) { - // each terms aggregation increases the overall number of buckets per grid - bucketsPerGrid++; - } - }); - - const features = []; - // Do not use composite aggregation when there are no terms sub-aggregations - // see https://github.com/elastic/kibana/pull/57875#issuecomment-590515482 for explanation on using separate code paths - if (bucketsPerGrid === 1) { - searchSource.setField('aggs', { - gridSplit: { - geotile_grid: { - field: this._descriptor.geoField, - precision: searchFilters.geogridPrecision, - }, - aggs: { - gridCentroid: { - geo_centroid: { - field: this._descriptor.geoField, + async compositeAggRequest({ + searchSource, + indexPattern, + precision, + layerName, + registerCancelCallback, + bucketsPerGrid, + isRequestStillActive, + }) { + const gridsPerRequest = Math.floor(DEFAULT_MAX_BUCKETS_LIMIT / bucketsPerGrid); + const aggs = { + compositeSplit: { + composite: { + size: gridsPerRequest, + sources: [ + { + gridSplit: { + geotile_grid: { + field: this._descriptor.geoField, + precision, + }, }, }, - ...this.getValueAggsDsl(indexPattern), + ], + }, + aggs: { + gridCentroid: { + geo_centroid: { + field: this._descriptor.geoField, + }, }, + ...this.getValueAggsDsl(indexPattern), }, - }); + }, + }; + + const features = []; + let requestCount = 0; + let afterKey = null; + while (true) { + if (!isRequestStillActive()) { + // Stop paging through results if request is obsolete + throw new DataRequestAbortError(); + } + + requestCount++; + + // circuit breaker to ensure reasonable number of requests + if (requestCount > 5) { + throw new Error( + i18n.translate('xpack.maps.source.esGrid.compositePaginationErrorMessage', { + defaultMessage: `{layerName} is causing too many requests. Reduce "Grid resolution" and/or reduce the number of top term "Metrics".`, + values: { layerName }, + }) + ); + } + if (afterKey) { + aggs.compositeSplit.composite.after = afterKey; + } + searchSource.setField('aggs', aggs); + const requestId = afterKey ? `${this.getId()} afterKey ${afterKey.geoSplit}` : this.getId(); const esResponse = await this._runEsQuery({ - requestId: this.getId(), - requestName: layerName, + requestId, + requestName: `${layerName} (${requestCount})`, searchSource, registerCancelCallback, - requestDescription: i18n.translate('xpack.maps.source.esGrid.inspectorDescription', { - defaultMessage: 'Elasticsearch geo grid aggregation request', - }), + requestDescription: i18n.translate( + 'xpack.maps.source.esGrid.compositeInspectorDescription', + { + defaultMessage: 'Elasticsearch geo grid aggregation request: {requestId}', + values: { requestId }, + } + ), }); - features.push(...convertRegularRespToGeoJson(esResponse, this._descriptor.requestType)); - } else { - const gridsPerRequest = Math.floor(DEFAULT_MAX_BUCKETS_LIMIT / bucketsPerGrid); - const aggs = { - compositeSplit: { - composite: { - size: gridsPerRequest, - sources: [ - { - gridSplit: { - geotile_grid: { - field: this._descriptor.geoField, - precision: searchFilters.geogridPrecision, - }, - }, - }, - ], - }, - aggs: { - gridCentroid: { - geo_centroid: { - field: this._descriptor.geoField, - }, + features.push(...convertCompositeRespToGeoJson(esResponse, this._descriptor.requestType)); + + afterKey = esResponse.aggregations.compositeSplit.after_key; + if (esResponse.aggregations.compositeSplit.buckets.length < gridsPerRequest) { + // Finished because request did not get full resultset back + break; + } + } + + return features; + } + + // Do not use composite aggregation when there are no terms sub-aggregations + // see https://github.com/elastic/kibana/pull/57875#issuecomment-590515482 for explanation on using separate code paths + async nonCompositeAggRequest({ + searchSource, + indexPattern, + precision, + layerName, + registerCancelCallback, + }) { + searchSource.setField('aggs', { + gridSplit: { + geotile_grid: { + field: this._descriptor.geoField, + precision, + }, + aggs: { + gridCentroid: { + geo_centroid: { + field: this._descriptor.geoField, }, - ...this.getValueAggsDsl(indexPattern), }, + ...this.getValueAggsDsl(indexPattern), }, - }; - - let requestCount = 0; - let afterKey = null; - while (true) { - if (!isRequestStillActive()) { - // Stop paging through results if request is obsolete - throw new DataRequestAbortError(); - } - - requestCount++; - - // circuit breaker to ensure reasonable number of requests - if (requestCount > 5) { - throw new Error( - i18n.translate('xpack.maps.source.esGrid.compositePaginationErrorMessage', { - defaultMessage: `{layerName} is causing too many requests. Reduce "Grid resolution" and/or reduce the number of top term "Metrics".`, - values: { layerName }, - }) - ); - } - - if (afterKey) { - aggs.compositeSplit.composite.after = afterKey; - } - searchSource.setField('aggs', aggs); - const requestId = afterKey ? `${this.getId()} afterKey ${afterKey.geoSplit}` : this.getId(); - const esResponse = await this._runEsQuery({ - requestId, - requestName: `${layerName} (${requestCount})`, - searchSource, - registerCancelCallback, - requestDescription: i18n.translate( - 'xpack.maps.source.esGrid.compositeInspectorDescription', - { - defaultMessage: 'Elasticsearch geo grid aggregation request: {requestId}', - values: { requestId }, - } - ), - }); - - features.push(...convertCompositeRespToGeoJson(esResponse, this._descriptor.requestType)); - - afterKey = esResponse.aggregations.compositeSplit.after_key; - if (esResponse.aggregations.compositeSplit.buckets.length < gridsPerRequest) { - // Finished because request did not get full resultset back - break; - } + }, + }); + + const esResponse = await this._runEsQuery({ + requestId: this.getId(), + requestName: layerName, + searchSource, + registerCancelCallback, + requestDescription: i18n.translate('xpack.maps.source.esGrid.inspectorDescription', { + defaultMessage: 'Elasticsearch geo grid aggregation request', + }), + }); + + return convertRegularRespToGeoJson(esResponse, this._descriptor.requestType); + } + + async getGeoJsonWithMeta(layerName, searchFilters, registerCancelCallback, isRequestStillActive) { + const indexPattern = await this.getIndexPattern(); + const searchSource = await this._makeSearchSource(searchFilters, 0); + + let bucketsPerGrid = 1; + this.getMetricFields().forEach(metricField => { + if (metricField.getAggType() === AGG_TYPE.TERMS) { + // each terms aggregation increases the overall number of buckets per grid + bucketsPerGrid++; } - } + }); + + const features = + bucketsPerGrid === 1 + ? await this.nonCompositeAggRequest({ + searchSource, + indexPattern, + precision: searchFilters.geogridPrecision, + layerName, + registerCancelCallback, + }) + : await this.compositeAggRequest({ + searchSource, + indexPattern, + precision: searchFilters.geogridPrecision, + layerName, + registerCancelCallback, + bucketsPerGrid, + isRequestStillActive, + }); return { data: { diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/geo_tile_utils.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/geo_tile_utils.js index e1d998ea8d506f..73039bf3bfc9f4 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/geo_tile_utils.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/geo_tile_utils.js @@ -118,17 +118,3 @@ export function expandToTileBoundaries(extent, zoom) { maxLat: tileToLatitude(upperLeftY, tileCount), }; } - -export function getTileCountInExtent(extent, zoom) { - const tileCount = getTileCount(zoom); - - const upperLeftX = longitudeToTile(extent.minLon, tileCount); - const upperLeftY = latitudeToTile(Math.min(extent.maxLat, 90), tileCount); - const lowerRightX = longitudeToTile(extent.maxLon, tileCount); - const lowerRightY = latitudeToTile(Math.max(extent.minLat, -90), tileCount); - - const tilesY = Math.abs(upperLeftY - lowerRightY); - const tilesX = Math.abs(upperLeftX - lowerRightX); - - return tilesX * tilesY; -} diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.test.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.test.ts similarity index 100% rename from x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.test.js rename to x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.test.ts From 79b5a3a04987887531e3a3d7ed4f62f197f4f1b1 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 25 Feb 2020 11:32:57 -0700 Subject: [PATCH 26/29] more review feedback --- .../public/connected_components/map/mb/view.js | 15 +++++---------- .../maps/public/elasticsearch_geo_utils.js | 18 ++++++++++++++++++ .../es_geo_grid_source/convert_to_geojson.js | 13 ++++--------- .../convert_to_geojson.test.ts | 2 ++ .../es_geo_grid_source/geo_tile_utils.js | 9 ++------- .../es_geo_grid_source/geo_tile_utils.test.js | 2 ++ 6 files changed, 33 insertions(+), 26 deletions(-) diff --git a/x-pack/legacy/plugins/maps/public/connected_components/map/mb/view.js b/x-pack/legacy/plugins/maps/public/connected_components/map/mb/view.js index 1e44c7225a564c..fdc8ad2176d085 100644 --- a/x-pack/legacy/plugins/maps/public/connected_components/map/mb/view.js +++ b/x-pack/legacy/plugins/maps/public/connected_components/map/mb/view.js @@ -23,6 +23,7 @@ import sprites1 from '@elastic/maki/dist/sprite@1.png'; import sprites2 from '@elastic/maki/dist/sprite@2.png'; import { DrawControl } from './draw_control'; import { TooltipControl } from './tooltip_control'; +import { clampToLatBounds, clampToLonBounds } from '../../../elasticsearch_geo_utils'; mapboxgl.workerUrl = mbWorkerUrl; mapboxgl.setRTLTextPlugin(mbRtlPlugin); @@ -234,12 +235,12 @@ export class MBMapContainer extends React.Component { //clamping ot -89/89 latitudes since Mapboxgl does not seem to handle bounds that contain the poles (logs errors to the console when using -90/90) const lnLatBounds = new mapboxgl.LngLatBounds( new mapboxgl.LngLat( - clamp(goto.bounds.min_lon, -180, 180), - clamp(goto.bounds.min_lat, -89, 89) + clampToLonBounds(goto.bounds.min_lon), + clampToLatBounds(goto.bounds.min_lat) ), new mapboxgl.LngLat( - clamp(goto.bounds.max_lon, -180, 180), - clamp(goto.bounds.max_lat, -89, 89) + clampToLonBounds(goto.bounds.max_lon), + clampToLatBounds(goto.bounds.max_lat) ) ); //maxZoom ensure we're not zooming in too far on single points or small shapes @@ -306,9 +307,3 @@ export class MBMapContainer extends React.Component { ); } } - -function clamp(val, min, max) { - if (val > max) val = max; - else if (val < min) val = min; - return val; -} diff --git a/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.js b/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.js index ec0ae4161b3f2c..9b33d3036785c9 100644 --- a/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.js +++ b/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.js @@ -433,3 +433,21 @@ export function convertMapExtentToPolygon({ maxLat, maxLon, minLat, minLon }) { return formatEnvelopeAsPolygon({ maxLat, maxLon, minLat, minLon }); } + +export function clampToLatBounds(lat) { + return clamp(lat, -89, 89); +} + +export function clampToLonBounds(lon) { + return clamp(lon, -180, 180); +} + +export function clamp(val, min, max) { + if (val > max) { + return max; + } else if (val < min) { + return min; + } else { + return val; + } +} diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js index aaf7a66f07d0d0..bb9bf1b508f941 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js @@ -8,6 +8,7 @@ import _ from 'lodash'; import { RENDER_AS } from './render_as'; import { getTileBoundingBox } from './geo_tile_utils'; import { extractPropertiesFromBucket } from '../../util/es_agg_utils'; +import { clamp } from '../../../elasticsearch_geo_utils'; const GRID_BUCKET_KEYS_TO_IGNORE = ['key', 'gridCentroid']; @@ -77,10 +78,10 @@ function rowToGeometry({ gridKey, gridCentroid, renderAs }) { }; } - // see https://github.com/elastic/elasticsearch/issues/24694 for why clampGrid is used + // see https://github.com/elastic/elasticsearch/issues/24694 for why clamp is used const pointCoordinates = [ - clampGrid(gridCentroid.location.lon, left, right), - clampGrid(gridCentroid.location.lat, bottom, top), + clamp(gridCentroid.location.lon, left, right), + clamp(gridCentroid.location.lat, bottom, top), ]; return { @@ -88,9 +89,3 @@ function rowToGeometry({ gridKey, gridCentroid, renderAs }) { coordinates: pointCoordinates, }; } - -function clampGrid(val, min, max) { - if (val > max) val = max; - else if (val < min) val = min; - return val; -} diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.test.ts b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.test.ts index 09781bf335b759..0546e5de810b38 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.test.ts +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.test.ts @@ -4,6 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ +jest.mock('../../../kibana_services', () => {}); + import { convertCompositeRespToGeoJson, convertRegularRespToGeoJson } from './convert_to_geojson'; import { RENDER_AS } from './render_as'; diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/geo_tile_utils.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/geo_tile_utils.js index 73039bf3bfc9f4..251e33b9579cbc 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/geo_tile_utils.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/geo_tile_utils.js @@ -6,6 +6,7 @@ import _ from 'lodash'; import { DECIMAL_DEGREES_PRECISION } from '../../../../common/constants'; +import { clampToLatBounds } from '../../../elasticsearch_geo_utils'; const ZOOM_TILE_KEY_INDEX = 0; const X_TILE_KEY_INDEX = 1; @@ -87,13 +88,7 @@ function sec(value) { } function latitudeToTile(lat, tileCount) { - let boundedLat = lat; - if (lat >= 90) { - boundedLat = 89.9; - } else if (lat <= -90) { - boundedLat = -89.9; - } - const radians = (boundedLat * Math.PI) / 180; + const radians = (clampToLatBounds(lat) * Math.PI) / 180; const y = ((1 - Math.log(Math.tan(radians) + sec(radians)) / Math.PI) / 2) * tileCount; return Math.floor(y); } diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/geo_tile_utils.test.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/geo_tile_utils.test.js index ae2623e1687663..88a6ce048a1788 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/geo_tile_utils.test.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/geo_tile_utils.test.js @@ -4,6 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ +jest.mock('../../../kibana_services', () => {}); + import { parseTileKey, getTileBoundingBox, expandToTileBoundaries } from './geo_tile_utils'; it('Should parse tile key', () => { From ac8c04b40a7ee034e17bd684a2d3b559ad844f0f Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 25 Feb 2020 13:03:00 -0700 Subject: [PATCH 27/29] ts-ignore for untyped imports in tests --- .../sources/es_geo_grid_source/convert_to_geojson.test.ts | 2 ++ .../layers/sources/es_pew_pew_source/convert_to_lines.test.ts | 1 + 2 files changed, 3 insertions(+) diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.test.ts b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.test.ts index 0546e5de810b38..ba79464a01a9bf 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.test.ts +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.test.ts @@ -6,7 +6,9 @@ jest.mock('../../../kibana_services', () => {}); +// @ts-ignore import { convertCompositeRespToGeoJson, convertRegularRespToGeoJson } from './convert_to_geojson'; +// @ts-ignore import { RENDER_AS } from './render_as'; describe('convertCompositeRespToGeoJson', () => { diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.test.ts b/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.test.ts index 54f7c9d0bd84df..5fbd5a3ad20c0e 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.test.ts +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.test.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +// @ts-ignore import { convertToLines } from './convert_to_lines'; const esResponse = { From 122491a2ccd867bf7d0c35a86b540a1c6ef99d5e Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 25 Feb 2020 17:04:26 -0700 Subject: [PATCH 28/29] more review feedback --- x-pack/legacy/plugins/maps/public/actions/map_actions.js | 4 ++-- .../sources/es_geo_grid_source/es_geo_grid_source.js | 8 ++++---- .../util/{es_agg_utils.test.js => es_agg_utils.test.ts} | 0 .../legacy/plugins/maps/public/selectors/map_selectors.js | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) rename x-pack/legacy/plugins/maps/public/layers/util/{es_agg_utils.test.js => es_agg_utils.test.ts} (100%) diff --git a/x-pack/legacy/plugins/maps/public/actions/map_actions.js b/x-pack/legacy/plugins/maps/public/actions/map_actions.js index 607185a30c77f0..cfca044ea759a1 100644 --- a/x-pack/legacy/plugins/maps/public/actions/map_actions.js +++ b/x-pack/legacy/plugins/maps/public/actions/map_actions.js @@ -18,7 +18,7 @@ import { getTransientLayerId, getOpenTooltips, getQuery, - getDataRequest, + getDataRequestDescriptor, } from '../selectors/map_selectors'; import { FLYOUT_STATE } from '../reducers/ui'; import { @@ -89,7 +89,7 @@ function getLayerLoadingCallbacks(dispatch, getState, layerId) { dispatch(updateSourceDataRequest(layerId, newData)); }, isRequestStillActive: (dataId, requestToken) => { - const dataRequest = getDataRequest(getState(), layerId, dataId); + const dataRequest = getDataRequestDescriptor(getState(), layerId, dataId); if (!dataRequest) { return false; } diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js index 9906c10ede001b..a0ddf584bcebc1 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js @@ -164,7 +164,7 @@ export class ESGeoGridSource extends AbstractESAggSource { ); } - async compositeAggRequest({ + async _compositeAggRequest({ searchSource, indexPattern, precision, @@ -254,7 +254,7 @@ export class ESGeoGridSource extends AbstractESAggSource { // Do not use composite aggregation when there are no terms sub-aggregations // see https://github.com/elastic/kibana/pull/57875#issuecomment-590515482 for explanation on using separate code paths - async nonCompositeAggRequest({ + async _nonCompositeAggRequest({ searchSource, indexPattern, precision, @@ -305,14 +305,14 @@ export class ESGeoGridSource extends AbstractESAggSource { const features = bucketsPerGrid === 1 - ? await this.nonCompositeAggRequest({ + ? await this._nonCompositeAggRequest({ searchSource, indexPattern, precision: searchFilters.geogridPrecision, layerName, registerCancelCallback, }) - : await this.compositeAggRequest({ + : await this._compositeAggRequest({ searchSource, indexPattern, precision: searchFilters.geogridPrecision, diff --git a/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.test.js b/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.test.ts similarity index 100% rename from x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.test.js rename to x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.test.ts diff --git a/x-pack/legacy/plugins/maps/public/selectors/map_selectors.js b/x-pack/legacy/plugins/maps/public/selectors/map_selectors.js index b554b459f51b4d..4074344916390c 100644 --- a/x-pack/legacy/plugins/maps/public/selectors/map_selectors.js +++ b/x-pack/legacy/plugins/maps/public/selectors/map_selectors.js @@ -125,12 +125,12 @@ export const getRefreshConfig = ({ map }) => { export const getRefreshTimerLastTriggeredAt = ({ map }) => map.mapState.refreshTimerLastTriggeredAt; -export function getLayerDescriptor(state = {}, layerId) { +function getLayerDescriptor(state = {}, layerId) { const layerListRaw = getLayerListRaw(state); return layerListRaw.find(layer => layer.id === layerId); } -export function getDataRequest(state = {}, layerId, dataId) { +export function getDataRequestDescriptor(state = {}, layerId, dataId) { const layerDescriptor = getLayerDescriptor(state, layerId); if (!layerDescriptor || !layerDescriptor.__dataRequests) { return; From 104acf6864391baa85835a31df7664949fc965df Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 25 Feb 2020 18:27:40 -0700 Subject: [PATCH 29/29] add bucket.hasOwnProperty check --- x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.ts b/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.ts index 933b4d6b613383..7af176acfaf467 100644 --- a/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.ts +++ b/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.ts @@ -35,7 +35,7 @@ export function addFieldToDSL(dsl: object, field: IFieldType) { export function extractPropertiesFromBucket(bucket: any, ignoreKeys: string[] = []) { const properties: Record = {}; for (const key in bucket) { - if (ignoreKeys.includes(key)) { + if (ignoreKeys.includes(key) || !bucket.hasOwnProperty(key)) { continue; }